app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19 #include <rte_interrupts.h>
  20
  21 #include "main.h"
  22 #include "test_bbdev_vector.h"
  23
  24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  25
  26 #define MAX_QUEUES RTE_MAX_LCORE
  27 #define TEST_REPETITIONS 1000
  28
  29 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
  30 #include <fpga_lte_fec.h>
  31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
  32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
  33 #define VF_UL_4G_QUEUE_VALUE 4
  34 #define VF_DL_4G_QUEUE_VALUE 4
  35 #define UL_4G_BANDWIDTH 3
  36 #define DL_4G_BANDWIDTH 3
  37 #define UL_4G_LOAD_BALANCE 128
  38 #define DL_4G_LOAD_BALANCE 128
  39 #define FLR_4G_TIMEOUT 610
  40 #endif
  41
  42 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
  43 #include <rte_pmd_fpga_5gnr_fec.h>
  44 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
  45 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
  46 #define VF_UL_5G_QUEUE_VALUE 4
  47 #define VF_DL_5G_QUEUE_VALUE 4
  48 #define UL_5G_BANDWIDTH 3
  49 #define DL_5G_BANDWIDTH 3
  50 #define UL_5G_LOAD_BALANCE 128
  51 #define DL_5G_LOAD_BALANCE 128
  52 #define FLR_5G_TIMEOUT 610
  53 #endif
  54
  55 #ifdef RTE_BASEBAND_ACC100
  56 #include <rte_acc100_cfg.h>
  57 #define ACC100PF_DRIVER_NAME   ("intel_acc100_pf")
  58 #define ACC100VF_DRIVER_NAME   ("intel_acc100_vf")
  59 #define ACC100_QMGR_NUM_AQS 16
  60 #define ACC100_QMGR_NUM_QGS 2
  61 #define ACC100_QMGR_AQ_DEPTH 5
  62 #define ACC100_QMGR_INVALID_IDX -1
  63 #define ACC100_QMGR_RR 1
  64 #define ACC100_QOS_GBR 0
  65 #endif
  66
  67 #define OPS_CACHE_SIZE 256U
  68 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  69
  70 #define SYNC_WAIT 0
  71 #define SYNC_START 1
  72 #define INVALID_OPAQUE -1
  73
  74 #define INVALID_QUEUE_ID -1
  75 /* Increment for next code block in external HARQ memory */
  76 #define HARQ_INCR 32768
  77 /* Headroom for filler LLRs insertion in HARQ buffer */
  78 #define FILLER_HEADROOM 1024
  79 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
  80 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
  81 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
  82 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
  83 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
  84 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
  85 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
  86 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
  87 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
  88
  89 static struct test_bbdev_vector test_vector;
  90
  91 /* Switch between PMD and Interrupt for throughput TC */
  92 static bool intr_enabled;
  93
  94 /* LLR arithmetic representation for numerical conversion */
  95 static int ldpc_llr_decimals;
  96 static int ldpc_llr_size;
  97 /* Keep track of the LDPC decoder device capability flag */
  98 static uint32_t ldpc_cap_flags;
  99
 100 /* Represents tested active devices */
 101 static struct active_device {
 102         const char *driver_name;
 103         uint8_t dev_id;
 104         uint16_t supported_ops;
 105         uint16_t queue_ids[MAX_QUEUES];
 106         uint16_t nb_queues;
 107         struct rte_mempool *ops_mempool;
 108         struct rte_mempool *in_mbuf_pool;
 109         struct rte_mempool *hard_out_mbuf_pool;
 110         struct rte_mempool *soft_out_mbuf_pool;
 111         struct rte_mempool *harq_in_mbuf_pool;
 112         struct rte_mempool *harq_out_mbuf_pool;
 113 } active_devs[RTE_BBDEV_MAX_DEVS];
 114
 115 static uint8_t nb_active_devs;
 116
 117 /* Data buffers used by BBDEV ops */
 118 struct test_buffers {
 119         struct rte_bbdev_op_data *inputs;
 120         struct rte_bbdev_op_data *hard_outputs;
 121         struct rte_bbdev_op_data *soft_outputs;
 122         struct rte_bbdev_op_data *harq_inputs;
 123         struct rte_bbdev_op_data *harq_outputs;
 124 };
 125
 126 /* Operation parameters specific for given test case */
 127 struct test_op_params {
 128         struct rte_mempool *mp;
 129         struct rte_bbdev_dec_op *ref_dec_op;
 130         struct rte_bbdev_enc_op *ref_enc_op;
 131         uint16_t burst_sz;
 132         uint16_t num_to_process;
 133         uint16_t num_lcores;
 134         int vector_mask;
 135         rte_atomic16_t sync;
 136         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
 137 };
 138
 139 /* Contains per lcore params */
 140 struct thread_params {
 141         uint8_t dev_id;
 142         uint16_t queue_id;
 143         uint32_t lcore_id;
 144         uint64_t start_time;
 145         double ops_per_sec;
 146         double mbps;
 147         uint8_t iter_count;
 148         double iter_average;
 149         double bler;
 150         rte_atomic16_t nb_dequeued;
 151         rte_atomic16_t processing_status;
 152         rte_atomic16_t burst_sz;
 153         struct test_op_params *op_params;
 154         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
 155         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
 156 };
 157
 158 #ifdef RTE_BBDEV_OFFLOAD_COST
 159 /* Stores time statistics */
 160 struct test_time_stats {
 161         /* Stores software enqueue total working time */
 162         uint64_t enq_sw_total_time;
 163         /* Stores minimum value of software enqueue working time */
 164         uint64_t enq_sw_min_time;
 165         /* Stores maximum value of software enqueue working time */
 166         uint64_t enq_sw_max_time;
 167         /* Stores turbo enqueue total working time */
 168         uint64_t enq_acc_total_time;
 169         /* Stores minimum value of accelerator enqueue working time */
 170         uint64_t enq_acc_min_time;
 171         /* Stores maximum value of accelerator enqueue working time */
 172         uint64_t enq_acc_max_time;
 173         /* Stores dequeue total working time */
 174         uint64_t deq_total_time;
 175         /* Stores minimum value of dequeue working time */
 176         uint64_t deq_min_time;
 177         /* Stores maximum value of dequeue working time */
 178         uint64_t deq_max_time;
 179 };
 180 #endif
 181
 182 typedef int (test_case_function)(struct active_device *ad,
 183                 struct test_op_params *op_params);
 184
 185 static inline void
 186 mbuf_reset(struct rte_mbuf *m)
 187 {
 188         m->pkt_len = 0;
 189
 190         do {
 191                 m->data_len = 0;
 192                 m = m->next;
 193         } while (m != NULL);
 194 }
 195
 196 /* Read flag value 0/1 from bitmap */
 197 static inline bool
 198 check_bit(uint32_t bitmap, uint32_t bitmask)
 199 {
 200         return bitmap & bitmask;
 201 }
 202
 203 static inline void
 204 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 205 {
 206         ad->supported_ops |= (1 << op_type);
 207 }
 208
 209 static inline bool
 210 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 211 {
 212         return ad->supported_ops & (1 << op_type);
 213 }
 214
 215 static inline bool
 216 flags_match(uint32_t flags_req, uint32_t flags_present)
 217 {
 218         return (flags_req & flags_present) == flags_req;
 219 }
 220
 221 static void
 222 clear_soft_out_cap(uint32_t *op_flags)
 223 {
 224         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 225         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 226         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 227 }
 228
 229 static int
 230 check_dev_cap(const struct rte_bbdev_info *dev_info)
 231 {
 232         unsigned int i;
 233         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
 234                 nb_harq_inputs, nb_harq_outputs;
 235         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 236
 237         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 238         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 239         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 240         nb_harq_inputs  = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
 241         nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
 242
 243         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 244                 if (op_cap->type != test_vector.op_type)
 245                         continue;
 246
 247                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 248                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 249                                         &op_cap->cap.turbo_dec;
 250                         /* Ignore lack of soft output capability, just skip
 251                          * checking if soft output is valid.
 252                          */
 253                         if ((test_vector.turbo_dec.op_flags &
 254                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 255                                         !(cap->capability_flags &
 256                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 257                                 printf(
 258                                         "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 259                                         dev_info->dev_name);
 260                                 clear_soft_out_cap(
 261                                         &test_vector.turbo_dec.op_flags);
 262                         }
 263
 264                         if (!flags_match(test_vector.turbo_dec.op_flags,
 265                                         cap->capability_flags))
 266                                 return TEST_FAILED;
 267                         if (nb_inputs > cap->num_buffers_src) {
 268                                 printf("Too many inputs defined: %u, max: %u\n",
 269                                         nb_inputs, cap->num_buffers_src);
 270                                 return TEST_FAILED;
 271                         }
 272                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 273                                         (test_vector.turbo_dec.op_flags &
 274                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 275                                 printf(
 276                                         "Too many soft outputs defined: %u, max: %u\n",
 277                                                 nb_soft_outputs,
 278                                                 cap->num_buffers_soft_out);
 279                                 return TEST_FAILED;
 280                         }
 281                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 282                                 printf(
 283                                         "Too many hard outputs defined: %u, max: %u\n",
 284                                                 nb_hard_outputs,
 285                                                 cap->num_buffers_hard_out);
 286                                 return TEST_FAILED;
 287                         }
 288                         if (intr_enabled && !(cap->capability_flags &
 289                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 290                                 printf(
 291                                         "Dequeue interrupts are not supported!\n");
 292                                 return TEST_FAILED;
 293                         }
 294
 295                         return TEST_SUCCESS;
 296                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 297                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 298                                         &op_cap->cap.turbo_enc;
 299
 300                         if (!flags_match(test_vector.turbo_enc.op_flags,
 301                                         cap->capability_flags))
 302                                 return TEST_FAILED;
 303                         if (nb_inputs > cap->num_buffers_src) {
 304                                 printf("Too many inputs defined: %u, max: %u\n",
 305                                         nb_inputs, cap->num_buffers_src);
 306                                 return TEST_FAILED;
 307                         }
 308                         if (nb_hard_outputs > cap->num_buffers_dst) {
 309                                 printf(
 310                                         "Too many hard outputs defined: %u, max: %u\n",
 311                                         nb_hard_outputs, cap->num_buffers_dst);
 312                                 return TEST_FAILED;
 313                         }
 314                         if (intr_enabled && !(cap->capability_flags &
 315                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 316                                 printf(
 317                                         "Dequeue interrupts are not supported!\n");
 318                                 return TEST_FAILED;
 319                         }
 320
 321                         return TEST_SUCCESS;
 322                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
 323                         const struct rte_bbdev_op_cap_ldpc_enc *cap =
 324                                         &op_cap->cap.ldpc_enc;
 325
 326                         if (!flags_match(test_vector.ldpc_enc.op_flags,
 327                                         cap->capability_flags)){
 328                                 printf("Flag Mismatch\n");
 329                                 return TEST_FAILED;
 330                         }
 331                         if (nb_inputs > cap->num_buffers_src) {
 332                                 printf("Too many inputs defined: %u, max: %u\n",
 333                                         nb_inputs, cap->num_buffers_src);
 334                                 return TEST_FAILED;
 335                         }
 336                         if (nb_hard_outputs > cap->num_buffers_dst) {
 337                                 printf(
 338                                         "Too many hard outputs defined: %u, max: %u\n",
 339                                         nb_hard_outputs, cap->num_buffers_dst);
 340                                 return TEST_FAILED;
 341                         }
 342                         if (intr_enabled && !(cap->capability_flags &
 343                                         RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
 344                                 printf(
 345                                         "Dequeue interrupts are not supported!\n");
 346                                 return TEST_FAILED;
 347                         }
 348
 349                         return TEST_SUCCESS;
 350                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
 351                         const struct rte_bbdev_op_cap_ldpc_dec *cap =
 352                                         &op_cap->cap.ldpc_dec;
 353
 354                         if (!flags_match(test_vector.ldpc_dec.op_flags,
 355                                         cap->capability_flags)){
 356                                 printf("Flag Mismatch\n");
 357                                 return TEST_FAILED;
 358                         }
 359                         if (nb_inputs > cap->num_buffers_src) {
 360                                 printf("Too many inputs defined: %u, max: %u\n",
 361                                         nb_inputs, cap->num_buffers_src);
 362                                 return TEST_FAILED;
 363                         }
 364                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 365                                 printf(
 366                                         "Too many hard outputs defined: %u, max: %u\n",
 367                                         nb_hard_outputs,
 368                                         cap->num_buffers_hard_out);
 369                                 return TEST_FAILED;
 370                         }
 371                         if (nb_harq_inputs > cap->num_buffers_hard_out) {
 372                                 printf(
 373                                         "Too many HARQ inputs defined: %u, max: %u\n",
 374                                         nb_hard_outputs,
 375                                         cap->num_buffers_hard_out);
 376                                 return TEST_FAILED;
 377                         }
 378                         if (nb_harq_outputs > cap->num_buffers_hard_out) {
 379                                 printf(
 380                                         "Too many HARQ outputs defined: %u, max: %u\n",
 381                                         nb_hard_outputs,
 382                                         cap->num_buffers_hard_out);
 383                                 return TEST_FAILED;
 384                         }
 385                         if (intr_enabled && !(cap->capability_flags &
 386                                         RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
 387                                 printf(
 388                                         "Dequeue interrupts are not supported!\n");
 389                                 return TEST_FAILED;
 390                         }
 391                         if (intr_enabled && (test_vector.ldpc_dec.op_flags &
 392                                 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
 393                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
 394                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
 395                                         ))) {
 396                                 printf("Skip loop-back with interrupt\n");
 397                                 return TEST_FAILED;
 398                         }
 399                         return TEST_SUCCESS;
 400                 }
 401         }
 402
 403         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 404                 return TEST_SUCCESS; /* Special case for NULL device */
 405
 406         return TEST_FAILED;
 407 }
 408
 409 /* calculates optimal mempool size not smaller than the val */
 410 static unsigned int
 411 optimal_mempool_size(unsigned int val)
 412 {
 413         return rte_align32pow2(val + 1) - 1;
 414 }
 415
 416 /* allocates mbuf mempool for inputs and outputs */
 417 static struct rte_mempool *
 418 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 419                 int socket_id, unsigned int mbuf_pool_size,
 420                 const char *op_type_str)
 421 {
 422         unsigned int i;
 423         uint32_t max_seg_sz = 0;
 424         char pool_name[RTE_MEMPOOL_NAMESIZE];
 425
 426         /* find max input segment size */
 427         for (i = 0; i < entries->nb_segments; ++i)
 428                 if (entries->segments[i].length > max_seg_sz)
 429                         max_seg_sz = entries->segments[i].length;
 430
 431         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 432                         dev_id);
 433         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 434                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
 435                                         + FILLER_HEADROOM,
 436                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 437 }
 438
 439 static int
 440 create_mempools(struct active_device *ad, int socket_id,
 441                 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
 442 {
 443         struct rte_mempool *mp;
 444         unsigned int ops_pool_size, mbuf_pool_size = 0;
 445         char pool_name[RTE_MEMPOOL_NAMESIZE];
 446         const char *op_type_str;
 447         enum rte_bbdev_op_type op_type = org_op_type;
 448
 449         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 450         struct op_data_entries *hard_out =
 451                         &test_vector.entries[DATA_HARD_OUTPUT];
 452         struct op_data_entries *soft_out =
 453                         &test_vector.entries[DATA_SOFT_OUTPUT];
 454         struct op_data_entries *harq_in =
 455                         &test_vector.entries[DATA_HARQ_INPUT];
 456         struct op_data_entries *harq_out =
 457                         &test_vector.entries[DATA_HARQ_OUTPUT];
 458
 459         /* allocate ops mempool */
 460         ops_pool_size = optimal_mempool_size(RTE_MAX(
 461                         /* Ops used plus 1 reference op */
 462                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 463                         /* Minimal cache size plus 1 reference op */
 464                         (unsigned int)(1.5 * rte_lcore_count() *
 465                                         OPS_CACHE_SIZE + 1)),
 466                         OPS_POOL_SIZE_MIN));
 467
 468         if (org_op_type == RTE_BBDEV_OP_NONE)
 469                 op_type = RTE_BBDEV_OP_TURBO_ENC;
 470
 471         op_type_str = rte_bbdev_op_type_str(op_type);
 472         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 473
 474         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 475                         ad->dev_id);
 476         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 477                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 478         TEST_ASSERT_NOT_NULL(mp,
 479                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 480                         ops_pool_size,
 481                         ad->dev_id,
 482                         socket_id);
 483         ad->ops_mempool = mp;
 484
 485         /* Do not create inputs and outputs mbufs for BaseBand Null Device */
 486         if (org_op_type == RTE_BBDEV_OP_NONE)
 487                 return TEST_SUCCESS;
 488
 489         /* Inputs */
 490         if (in->nb_segments > 0) {
 491                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 492                                 in->nb_segments);
 493                 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
 494                                 mbuf_pool_size, "in");
 495                 TEST_ASSERT_NOT_NULL(mp,
 496                                 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 497                                 mbuf_pool_size,
 498                                 ad->dev_id,
 499                                 socket_id);
 500                 ad->in_mbuf_pool = mp;
 501         }
 502
 503         /* Hard outputs */
 504         if (hard_out->nb_segments > 0) {
 505                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 506                                 hard_out->nb_segments);
 507                 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
 508                                 mbuf_pool_size,
 509                                 "hard_out");
 510                 TEST_ASSERT_NOT_NULL(mp,
 511                                 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 512                                 mbuf_pool_size,
 513                                 ad->dev_id,
 514                                 socket_id);
 515                 ad->hard_out_mbuf_pool = mp;
 516         }
 517
 518         /* Soft outputs */
 519         if (soft_out->nb_segments > 0) {
 520                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 521                                 soft_out->nb_segments);
 522                 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
 523                                 mbuf_pool_size,
 524                                 "soft_out");
 525                 TEST_ASSERT_NOT_NULL(mp,
 526                                 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 527                                 mbuf_pool_size,
 528                                 ad->dev_id,
 529                                 socket_id);
 530                 ad->soft_out_mbuf_pool = mp;
 531         }
 532
 533         /* HARQ inputs */
 534         if (harq_in->nb_segments > 0) {
 535                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 536                                 harq_in->nb_segments);
 537                 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
 538                                 mbuf_pool_size,
 539                                 "harq_in");
 540                 TEST_ASSERT_NOT_NULL(mp,
 541                                 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
 542                                 mbuf_pool_size,
 543                                 ad->dev_id,
 544                                 socket_id);
 545                 ad->harq_in_mbuf_pool = mp;
 546         }
 547
 548         /* HARQ outputs */
 549         if (harq_out->nb_segments > 0) {
 550                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 551                                 harq_out->nb_segments);
 552                 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
 553                                 mbuf_pool_size,
 554                                 "harq_out");
 555                 TEST_ASSERT_NOT_NULL(mp,
 556                                 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
 557                                 mbuf_pool_size,
 558                                 ad->dev_id,
 559                                 socket_id);
 560                 ad->harq_out_mbuf_pool = mp;
 561         }
 562
 563         return TEST_SUCCESS;
 564 }
 565
 566 static int
 567 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 568                 struct test_bbdev_vector *vector)
 569 {
 570         int ret;
 571         unsigned int queue_id;
 572         struct rte_bbdev_queue_conf qconf;
 573         struct active_device *ad = &active_devs[nb_active_devs];
 574         unsigned int nb_queues;
 575         enum rte_bbdev_op_type op_type = vector->op_type;
 576
 577 /* Configure fpga lte fec with PF & VF values
 578  * if '-i' flag is set and using fpga device
 579  */
 580 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
 581         if ((get_init_device() == true) &&
 582                 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
 583                 struct rte_fpga_lte_fec_conf conf;
 584                 unsigned int i;
 585
 586                 printf("Configure FPGA LTE FEC Driver %s with default values\n",
 587                                 info->drv.driver_name);
 588
 589                 /* clear default configuration before initialization */
 590                 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf));
 591
 592                 /* Set PF mode :
 593                  * true if PF is used for data plane
 594                  * false for VFs
 595                  */
 596                 conf.pf_mode_en = true;
 597
 598                 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
 599                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 600                         conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
 601                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 602                         conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
 603                 }
 604
 605                 /* UL bandwidth. Needed for schedule algorithm */
 606                 conf.ul_bandwidth = UL_4G_BANDWIDTH;
 607                 /* DL bandwidth */
 608                 conf.dl_bandwidth = DL_4G_BANDWIDTH;
 609
 610                 /* UL & DL load Balance Factor to 64 */
 611                 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
 612                 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
 613
 614                 /**< FLR timeout value */
 615                 conf.flr_time_out = FLR_4G_TIMEOUT;
 616
 617                 /* setup FPGA PF with configuration information */
 618                 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf);
 619                 TEST_ASSERT_SUCCESS(ret,
 620                                 "Failed to configure 4G FPGA PF for bbdev %s",
 621                                 info->dev_name);
 622         }
 623 #endif
 624 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
 625         if ((get_init_device() == true) &&
 626                 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
 627                 struct rte_fpga_5gnr_fec_conf conf;
 628                 unsigned int i;
 629
 630                 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
 631                                 info->drv.driver_name);
 632
 633                 /* clear default configuration before initialization */
 634                 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
 635
 636                 /* Set PF mode :
 637                  * true if PF is used for data plane
 638                  * false for VFs
 639                  */
 640                 conf.pf_mode_en = true;
 641
 642                 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
 643                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 644                         conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
 645                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 646                         conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
 647                 }
 648
 649                 /* UL bandwidth. Needed for schedule algorithm */
 650                 conf.ul_bandwidth = UL_5G_BANDWIDTH;
 651                 /* DL bandwidth */
 652                 conf.dl_bandwidth = DL_5G_BANDWIDTH;
 653
 654                 /* UL & DL load Balance Factor to 64 */
 655                 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
 656                 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
 657
 658                 /**< FLR timeout value */
 659                 conf.flr_time_out = FLR_5G_TIMEOUT;
 660
 661                 /* setup FPGA PF with configuration information */
 662                 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf);
 663                 TEST_ASSERT_SUCCESS(ret,
 664                                 "Failed to configure 5G FPGA PF for bbdev %s",
 665                                 info->dev_name);
 666         }
 667 #endif
 668 #ifdef RTE_BASEBAND_ACC100
 669         if ((get_init_device() == true) &&
 670                 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) {
 671                 struct rte_acc100_conf conf;
 672                 unsigned int i;
 673
 674                 printf("Configure ACC100 FEC Driver %s with default values\n",
 675                                 info->drv.driver_name);
 676
 677                 /* clear default configuration before initialization */
 678                 memset(&conf, 0, sizeof(struct rte_acc100_conf));
 679
 680                 /* Always set in PF mode for built-in configuration */
 681                 conf.pf_mode_en = true;
 682                 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) {
 683                         conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 684                         conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 685                         conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR;
 686                         conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 687                         conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 688                         conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR;
 689                         conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 690                         conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 691                         conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR;
 692                         conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 693                         conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 694                         conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR;
 695                 }
 696
 697                 conf.input_pos_llr_1_bit = true;
 698                 conf.output_pos_llr_1_bit = true;
 699                 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */
 700
 701                 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
 702                 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 703                 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 704                 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 705                 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
 706                 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 707                 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 708                 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 709                 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
 710                 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 711                 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 712                 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 713                 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
 714                 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 715                 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 716                 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 717
 718                 /* setup PF with configuration information */
 719                 ret = rte_acc100_configure(info->dev_name, &conf);
 720                 TEST_ASSERT_SUCCESS(ret,
 721                                 "Failed to configure ACC100 PF for bbdev %s",
 722                                 info->dev_name);
 723         }
 724 #endif
 725         /* Let's refresh this now this is configured */
 726         rte_bbdev_info_get(dev_id, info);
 727         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 728         nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
 729
 730         /* setup device */
 731         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 732         if (ret < 0) {
 733                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 734                                 dev_id, nb_queues, info->socket_id, ret);
 735                 return TEST_FAILED;
 736         }
 737
 738         /* configure interrupts if needed */
 739         if (intr_enabled) {
 740                 ret = rte_bbdev_intr_enable(dev_id);
 741                 if (ret < 0) {
 742                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 743                                         ret);
 744                         return TEST_FAILED;
 745                 }
 746         }
 747
 748         /* setup device queues */
 749         qconf.socket = info->socket_id;
 750         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 751         qconf.priority = 0;
 752         qconf.deferred_start = 0;
 753         qconf.op_type = op_type;
 754
 755         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 756                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 757                 if (ret != 0) {
 758                         printf(
 759                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 760                                         queue_id, qconf.priority, dev_id);
 761                         qconf.priority++;
 762                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 763                                         &qconf);
 764                 }
 765                 if (ret != 0) {
 766                         printf("All queues on dev %u allocated: %u\n",
 767                                         dev_id, queue_id);
 768                         break;
 769                 }
 770                 ad->queue_ids[queue_id] = queue_id;
 771         }
 772         TEST_ASSERT(queue_id != 0,
 773                         "ERROR Failed to configure any queues on dev %u",
 774                         dev_id);
 775         ad->nb_queues = queue_id;
 776
 777         set_avail_op(ad, op_type);
 778
 779         return TEST_SUCCESS;
 780 }
 781
 782 static int
 783 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 784                 struct test_bbdev_vector *vector)
 785 {
 786         int ret;
 787
 788         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 789         active_devs[nb_active_devs].dev_id = dev_id;
 790
 791         ret = add_bbdev_dev(dev_id, info, vector);
 792         if (ret == TEST_SUCCESS)
 793                 ++nb_active_devs;
 794         return ret;
 795 }
 796
 797 static uint8_t
 798 populate_active_devices(void)
 799 {
 800         int ret;
 801         uint8_t dev_id;
 802         uint8_t nb_devs_added = 0;
 803         struct rte_bbdev_info info;
 804
 805         RTE_BBDEV_FOREACH(dev_id) {
 806                 rte_bbdev_info_get(dev_id, &info);
 807
 808                 if (check_dev_cap(&info)) {
 809                         printf(
 810                                 "Device %d (%s) does not support specified capabilities\n",
 811                                         dev_id, info.dev_name);
 812                         continue;
 813                 }
 814
 815                 ret = add_active_device(dev_id, &info, &test_vector);
 816                 if (ret != 0) {
 817                         printf("Adding active bbdev %s skipped\n",
 818                                         info.dev_name);
 819                         continue;
 820                 }
 821                 nb_devs_added++;
 822         }
 823
 824         return nb_devs_added;
 825 }
 826
 827 static int
 828 read_test_vector(void)
 829 {
 830         int ret;
 831
 832         memset(&test_vector, 0, sizeof(test_vector));
 833         printf("Test vector file = %s\n", get_vector_filename());
 834         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 835         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 836                         get_vector_filename());
 837
 838         return TEST_SUCCESS;
 839 }
 840
 841 static int
 842 testsuite_setup(void)
 843 {
 844         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 845
 846         if (populate_active_devices() == 0) {
 847                 printf("No suitable devices found!\n");
 848                 return TEST_SKIPPED;
 849         }
 850
 851         return TEST_SUCCESS;
 852 }
 853
 854 static int
 855 interrupt_testsuite_setup(void)
 856 {
 857         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 858
 859         /* Enable interrupts */
 860         intr_enabled = true;
 861
 862         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 863         if (populate_active_devices() == 0 ||
 864                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 865                 intr_enabled = false;
 866                 printf("No suitable devices found!\n");
 867                 return TEST_SKIPPED;
 868         }
 869
 870         return TEST_SUCCESS;
 871 }
 872
 873 static void
 874 testsuite_teardown(void)
 875 {
 876         uint8_t dev_id;
 877
 878         /* Unconfigure devices */
 879         RTE_BBDEV_FOREACH(dev_id)
 880                 rte_bbdev_close(dev_id);
 881
 882         /* Clear active devices structs. */
 883         memset(active_devs, 0, sizeof(active_devs));
 884         nb_active_devs = 0;
 885
 886         /* Disable interrupts */
 887         intr_enabled = false;
 888 }
 889
 890 static int
 891 ut_setup(void)
 892 {
 893         uint8_t i, dev_id;
 894
 895         for (i = 0; i < nb_active_devs; i++) {
 896                 dev_id = active_devs[i].dev_id;
 897                 /* reset bbdev stats */
 898                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 899                                 "Failed to reset stats of bbdev %u", dev_id);
 900                 /* start the device */
 901                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 902                                 "Failed to start bbdev %u", dev_id);
 903         }
 904
 905         return TEST_SUCCESS;
 906 }
 907
 908 static void
 909 ut_teardown(void)
 910 {
 911         uint8_t i, dev_id;
 912         struct rte_bbdev_stats stats;
 913
 914         for (i = 0; i < nb_active_devs; i++) {
 915                 dev_id = active_devs[i].dev_id;
 916                 /* read stats and print */
 917                 rte_bbdev_stats_get(dev_id, &stats);
 918                 /* Stop the device */
 919                 rte_bbdev_stop(dev_id);
 920         }
 921 }
 922
 923 static int
 924 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 925                 struct op_data_entries *ref_entries,
 926                 struct rte_mempool *mbuf_pool, const uint16_t n,
 927                 enum op_data_type op_type, uint16_t min_alignment)
 928 {
 929         int ret;
 930         unsigned int i, j;
 931         bool large_input = false;
 932
 933         for (i = 0; i < n; ++i) {
 934                 char *data;
 935                 struct op_data_buf *seg = &ref_entries->segments[0];
 936                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 937                 TEST_ASSERT_NOT_NULL(m_head,
 938                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 939                                 op_type, n * ref_entries->nb_segments,
 940                                 mbuf_pool->size);
 941
 942                 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
 943                         /*
 944                          * Special case when DPDK mbuf cannot handle
 945                          * the required input size
 946                          */
 947                         printf("Warning: Larger input size than DPDK mbuf %d\n",
 948                                         seg->length);
 949                         large_input = true;
 950                 }
 951                 bufs[i].data = m_head;
 952                 bufs[i].offset = 0;
 953                 bufs[i].length = 0;
 954
 955                 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
 956                         if ((op_type == DATA_INPUT) && large_input) {
 957                                 /* Allocate a fake overused mbuf */
 958                                 data = rte_malloc(NULL, seg->length, 0);
 959                                 memcpy(data, seg->addr, seg->length);
 960                                 m_head->buf_addr = data;
 961                                 m_head->buf_iova = rte_malloc_virt2iova(data);
 962                                 m_head->data_off = 0;
 963                                 m_head->data_len = seg->length;
 964                         } else {
 965                                 data = rte_pktmbuf_append(m_head, seg->length);
 966                                 TEST_ASSERT_NOT_NULL(data,
 967                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 968                                         seg->length, op_type);
 969
 970                                 TEST_ASSERT(data == RTE_PTR_ALIGN(
 971                                                 data, min_alignment),
 972                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 973                                         data, min_alignment);
 974                                 rte_memcpy(data, seg->addr, seg->length);
 975                         }
 976
 977                         bufs[i].length += seg->length;
 978
 979                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 980                                 struct rte_mbuf *m_tail =
 981                                                 rte_pktmbuf_alloc(mbuf_pool);
 982                                 TEST_ASSERT_NOT_NULL(m_tail,
 983                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 984                                                 op_type,
 985                                                 n * ref_entries->nb_segments,
 986                                                 mbuf_pool->size);
 987                                 seg += 1;
 988
 989                                 data = rte_pktmbuf_append(m_tail, seg->length);
 990                                 TEST_ASSERT_NOT_NULL(data,
 991                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 992                                                 seg->length, op_type);
 993
 994                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 995                                                 min_alignment),
 996                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 997                                                 data, min_alignment);
 998                                 rte_memcpy(data, seg->addr, seg->length);
 999                                 bufs[i].length += seg->length;
1000
1001                                 ret = rte_pktmbuf_chain(m_head, m_tail);
1002                                 TEST_ASSERT_SUCCESS(ret,
1003                                                 "Couldn't chain mbufs from %d data type mbuf pool",
1004                                                 op_type);
1005                         }
1006                 } else {
1007
1008                         /* allocate chained-mbuf for output buffer */
1009                         for (j = 1; j < ref_entries->nb_segments; ++j) {
1010                                 struct rte_mbuf *m_tail =
1011                                                 rte_pktmbuf_alloc(mbuf_pool);
1012                                 TEST_ASSERT_NOT_NULL(m_tail,
1013                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1014                                                 op_type,
1015                                                 n * ref_entries->nb_segments,
1016                                                 mbuf_pool->size);
1017
1018                                 ret = rte_pktmbuf_chain(m_head, m_tail);
1019                                 TEST_ASSERT_SUCCESS(ret,
1020                                                 "Couldn't chain mbufs from %d data type mbuf pool",
1021                                                 op_type);
1022                         }
1023                 }
1024         }
1025
1026         return 0;
1027 }
1028
1029 static int
1030 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
1031                 const int socket)
1032 {
1033         int i;
1034
1035         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
1036         if (*buffers == NULL) {
1037                 printf("WARNING: Failed to allocate op_data on socket %d\n",
1038                                 socket);
1039                 /* try to allocate memory on other detected sockets */
1040                 for (i = 0; i < socket; i++) {
1041                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
1042                         if (*buffers != NULL)
1043                                 break;
1044                 }
1045         }
1046
1047         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
1048 }
1049
1050 static void
1051 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
1052                 const uint16_t n, const int8_t max_llr_modulus)
1053 {
1054         uint16_t i, byte_idx;
1055
1056         for (i = 0; i < n; ++i) {
1057                 struct rte_mbuf *m = input_ops[i].data;
1058                 while (m != NULL) {
1059                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1060                                         input_ops[i].offset);
1061                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1062                                         ++byte_idx)
1063                                 llr[byte_idx] = round((double)max_llr_modulus *
1064                                                 llr[byte_idx] / INT8_MAX);
1065
1066                         m = m->next;
1067                 }
1068         }
1069 }
1070
1071 /*
1072  * We may have to insert filler bits
1073  * when they are required by the HARQ assumption
1074  */
1075 static void
1076 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1077                 const uint16_t n, struct test_op_params *op_params)
1078 {
1079         struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1080
1081         if (input_ops == NULL)
1082                 return;
1083         /* No need to add filler if not required by device */
1084         if (!(ldpc_cap_flags &
1085                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1086                 return;
1087         /* No need to add filler for loopback operation */
1088         if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1089                 return;
1090
1091         uint16_t i, j, parity_offset;
1092         for (i = 0; i < n; ++i) {
1093                 struct rte_mbuf *m = input_ops[i].data;
1094                 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1095                                 input_ops[i].offset);
1096                 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1097                                 * dec.z_c - dec.n_filler;
1098                 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1099                 m->data_len = new_hin_size;
1100                 input_ops[i].length = new_hin_size;
1101                 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1102                                 j--)
1103                         llr[j] = llr[j - dec.n_filler];
1104                 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1105                 for (j = 0; j < dec.n_filler; j++)
1106                         llr[parity_offset + j] = llr_max_pre_scaling;
1107         }
1108 }
1109
1110 static void
1111 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1112                 const uint16_t n, const int8_t llr_size,
1113                 const int8_t llr_decimals)
1114 {
1115         if (input_ops == NULL)
1116                 return;
1117
1118         uint16_t i, byte_idx;
1119
1120         int16_t llr_max, llr_min, llr_tmp;
1121         llr_max = (1 << (llr_size - 1)) - 1;
1122         llr_min = -llr_max;
1123         for (i = 0; i < n; ++i) {
1124                 struct rte_mbuf *m = input_ops[i].data;
1125                 while (m != NULL) {
1126                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1127                                         input_ops[i].offset);
1128                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1129                                         ++byte_idx) {
1130
1131                                 llr_tmp = llr[byte_idx];
1132                                 if (llr_decimals == 4)
1133                                         llr_tmp *= 8;
1134                                 else if (llr_decimals == 2)
1135                                         llr_tmp *= 2;
1136                                 else if (llr_decimals == 0)
1137                                         llr_tmp /= 2;
1138                                 llr_tmp = RTE_MIN(llr_max,
1139                                                 RTE_MAX(llr_min, llr_tmp));
1140                                 llr[byte_idx] = (int8_t) llr_tmp;
1141                         }
1142
1143                         m = m->next;
1144                 }
1145         }
1146 }
1147
1148
1149
1150 static int
1151 fill_queue_buffers(struct test_op_params *op_params,
1152                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1153                 struct rte_mempool *soft_out_mp,
1154                 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1155                 uint16_t queue_id,
1156                 const struct rte_bbdev_op_cap *capabilities,
1157                 uint16_t min_alignment, const int socket_id)
1158 {
1159         int ret;
1160         enum op_data_type type;
1161         const uint16_t n = op_params->num_to_process;
1162
1163         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1164                 in_mp,
1165                 soft_out_mp,
1166                 hard_out_mp,
1167                 harq_in_mp,
1168                 harq_out_mp,
1169         };
1170
1171         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1172                 &op_params->q_bufs[socket_id][queue_id].inputs,
1173                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1174                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1175                 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1176                 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1177         };
1178
1179         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1180                 struct op_data_entries *ref_entries =
1181                                 &test_vector.entries[type];
1182                 if (ref_entries->nb_segments == 0)
1183                         continue;
1184
1185                 ret = allocate_buffers_on_socket(queue_ops[type],
1186                                 n * sizeof(struct rte_bbdev_op_data),
1187                                 socket_id);
1188                 TEST_ASSERT_SUCCESS(ret,
1189                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
1190
1191                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1192                                 mbuf_pools[type], n, type, min_alignment);
1193                 TEST_ASSERT_SUCCESS(ret,
1194                                 "Couldn't init rte_bbdev_op_data structs");
1195         }
1196
1197         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1198                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1199                         capabilities->cap.turbo_dec.max_llr_modulus);
1200
1201         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1202                 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1203                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1204                 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1205                                 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1206                 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1207                                 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1208                 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1209                 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1210                 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1211                 if (!loopback && !llr_comp)
1212                         ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1213                                         ldpc_llr_size, ldpc_llr_decimals);
1214                 if (!loopback && !harq_comp)
1215                         ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1216                                         ldpc_llr_size, ldpc_llr_decimals);
1217                 if (!loopback)
1218                         ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1219                                         op_params);
1220         }
1221
1222         return 0;
1223 }
1224
1225 static void
1226 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1227 {
1228         unsigned int i, j;
1229
1230         rte_mempool_free(ad->ops_mempool);
1231         rte_mempool_free(ad->in_mbuf_pool);
1232         rte_mempool_free(ad->hard_out_mbuf_pool);
1233         rte_mempool_free(ad->soft_out_mbuf_pool);
1234         rte_mempool_free(ad->harq_in_mbuf_pool);
1235         rte_mempool_free(ad->harq_out_mbuf_pool);
1236
1237         for (i = 0; i < rte_lcore_count(); ++i) {
1238                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1239                         rte_free(op_params->q_bufs[j][i].inputs);
1240                         rte_free(op_params->q_bufs[j][i].hard_outputs);
1241                         rte_free(op_params->q_bufs[j][i].soft_outputs);
1242                         rte_free(op_params->q_bufs[j][i].harq_inputs);
1243                         rte_free(op_params->q_bufs[j][i].harq_outputs);
1244                 }
1245         }
1246 }
1247
1248 static void
1249 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1250                 unsigned int start_idx,
1251                 struct rte_bbdev_op_data *inputs,
1252                 struct rte_bbdev_op_data *hard_outputs,
1253                 struct rte_bbdev_op_data *soft_outputs,
1254                 struct rte_bbdev_dec_op *ref_op)
1255 {
1256         unsigned int i;
1257         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1258
1259         for (i = 0; i < n; ++i) {
1260                 if (turbo_dec->code_block_mode == 0) {
1261                         ops[i]->turbo_dec.tb_params.ea =
1262                                         turbo_dec->tb_params.ea;
1263                         ops[i]->turbo_dec.tb_params.eb =
1264                                         turbo_dec->tb_params.eb;
1265                         ops[i]->turbo_dec.tb_params.k_pos =
1266                                         turbo_dec->tb_params.k_pos;
1267                         ops[i]->turbo_dec.tb_params.k_neg =
1268                                         turbo_dec->tb_params.k_neg;
1269                         ops[i]->turbo_dec.tb_params.c =
1270                                         turbo_dec->tb_params.c;
1271                         ops[i]->turbo_dec.tb_params.c_neg =
1272                                         turbo_dec->tb_params.c_neg;
1273                         ops[i]->turbo_dec.tb_params.cab =
1274                                         turbo_dec->tb_params.cab;
1275                         ops[i]->turbo_dec.tb_params.r =
1276                                         turbo_dec->tb_params.r;
1277                 } else {
1278                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1279                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1280                 }
1281
1282                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1283                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1284                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1285                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1286                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1287                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1288                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1289
1290                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1291                 ops[i]->turbo_dec.input = inputs[start_idx + i];
1292                 if (soft_outputs != NULL)
1293                         ops[i]->turbo_dec.soft_output =
1294                                 soft_outputs[start_idx + i];
1295         }
1296 }
1297
1298 static void
1299 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1300                 unsigned int start_idx,
1301                 struct rte_bbdev_op_data *inputs,
1302                 struct rte_bbdev_op_data *outputs,
1303                 struct rte_bbdev_enc_op *ref_op)
1304 {
1305         unsigned int i;
1306         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1307         for (i = 0; i < n; ++i) {
1308                 if (turbo_enc->code_block_mode == 0) {
1309                         ops[i]->turbo_enc.tb_params.ea =
1310                                         turbo_enc->tb_params.ea;
1311                         ops[i]->turbo_enc.tb_params.eb =
1312                                         turbo_enc->tb_params.eb;
1313                         ops[i]->turbo_enc.tb_params.k_pos =
1314                                         turbo_enc->tb_params.k_pos;
1315                         ops[i]->turbo_enc.tb_params.k_neg =
1316                                         turbo_enc->tb_params.k_neg;
1317                         ops[i]->turbo_enc.tb_params.c =
1318                                         turbo_enc->tb_params.c;
1319                         ops[i]->turbo_enc.tb_params.c_neg =
1320                                         turbo_enc->tb_params.c_neg;
1321                         ops[i]->turbo_enc.tb_params.cab =
1322                                         turbo_enc->tb_params.cab;
1323                         ops[i]->turbo_enc.tb_params.ncb_pos =
1324                                         turbo_enc->tb_params.ncb_pos;
1325                         ops[i]->turbo_enc.tb_params.ncb_neg =
1326                                         turbo_enc->tb_params.ncb_neg;
1327                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1328                 } else {
1329                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1330                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1331                         ops[i]->turbo_enc.cb_params.ncb =
1332                                         turbo_enc->cb_params.ncb;
1333                 }
1334                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1335                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1336                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1337
1338                 ops[i]->turbo_enc.output = outputs[start_idx + i];
1339                 ops[i]->turbo_enc.input = inputs[start_idx + i];
1340         }
1341 }
1342
1343
1344 /* Returns a random number drawn from a normal distribution
1345  * with mean of 0 and variance of 1
1346  * Marsaglia algorithm
1347  */
1348 static double
1349 randn(int n)
1350 {
1351         double S, Z, U1, U2, u, v, fac;
1352
1353         do {
1354                 U1 = (double)rand() / RAND_MAX;
1355                 U2 = (double)rand() / RAND_MAX;
1356                 u = 2. * U1 - 1.;
1357                 v = 2. * U2 - 1.;
1358                 S = u * u + v * v;
1359         } while (S >= 1 || S == 0);
1360         fac = sqrt(-2. * log(S) / S);
1361         Z = (n % 2) ? u * fac : v * fac;
1362         return Z;
1363 }
1364
1365 static inline double
1366 maxstar(double A, double B)
1367 {
1368         if (fabs(A - B) > 5)
1369                 return RTE_MAX(A, B);
1370         else
1371                 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1372 }
1373
1374 /*
1375  * Generate Qm LLRS for Qm==8
1376  * Modulation, AWGN and LLR estimation from max log development
1377  */
1378 static void
1379 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1380 {
1381         int qm = 8;
1382         int qam = 256;
1383         int m, k;
1384         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1385         /* 5.1.4 of TS38.211 */
1386         const double symbols_I[256] = {
1387                         5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1388                         5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1389                         11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1390                         15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1391                         15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1392                         1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1393                         1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1394                         15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1395                         13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1396                         -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1397                         -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1398                         -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1399                         -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1400                         -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1401                         -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1402                         -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1403                         -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1404                         -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1405                         -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1406                         -13, -13, -15, -15, -13, -13, -15, -15};
1407         const double symbols_Q[256] = {
1408                         5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1409                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1410                         15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1411                         11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1412                         15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1413                         -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1414                         -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1415                         -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1416                         -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1417                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1418                         7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1419                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1420                         13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1421                         3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1422                         13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1423                         -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1424                         -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1425                         -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1426                         -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1427                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1428         /* Average constellation point energy */
1429         N0 *= 170.0;
1430         for (k = 0; k < qm; k++)
1431                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1432         /* 5.1.4 of TS38.211 */
1433         I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1434                         (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1435         Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1436                         (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1437         /* AWGN channel */
1438         I += sqrt(N0 / 2) * randn(0);
1439         Q += sqrt(N0 / 2) * randn(1);
1440         /*
1441          * Calculate the log of the probability that each of
1442          * the constellation points was transmitted
1443          */
1444         for (m = 0; m < qam; m++)
1445                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1446                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1447         /* Calculate an LLR for each of the k_64QAM bits in the set */
1448         for (k = 0; k < qm; k++) {
1449                 p0 = -999999;
1450                 p1 = -999999;
1451                 /* For each constellation point */
1452                 for (m = 0; m < qam; m++) {
1453                         if ((m >> (qm - k - 1)) & 1)
1454                                 p1 = maxstar(p1, log_syml_prob[m]);
1455                         else
1456                                 p0 = maxstar(p0, log_syml_prob[m]);
1457                 }
1458                 /* Calculate the LLR */
1459                 llr_ = p0 - p1;
1460                 llr_ *= (1 << ldpc_llr_decimals);
1461                 llr_ = round(llr_);
1462                 if (llr_ > llr_max)
1463                         llr_ = llr_max;
1464                 if (llr_ < -llr_max)
1465                         llr_ = -llr_max;
1466                 llrs[qm * i + k] = (int8_t) llr_;
1467         }
1468 }
1469
1470
1471 /*
1472  * Generate Qm LLRS for Qm==6
1473  * Modulation, AWGN and LLR estimation from max log development
1474  */
1475 static void
1476 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1477 {
1478         int qm = 6;
1479         int qam = 64;
1480         int m, k;
1481         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1482         /* 5.1.4 of TS38.211 */
1483         const double symbols_I[64] = {
1484                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1485                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1486                         -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1487                         -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1488                         -5, -5, -7, -7, -5, -5, -7, -7};
1489         const double symbols_Q[64] = {
1490                         3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1491                         -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1492                         -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1493                         5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1494                         -3, -1, -3, -1, -5, -7, -5, -7};
1495         /* Average constellation point energy */
1496         N0 *= 42.0;
1497         for (k = 0; k < qm; k++)
1498                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1499         /* 5.1.4 of TS38.211 */
1500         I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1501         Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1502         /* AWGN channel */
1503         I += sqrt(N0 / 2) * randn(0);
1504         Q += sqrt(N0 / 2) * randn(1);
1505         /*
1506          * Calculate the log of the probability that each of
1507          * the constellation points was transmitted
1508          */
1509         for (m = 0; m < qam; m++)
1510                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1511                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1512         /* Calculate an LLR for each of the k_64QAM bits in the set */
1513         for (k = 0; k < qm; k++) {
1514                 p0 = -999999;
1515                 p1 = -999999;
1516                 /* For each constellation point */
1517                 for (m = 0; m < qam; m++) {
1518                         if ((m >> (qm - k - 1)) & 1)
1519                                 p1 = maxstar(p1, log_syml_prob[m]);
1520                         else
1521                                 p0 = maxstar(p0, log_syml_prob[m]);
1522                 }
1523                 /* Calculate the LLR */
1524                 llr_ = p0 - p1;
1525                 llr_ *= (1 << ldpc_llr_decimals);
1526                 llr_ = round(llr_);
1527                 if (llr_ > llr_max)
1528                         llr_ = llr_max;
1529                 if (llr_ < -llr_max)
1530                         llr_ = -llr_max;
1531                 llrs[qm * i + k] = (int8_t) llr_;
1532         }
1533 }
1534
1535 /*
1536  * Generate Qm LLRS for Qm==4
1537  * Modulation, AWGN and LLR estimation from max log development
1538  */
1539 static void
1540 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1541 {
1542         int qm = 4;
1543         int qam = 16;
1544         int m, k;
1545         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1546         /* 5.1.4 of TS38.211 */
1547         const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1548                         -1, -1, -3, -3, -1, -1, -3, -3};
1549         const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1550                         1, 3, 1, 3, -1, -3, -1, -3};
1551         /* Average constellation point energy */
1552         N0 *= 10.0;
1553         for (k = 0; k < qm; k++)
1554                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1555         /* 5.1.4 of TS38.211 */
1556         I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1557         Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1558         /* AWGN channel */
1559         I += sqrt(N0 / 2) * randn(0);
1560         Q += sqrt(N0 / 2) * randn(1);
1561         /*
1562          * Calculate the log of the probability that each of
1563          * the constellation points was transmitted
1564          */
1565         for (m = 0; m < qam; m++)
1566                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1567                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1568         /* Calculate an LLR for each of the k_64QAM bits in the set */
1569         for (k = 0; k < qm; k++) {
1570                 p0 = -999999;
1571                 p1 = -999999;
1572                 /* For each constellation point */
1573                 for (m = 0; m < qam; m++) {
1574                         if ((m >> (qm - k - 1)) & 1)
1575                                 p1 = maxstar(p1, log_syml_prob[m]);
1576                         else
1577                                 p0 = maxstar(p0, log_syml_prob[m]);
1578                 }
1579                 /* Calculate the LLR */
1580                 llr_ = p0 - p1;
1581                 llr_ *= (1 << ldpc_llr_decimals);
1582                 llr_ = round(llr_);
1583                 if (llr_ > llr_max)
1584                         llr_ = llr_max;
1585                 if (llr_ < -llr_max)
1586                         llr_ = -llr_max;
1587                 llrs[qm * i + k] = (int8_t) llr_;
1588         }
1589 }
1590
1591 static void
1592 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1593 {
1594         double b, b1, n;
1595         double coeff = 2.0 * sqrt(N0);
1596
1597         /* Ignore in vectors rare quasi null LLRs not to be saturated */
1598         if (llrs[j] < 8 && llrs[j] > -8)
1599                 return;
1600
1601         /* Note don't change sign here */
1602         n = randn(j % 2);
1603         b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1604                         + coeff * n) / N0;
1605         b = b1 * (1 << ldpc_llr_decimals);
1606         b = round(b);
1607         if (b > llr_max)
1608                 b = llr_max;
1609         if (b < -llr_max)
1610                 b = -llr_max;
1611         llrs[j] = (int8_t) b;
1612 }
1613
1614 /* Generate LLR for a given SNR */
1615 static void
1616 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1617                 struct rte_bbdev_dec_op *ref_op)
1618 {
1619         struct rte_mbuf *m;
1620         uint16_t qm;
1621         uint32_t i, j, e, range;
1622         double N0, llr_max;
1623
1624         e = ref_op->ldpc_dec.cb_params.e;
1625         qm = ref_op->ldpc_dec.q_m;
1626         llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1627         range = e / qm;
1628         N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1629
1630         for (i = 0; i < n; ++i) {
1631                 m = inputs[i].data;
1632                 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1633                 if (qm == 8) {
1634                         for (j = 0; j < range; ++j)
1635                                 gen_qm8_llr(llrs, j, N0, llr_max);
1636                 } else if (qm == 6) {
1637                         for (j = 0; j < range; ++j)
1638                                 gen_qm6_llr(llrs, j, N0, llr_max);
1639                 } else if (qm == 4) {
1640                         for (j = 0; j < range; ++j)
1641                                 gen_qm4_llr(llrs, j, N0, llr_max);
1642                 } else {
1643                         for (j = 0; j < e; ++j)
1644                                 gen_qm2_llr(llrs, j, N0, llr_max);
1645                 }
1646         }
1647 }
1648
1649 static void
1650 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1651                 unsigned int start_idx,
1652                 struct rte_bbdev_op_data *inputs,
1653                 struct rte_bbdev_op_data *hard_outputs,
1654                 struct rte_bbdev_op_data *soft_outputs,
1655                 struct rte_bbdev_op_data *harq_inputs,
1656                 struct rte_bbdev_op_data *harq_outputs,
1657                 struct rte_bbdev_dec_op *ref_op)
1658 {
1659         unsigned int i;
1660         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1661
1662         for (i = 0; i < n; ++i) {
1663                 if (ldpc_dec->code_block_mode == 0) {
1664                         ops[i]->ldpc_dec.tb_params.ea =
1665                                         ldpc_dec->tb_params.ea;
1666                         ops[i]->ldpc_dec.tb_params.eb =
1667                                         ldpc_dec->tb_params.eb;
1668                         ops[i]->ldpc_dec.tb_params.c =
1669                                         ldpc_dec->tb_params.c;
1670                         ops[i]->ldpc_dec.tb_params.cab =
1671                                         ldpc_dec->tb_params.cab;
1672                         ops[i]->ldpc_dec.tb_params.r =
1673                                         ldpc_dec->tb_params.r;
1674                 } else {
1675                         ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1676                 }
1677
1678                 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1679                 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1680                 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1681                 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1682                 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1683                 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1684                 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1685                 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1686                 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1687
1688                 if (hard_outputs != NULL)
1689                         ops[i]->ldpc_dec.hard_output =
1690                                         hard_outputs[start_idx + i];
1691                 if (inputs != NULL)
1692                         ops[i]->ldpc_dec.input =
1693                                         inputs[start_idx + i];
1694                 if (soft_outputs != NULL)
1695                         ops[i]->ldpc_dec.soft_output =
1696                                         soft_outputs[start_idx + i];
1697                 if (harq_inputs != NULL)
1698                         ops[i]->ldpc_dec.harq_combined_input =
1699                                         harq_inputs[start_idx + i];
1700                 if (harq_outputs != NULL)
1701                         ops[i]->ldpc_dec.harq_combined_output =
1702                                         harq_outputs[start_idx + i];
1703         }
1704 }
1705
1706
1707 static void
1708 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1709                 unsigned int start_idx,
1710                 struct rte_bbdev_op_data *inputs,
1711                 struct rte_bbdev_op_data *outputs,
1712                 struct rte_bbdev_enc_op *ref_op)
1713 {
1714         unsigned int i;
1715         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1716         for (i = 0; i < n; ++i) {
1717                 if (ldpc_enc->code_block_mode == 0) {
1718                         ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1719                         ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1720                         ops[i]->ldpc_enc.tb_params.cab =
1721                                         ldpc_enc->tb_params.cab;
1722                         ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1723                         ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1724                 } else {
1725                         ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1726                 }
1727                 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1728                 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1729                 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1730                 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1731                 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1732                 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1733                 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1734                 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1735                 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1736                 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1737         }
1738 }
1739
1740 static int
1741 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1742                 unsigned int order_idx, const int expected_status)
1743 {
1744         int status = op->status;
1745         /* ignore parity mismatch false alarms for long iterations */
1746         if (get_iter_max() >= 10) {
1747                 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1748                                 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1749                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1750                         status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1751                 }
1752                 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1753                                 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1754                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1755                         status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1756                 }
1757         }
1758
1759         TEST_ASSERT(status == expected_status,
1760                         "op_status (%d) != expected_status (%d)",
1761                         op->status, expected_status);
1762
1763         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1764                         "Ordering error, expected %p, got %p",
1765                         (void *)(uintptr_t)order_idx, op->opaque_data);
1766
1767         return TEST_SUCCESS;
1768 }
1769
1770 static int
1771 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1772                 unsigned int order_idx, const int expected_status)
1773 {
1774         TEST_ASSERT(op->status == expected_status,
1775                         "op_status (%d) != expected_status (%d)",
1776                         op->status, expected_status);
1777
1778         if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1779                 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1780                                 "Ordering error, expected %p, got %p",
1781                                 (void *)(uintptr_t)order_idx, op->opaque_data);
1782
1783         return TEST_SUCCESS;
1784 }
1785
1786 static inline int
1787 validate_op_chain(struct rte_bbdev_op_data *op,
1788                 struct op_data_entries *orig_op)
1789 {
1790         uint8_t i;
1791         struct rte_mbuf *m = op->data;
1792         uint8_t nb_dst_segments = orig_op->nb_segments;
1793         uint32_t total_data_size = 0;
1794
1795         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1796                         "Number of segments differ in original (%u) and filled (%u) op",
1797                         nb_dst_segments, m->nb_segs);
1798
1799         /* Validate each mbuf segment length */
1800         for (i = 0; i < nb_dst_segments; ++i) {
1801                 /* Apply offset to the first mbuf segment */
1802                 uint16_t offset = (i == 0) ? op->offset : 0;
1803                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1804                 total_data_size += orig_op->segments[i].length;
1805
1806                 TEST_ASSERT(orig_op->segments[i].length == data_len,
1807                                 "Length of segment differ in original (%u) and filled (%u) op",
1808                                 orig_op->segments[i].length, data_len);
1809                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1810                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1811                                 data_len,
1812                                 "Output buffers (CB=%u) are not equal", i);
1813                 m = m->next;
1814         }
1815
1816         /* Validate total mbuf pkt length */
1817         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1818         TEST_ASSERT(total_data_size == pkt_len,
1819                         "Length of data differ in original (%u) and filled (%u) op",
1820                         total_data_size, pkt_len);
1821
1822         return TEST_SUCCESS;
1823 }
1824
1825 /*
1826  * Compute K0 for a given configuration for HARQ output length computation
1827  * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1828  */
1829 static inline uint16_t
1830 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1831 {
1832         if (rv_index == 0)
1833                 return 0;
1834         uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1835         if (n_cb == n) {
1836                 if (rv_index == 1)
1837                         return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1838                 else if (rv_index == 2)
1839                         return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1840                 else
1841                         return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1842         }
1843         /* LBRM case - includes a division by N */
1844         if (rv_index == 1)
1845                 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1846                                 / n) * z_c;
1847         else if (rv_index == 2)
1848                 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1849                                 / n) * z_c;
1850         else
1851                 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1852                                 / n) * z_c;
1853 }
1854
1855 /* HARQ output length including the Filler bits */
1856 static inline uint16_t
1857 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1858 {
1859         uint16_t k0 = 0;
1860         uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1861         k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1862         /* Compute RM out size and number of rows */
1863         uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1864                         * ops_ld->z_c - ops_ld->n_filler;
1865         uint16_t deRmOutSize = RTE_MIN(
1866                         k0 + ops_ld->cb_params.e +
1867                         ((k0 > parity_offset) ?
1868                                         0 : ops_ld->n_filler),
1869                                         ops_ld->n_cb);
1870         uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1871                         / ops_ld->z_c);
1872         uint16_t harq_output_len = numRows * ops_ld->z_c;
1873         return harq_output_len;
1874 }
1875
1876 static inline int
1877 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1878                 struct op_data_entries *orig_op,
1879                 struct rte_bbdev_op_ldpc_dec *ops_ld)
1880 {
1881         uint8_t i;
1882         uint32_t j, jj, k;
1883         struct rte_mbuf *m = op->data;
1884         uint8_t nb_dst_segments = orig_op->nb_segments;
1885         uint32_t total_data_size = 0;
1886         int8_t *harq_orig, *harq_out, abs_harq_origin;
1887         uint32_t byte_error = 0, cum_error = 0, error;
1888         int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1889         int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1890         uint16_t parity_offset;
1891
1892         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1893                         "Number of segments differ in original (%u) and filled (%u) op",
1894                         nb_dst_segments, m->nb_segs);
1895
1896         /* Validate each mbuf segment length */
1897         for (i = 0; i < nb_dst_segments; ++i) {
1898                 /* Apply offset to the first mbuf segment */
1899                 uint16_t offset = (i == 0) ? op->offset : 0;
1900                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1901                 total_data_size += orig_op->segments[i].length;
1902
1903                 TEST_ASSERT(orig_op->segments[i].length <
1904                                 (uint32_t)(data_len + 64),
1905                                 "Length of segment differ in original (%u) and filled (%u) op",
1906                                 orig_op->segments[i].length, data_len);
1907                 harq_orig = (int8_t *) orig_op->segments[i].addr;
1908                 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1909
1910                 if (!(ldpc_cap_flags &
1911                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1912                                 ) || (ops_ld->op_flags &
1913                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1914                         data_len -= ops_ld->z_c;
1915                         parity_offset = data_len;
1916                 } else {
1917                         /* Compute RM out size and number of rows */
1918                         parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1919                                         * ops_ld->z_c - ops_ld->n_filler;
1920                         uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1921                                         ops_ld->n_filler;
1922                         if (data_len > deRmOutSize)
1923                                 data_len = deRmOutSize;
1924                         if (data_len > orig_op->segments[i].length)
1925                                 data_len = orig_op->segments[i].length;
1926                 }
1927                 /*
1928                  * HARQ output can have minor differences
1929                  * due to integer representation and related scaling
1930                  */
1931                 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1932                         if (j == parity_offset) {
1933                                 /* Special Handling of the filler bits */
1934                                 for (k = 0; k < ops_ld->n_filler; k++) {
1935                                         if (harq_out[jj] !=
1936                                                         llr_max_pre_scaling) {
1937                                                 printf("HARQ Filler issue %d: %d %d\n",
1938                                                         jj, harq_out[jj],
1939                                                         llr_max);
1940                                                 byte_error++;
1941                                         }
1942                                         jj++;
1943                                 }
1944                         }
1945                         if (!(ops_ld->op_flags &
1946                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1947                                 if (ldpc_llr_decimals > 1)
1948                                         harq_out[jj] = (harq_out[jj] + 1)
1949                                                 >> (ldpc_llr_decimals - 1);
1950                                 /* Saturated to S7 */
1951                                 if (harq_orig[j] > llr_max)
1952                                         harq_orig[j] = llr_max;
1953                                 if (harq_orig[j] < -llr_max)
1954                                         harq_orig[j] = -llr_max;
1955                         }
1956                         if (harq_orig[j] != harq_out[jj]) {
1957                                 error = (harq_orig[j] > harq_out[jj]) ?
1958                                                 harq_orig[j] - harq_out[jj] :
1959                                                 harq_out[jj] - harq_orig[j];
1960                                 abs_harq_origin = harq_orig[j] > 0 ?
1961                                                         harq_orig[j] :
1962                                                         -harq_orig[j];
1963                                 /* Residual quantization error */
1964                                 if ((error > 8 && (abs_harq_origin <
1965                                                 (llr_max - 16))) ||
1966                                                 (error > 16)) {
1967                                         printf("HARQ mismatch %d: exp %d act %d => %d\n",
1968                                                         j, harq_orig[j],
1969                                                         harq_out[jj], error);
1970                                         byte_error++;
1971                                         cum_error += error;
1972                                 }
1973                         }
1974                 }
1975                 m = m->next;
1976         }
1977
1978         if (byte_error)
1979                 TEST_ASSERT(byte_error <= 1,
1980                                 "HARQ output mismatch (%d) %d",
1981                                 byte_error, cum_error);
1982
1983         /* Validate total mbuf pkt length */
1984         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1985         TEST_ASSERT(total_data_size < pkt_len + 64,
1986                         "Length of data differ in original (%u) and filled (%u) op",
1987                         total_data_size, pkt_len);
1988
1989         return TEST_SUCCESS;
1990 }
1991
1992 static int
1993 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1994                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1995 {
1996         unsigned int i;
1997         int ret;
1998         struct op_data_entries *hard_data_orig =
1999                         &test_vector.entries[DATA_HARD_OUTPUT];
2000         struct op_data_entries *soft_data_orig =
2001                         &test_vector.entries[DATA_SOFT_OUTPUT];
2002         struct rte_bbdev_op_turbo_dec *ops_td;
2003         struct rte_bbdev_op_data *hard_output;
2004         struct rte_bbdev_op_data *soft_output;
2005         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
2006
2007         for (i = 0; i < n; ++i) {
2008                 ops_td = &ops[i]->turbo_dec;
2009                 hard_output = &ops_td->hard_output;
2010                 soft_output = &ops_td->soft_output;
2011
2012                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2013                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2014                                         "Returned iter_count (%d) > expected iter_count (%d)",
2015                                         ops_td->iter_count, ref_td->iter_count);
2016                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2017                 TEST_ASSERT_SUCCESS(ret,
2018                                 "Checking status and ordering for decoder failed");
2019
2020                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2021                                 hard_data_orig),
2022                                 "Hard output buffers (CB=%u) are not equal",
2023                                 i);
2024
2025                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
2026                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2027                                         soft_data_orig),
2028                                         "Soft output buffers (CB=%u) are not equal",
2029                                         i);
2030         }
2031
2032         return TEST_SUCCESS;
2033 }
2034
2035 /* Check Number of code blocks errors */
2036 static int
2037 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
2038 {
2039         unsigned int i;
2040         struct op_data_entries *hard_data_orig =
2041                         &test_vector.entries[DATA_HARD_OUTPUT];
2042         struct rte_bbdev_op_ldpc_dec *ops_td;
2043         struct rte_bbdev_op_data *hard_output;
2044         int errors = 0;
2045         struct rte_mbuf *m;
2046
2047         for (i = 0; i < n; ++i) {
2048                 ops_td = &ops[i]->ldpc_dec;
2049                 hard_output = &ops_td->hard_output;
2050                 m = hard_output->data;
2051                 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
2052                                 hard_data_orig->segments[0].addr,
2053                                 hard_data_orig->segments[0].length))
2054                         errors++;
2055         }
2056         return errors;
2057 }
2058
2059 static int
2060 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2061                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2062 {
2063         unsigned int i;
2064         int ret;
2065         struct op_data_entries *hard_data_orig =
2066                         &test_vector.entries[DATA_HARD_OUTPUT];
2067         struct op_data_entries *soft_data_orig =
2068                         &test_vector.entries[DATA_SOFT_OUTPUT];
2069         struct op_data_entries *harq_data_orig =
2070                                 &test_vector.entries[DATA_HARQ_OUTPUT];
2071         struct rte_bbdev_op_ldpc_dec *ops_td;
2072         struct rte_bbdev_op_data *hard_output;
2073         struct rte_bbdev_op_data *harq_output;
2074         struct rte_bbdev_op_data *soft_output;
2075         struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2076
2077         for (i = 0; i < n; ++i) {
2078                 ops_td = &ops[i]->ldpc_dec;
2079                 hard_output = &ops_td->hard_output;
2080                 harq_output = &ops_td->harq_combined_output;
2081                 soft_output = &ops_td->soft_output;
2082
2083                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2084                 TEST_ASSERT_SUCCESS(ret,
2085                                 "Checking status and ordering for decoder failed");
2086                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2087                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2088                                         "Returned iter_count (%d) > expected iter_count (%d)",
2089                                         ops_td->iter_count, ref_td->iter_count);
2090                 /*
2091                  * We can ignore output data when the decoding failed to
2092                  * converge or for loop-back cases
2093                  */
2094                 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2095                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2096                                 ) && (
2097                                 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2098                                                 )) == 0)
2099                         TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2100                                         hard_data_orig),
2101                                         "Hard output buffers (CB=%u) are not equal",
2102                                         i);
2103
2104                 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2105                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2106                                         soft_data_orig),
2107                                         "Soft output buffers (CB=%u) are not equal",
2108                                         i);
2109                 if (ref_op->ldpc_dec.op_flags &
2110                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2111                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2112                                         harq_data_orig, ops_td),
2113                                         "HARQ output buffers (CB=%u) are not equal",
2114                                         i);
2115                 }
2116                 if (ref_op->ldpc_dec.op_flags &
2117                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2118                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2119                                         harq_data_orig, ops_td),
2120                                         "HARQ output buffers (CB=%u) are not equal",
2121                                         i);
2122
2123         }
2124
2125         return TEST_SUCCESS;
2126 }
2127
2128
2129 static int
2130 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2131                 struct rte_bbdev_enc_op *ref_op)
2132 {
2133         unsigned int i;
2134         int ret;
2135         struct op_data_entries *hard_data_orig =
2136                         &test_vector.entries[DATA_HARD_OUTPUT];
2137
2138         for (i = 0; i < n; ++i) {
2139                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2140                 TEST_ASSERT_SUCCESS(ret,
2141                                 "Checking status and ordering for encoder failed");
2142                 TEST_ASSERT_SUCCESS(validate_op_chain(
2143                                 &ops[i]->turbo_enc.output,
2144                                 hard_data_orig),
2145                                 "Output buffers (CB=%u) are not equal",
2146                                 i);
2147         }
2148
2149         return TEST_SUCCESS;
2150 }
2151
2152 static int
2153 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2154                 struct rte_bbdev_enc_op *ref_op)
2155 {
2156         unsigned int i;
2157         int ret;
2158         struct op_data_entries *hard_data_orig =
2159                         &test_vector.entries[DATA_HARD_OUTPUT];
2160
2161         for (i = 0; i < n; ++i) {
2162                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2163                 TEST_ASSERT_SUCCESS(ret,
2164                                 "Checking status and ordering for encoder failed");
2165                 TEST_ASSERT_SUCCESS(validate_op_chain(
2166                                 &ops[i]->ldpc_enc.output,
2167                                 hard_data_orig),
2168                                 "Output buffers (CB=%u) are not equal",
2169                                 i);
2170         }
2171
2172         return TEST_SUCCESS;
2173 }
2174
2175 static void
2176 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2177 {
2178         unsigned int i;
2179         struct op_data_entries *entry;
2180
2181         op->turbo_dec = test_vector.turbo_dec;
2182         entry = &test_vector.entries[DATA_INPUT];
2183         for (i = 0; i < entry->nb_segments; ++i)
2184                 op->turbo_dec.input.length +=
2185                                 entry->segments[i].length;
2186 }
2187
2188 static void
2189 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2190 {
2191         unsigned int i;
2192         struct op_data_entries *entry;
2193
2194         op->ldpc_dec = test_vector.ldpc_dec;
2195         entry = &test_vector.entries[DATA_INPUT];
2196         for (i = 0; i < entry->nb_segments; ++i)
2197                 op->ldpc_dec.input.length +=
2198                                 entry->segments[i].length;
2199         if (test_vector.ldpc_dec.op_flags &
2200                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2201                 entry = &test_vector.entries[DATA_HARQ_INPUT];
2202                 for (i = 0; i < entry->nb_segments; ++i)
2203                         op->ldpc_dec.harq_combined_input.length +=
2204                                 entry->segments[i].length;
2205         }
2206 }
2207
2208
2209 static void
2210 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2211 {
2212         unsigned int i;
2213         struct op_data_entries *entry;
2214
2215         op->turbo_enc = test_vector.turbo_enc;
2216         entry = &test_vector.entries[DATA_INPUT];
2217         for (i = 0; i < entry->nb_segments; ++i)
2218                 op->turbo_enc.input.length +=
2219                                 entry->segments[i].length;
2220 }
2221
2222 static void
2223 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2224 {
2225         unsigned int i;
2226         struct op_data_entries *entry;
2227
2228         op->ldpc_enc = test_vector.ldpc_enc;
2229         entry = &test_vector.entries[DATA_INPUT];
2230         for (i = 0; i < entry->nb_segments; ++i)
2231                 op->ldpc_enc.input.length +=
2232                                 entry->segments[i].length;
2233 }
2234
2235 static uint32_t
2236 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2237 {
2238         uint8_t i;
2239         uint32_t c, r, tb_size = 0;
2240
2241         if (op->turbo_dec.code_block_mode) {
2242                 tb_size = op->turbo_dec.tb_params.k_neg;
2243         } else {
2244                 c = op->turbo_dec.tb_params.c;
2245                 r = op->turbo_dec.tb_params.r;
2246                 for (i = 0; i < c-r; i++)
2247                         tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2248                                 op->turbo_dec.tb_params.k_neg :
2249                                 op->turbo_dec.tb_params.k_pos;
2250         }
2251         return tb_size;
2252 }
2253
2254 static uint32_t
2255 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2256 {
2257         uint8_t i;
2258         uint32_t c, r, tb_size = 0;
2259         uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2260
2261         if (op->ldpc_dec.code_block_mode) {
2262                 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2263         } else {
2264                 c = op->ldpc_dec.tb_params.c;
2265                 r = op->ldpc_dec.tb_params.r;
2266                 for (i = 0; i < c-r; i++)
2267                         tb_size += sys_cols * op->ldpc_dec.z_c
2268                                         - op->ldpc_dec.n_filler;
2269         }
2270         return tb_size;
2271 }
2272
2273 static uint32_t
2274 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2275 {
2276         uint8_t i;
2277         uint32_t c, r, tb_size = 0;
2278
2279         if (op->turbo_enc.code_block_mode) {
2280                 tb_size = op->turbo_enc.tb_params.k_neg;
2281         } else {
2282                 c = op->turbo_enc.tb_params.c;
2283                 r = op->turbo_enc.tb_params.r;
2284                 for (i = 0; i < c-r; i++)
2285                         tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2286                                 op->turbo_enc.tb_params.k_neg :
2287                                 op->turbo_enc.tb_params.k_pos;
2288         }
2289         return tb_size;
2290 }
2291
2292 static uint32_t
2293 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2294 {
2295         uint8_t i;
2296         uint32_t c, r, tb_size = 0;
2297         uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2298
2299         if (op->turbo_enc.code_block_mode) {
2300                 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2301         } else {
2302                 c = op->turbo_enc.tb_params.c;
2303                 r = op->turbo_enc.tb_params.r;
2304                 for (i = 0; i < c-r; i++)
2305                         tb_size += sys_cols * op->ldpc_enc.z_c
2306                                         - op->ldpc_enc.n_filler;
2307         }
2308         return tb_size;
2309 }
2310
2311
2312 static int
2313 init_test_op_params(struct test_op_params *op_params,
2314                 enum rte_bbdev_op_type op_type, const int expected_status,
2315                 const int vector_mask, struct rte_mempool *ops_mp,
2316                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2317 {
2318         int ret = 0;
2319         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2320                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2321                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2322                                 &op_params->ref_dec_op, 1);
2323         else
2324                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2325                                 &op_params->ref_enc_op, 1);
2326
2327         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2328
2329         op_params->mp = ops_mp;
2330         op_params->burst_sz = burst_sz;
2331         op_params->num_to_process = num_to_process;
2332         op_params->num_lcores = num_lcores;
2333         op_params->vector_mask = vector_mask;
2334         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2335                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2336                 op_params->ref_dec_op->status = expected_status;
2337         else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2338                         || op_type == RTE_BBDEV_OP_LDPC_ENC)
2339                 op_params->ref_enc_op->status = expected_status;
2340         return 0;
2341 }
2342
2343 static int
2344 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2345                 struct test_op_params *op_params)
2346 {
2347         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2348         unsigned int i;
2349         struct active_device *ad;
2350         unsigned int burst_sz = get_burst_sz();
2351         enum rte_bbdev_op_type op_type = test_vector.op_type;
2352         const struct rte_bbdev_op_cap *capabilities = NULL;
2353
2354         ad = &active_devs[dev_id];
2355
2356         /* Check if device supports op_type */
2357         if (!is_avail_op(ad, test_vector.op_type))
2358                 return TEST_SUCCESS;
2359
2360         struct rte_bbdev_info info;
2361         rte_bbdev_info_get(ad->dev_id, &info);
2362         socket_id = GET_SOCKET(info.socket_id);
2363
2364         f_ret = create_mempools(ad, socket_id, op_type,
2365                         get_num_ops());
2366         if (f_ret != TEST_SUCCESS) {
2367                 printf("Couldn't create mempools");
2368                 goto fail;
2369         }
2370         if (op_type == RTE_BBDEV_OP_NONE)
2371                 op_type = RTE_BBDEV_OP_TURBO_ENC;
2372
2373         f_ret = init_test_op_params(op_params, test_vector.op_type,
2374                         test_vector.expected_status,
2375                         test_vector.mask,
2376                         ad->ops_mempool,
2377                         burst_sz,
2378                         get_num_ops(),
2379                         get_num_lcores());
2380         if (f_ret != TEST_SUCCESS) {
2381                 printf("Couldn't init test op params");
2382                 goto fail;
2383         }
2384
2385
2386         /* Find capabilities */
2387         const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2388         for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2389                 if (cap->type == test_vector.op_type) {
2390                         capabilities = cap;
2391                         break;
2392                 }
2393                 cap++;
2394         }
2395         TEST_ASSERT_NOT_NULL(capabilities,
2396                         "Couldn't find capabilities");
2397
2398         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2399                 create_reference_dec_op(op_params->ref_dec_op);
2400         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2401                 create_reference_enc_op(op_params->ref_enc_op);
2402         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2403                 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2404         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2405                 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2406
2407         for (i = 0; i < ad->nb_queues; ++i) {
2408                 f_ret = fill_queue_buffers(op_params,
2409                                 ad->in_mbuf_pool,
2410                                 ad->hard_out_mbuf_pool,
2411                                 ad->soft_out_mbuf_pool,
2412                                 ad->harq_in_mbuf_pool,
2413                                 ad->harq_out_mbuf_pool,
2414                                 ad->queue_ids[i],
2415                                 capabilities,
2416                                 info.drv.min_alignment,
2417                                 socket_id);
2418                 if (f_ret != TEST_SUCCESS) {
2419                         printf("Couldn't init queue buffers");
2420                         goto fail;
2421                 }
2422         }
2423
2424         /* Run test case function */
2425         t_ret = test_case_func(ad, op_params);
2426
2427         /* Free active device resources and return */
2428         free_buffers(ad, op_params);
2429         return t_ret;
2430
2431 fail:
2432         free_buffers(ad, op_params);
2433         return TEST_FAILED;
2434 }
2435
2436 /* Run given test function per active device per supported op type
2437  * per burst size.
2438  */
2439 static int
2440 run_test_case(test_case_function *test_case_func)
2441 {
2442         int ret = 0;
2443         uint8_t dev;
2444
2445         /* Alloc op_params */
2446         struct test_op_params *op_params = rte_zmalloc(NULL,
2447                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2448         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2449                         RTE_ALIGN(sizeof(struct test_op_params),
2450                                 RTE_CACHE_LINE_SIZE));
2451
2452         /* For each device run test case function */
2453         for (dev = 0; dev < nb_active_devs; ++dev)
2454                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2455
2456         rte_free(op_params);
2457
2458         return ret;
2459 }
2460
2461
2462 /* Push back the HARQ output from DDR to host */
2463 static void
2464 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2465                 struct rte_bbdev_dec_op **ops,
2466                 const uint16_t n)
2467 {
2468         uint16_t j;
2469         int save_status, ret;
2470         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2471         struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2472         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2473         bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2474         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2475         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2476         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2477         for (j = 0; j < n; ++j) {
2478                 if ((loopback && mem_out) || hc_out) {
2479                         save_status = ops[j]->status;
2480                         ops[j]->ldpc_dec.op_flags =
2481                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2482                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2483                         if (h_comp)
2484                                 ops[j]->ldpc_dec.op_flags +=
2485                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2486                         ops[j]->ldpc_dec.harq_combined_input.offset =
2487                                         harq_offset;
2488                         ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2489                         harq_offset += HARQ_INCR;
2490                         if (!loopback)
2491                                 ops[j]->ldpc_dec.harq_combined_input.length =
2492                                 ops[j]->ldpc_dec.harq_combined_output.length;
2493                         rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2494                                         &ops[j], 1);
2495                         ret = 0;
2496                         while (ret == 0)
2497                                 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2498                                                 dev_id, queue_id,
2499                                                 &ops_deq[j], 1);
2500                         ops[j]->ldpc_dec.op_flags = flags;
2501                         ops[j]->status = save_status;
2502                 }
2503         }
2504 }
2505
2506 /*
2507  * Push back the HARQ output from HW DDR to Host
2508  * Preload HARQ memory input and adjust HARQ offset
2509  */
2510 static void
2511 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2512                 struct rte_bbdev_dec_op **ops, const uint16_t n,
2513                 bool preload)
2514 {
2515         uint16_t j;
2516         int ret;
2517         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2518         struct rte_bbdev_op_data save_hc_in, save_hc_out;
2519         struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2520         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2521         bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2522         bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2523         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2524         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2525         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2526         for (j = 0; j < n; ++j) {
2527                 if ((mem_in || hc_in) && preload) {
2528                         save_hc_in = ops[j]->ldpc_dec.harq_combined_input;
2529                         save_hc_out = ops[j]->ldpc_dec.harq_combined_output;
2530                         ops[j]->ldpc_dec.op_flags =
2531                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2532                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2533                         if (h_comp)
2534                                 ops[j]->ldpc_dec.op_flags +=
2535                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2536                         ops[j]->ldpc_dec.harq_combined_output.offset =
2537                                         harq_offset;
2538                         ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2539                         rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2540                                         &ops[j], 1);
2541                         ret = 0;
2542                         while (ret == 0)
2543                                 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2544                                         dev_id, queue_id, &ops_deq[j], 1);
2545                         ops[j]->ldpc_dec.op_flags = flags;
2546                         ops[j]->ldpc_dec.harq_combined_input = save_hc_in;
2547                         ops[j]->ldpc_dec.harq_combined_output = save_hc_out;
2548                 }
2549                 /* Adjust HARQ offset when we reach external DDR */
2550                 if (mem_in || hc_in)
2551                         ops[j]->ldpc_dec.harq_combined_input.offset
2552                                 = harq_offset;
2553                 if (mem_out || hc_out)
2554                         ops[j]->ldpc_dec.harq_combined_output.offset
2555                                 = harq_offset;
2556                 harq_offset += HARQ_INCR;
2557         }
2558 }
2559
2560 static void
2561 dequeue_event_callback(uint16_t dev_id,
2562                 enum rte_bbdev_event_type event, void *cb_arg,
2563                 void *ret_param)
2564 {
2565         int ret;
2566         uint16_t i;
2567         uint64_t total_time;
2568         uint16_t deq, burst_sz, num_ops;
2569         uint16_t queue_id = *(uint16_t *) ret_param;
2570         struct rte_bbdev_info info;
2571         double tb_len_bits;
2572         struct thread_params *tp = cb_arg;
2573
2574         /* Find matching thread params using queue_id */
2575         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2576                 if (tp->queue_id == queue_id)
2577                         break;
2578
2579         if (i == MAX_QUEUES) {
2580                 printf("%s: Queue_id from interrupt details was not found!\n",
2581                                 __func__);
2582                 return;
2583         }
2584
2585         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2586                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2587                 printf(
2588                         "Dequeue interrupt handler called for incorrect event!\n");
2589                 return;
2590         }
2591
2592         burst_sz = rte_atomic16_read(&tp->burst_sz);
2593         num_ops = tp->op_params->num_to_process;
2594
2595         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2596                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2597                                 &tp->dec_ops[
2598                                         rte_atomic16_read(&tp->nb_dequeued)],
2599                                 burst_sz);
2600         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2601                 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2602                                 &tp->dec_ops[
2603                                         rte_atomic16_read(&tp->nb_dequeued)],
2604                                 burst_sz);
2605         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2606                 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2607                                 &tp->enc_ops[
2608                                         rte_atomic16_read(&tp->nb_dequeued)],
2609                                 burst_sz);
2610         else /*RTE_BBDEV_OP_TURBO_ENC*/
2611                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2612                                 &tp->enc_ops[
2613                                         rte_atomic16_read(&tp->nb_dequeued)],
2614                                 burst_sz);
2615
2616         if (deq < burst_sz) {
2617                 printf(
2618                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2619                         burst_sz, deq);
2620                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2621                 return;
2622         }
2623
2624         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2625                 rte_atomic16_add(&tp->nb_dequeued, deq);
2626                 return;
2627         }
2628
2629         total_time = rte_rdtsc_precise() - tp->start_time;
2630
2631         rte_bbdev_info_get(dev_id, &info);
2632
2633         ret = TEST_SUCCESS;
2634
2635         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2636                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2637                 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2638                                 tp->op_params->vector_mask);
2639                 /* get the max of iter_count for all dequeued ops */
2640                 for (i = 0; i < num_ops; ++i)
2641                         tp->iter_count = RTE_MAX(
2642                                         tp->dec_ops[i]->turbo_dec.iter_count,
2643                                         tp->iter_count);
2644                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2645         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2646                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2647                 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2648                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2649         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2650                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2651                 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2652                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2653         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2654                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2655                 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2656                                 tp->op_params->vector_mask);
2657                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2658         }
2659
2660         if (ret) {
2661                 printf("Buffers validation failed\n");
2662                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2663         }
2664
2665         switch (test_vector.op_type) {
2666         case RTE_BBDEV_OP_TURBO_DEC:
2667                 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2668                 break;
2669         case RTE_BBDEV_OP_TURBO_ENC:
2670                 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2671                 break;
2672         case RTE_BBDEV_OP_LDPC_DEC:
2673                 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2674                 break;
2675         case RTE_BBDEV_OP_LDPC_ENC:
2676                 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2677                 break;
2678         case RTE_BBDEV_OP_NONE:
2679                 tb_len_bits = 0.0;
2680                 break;
2681         default:
2682                 printf("Unknown op type: %d\n", test_vector.op_type);
2683                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2684                 return;
2685         }
2686
2687         tp->ops_per_sec += ((double)num_ops) /
2688                         ((double)total_time / (double)rte_get_tsc_hz());
2689         tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2690                         ((double)total_time / (double)rte_get_tsc_hz());
2691
2692         rte_atomic16_add(&tp->nb_dequeued, deq);
2693 }
2694
2695 static int
2696 throughput_intr_lcore_ldpc_dec(void *arg)
2697 {
2698         struct thread_params *tp = arg;
2699         unsigned int enqueued;
2700         const uint16_t queue_id = tp->queue_id;
2701         const uint16_t burst_sz = tp->op_params->burst_sz;
2702         const uint16_t num_to_process = tp->op_params->num_to_process;
2703         struct rte_bbdev_dec_op *ops[num_to_process];
2704         struct test_buffers *bufs = NULL;
2705         struct rte_bbdev_info info;
2706         int ret, i, j;
2707         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2708         uint16_t num_to_enq, enq;
2709
2710         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2711                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2712         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2713                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2714
2715         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2716                         "BURST_SIZE should be <= %u", MAX_BURST);
2717
2718         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2719                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2720                         tp->dev_id, queue_id);
2721
2722         rte_bbdev_info_get(tp->dev_id, &info);
2723
2724         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2725                         "NUM_OPS cannot exceed %u for this device",
2726                         info.drv.queue_size_lim);
2727
2728         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2729
2730         rte_atomic16_clear(&tp->processing_status);
2731         rte_atomic16_clear(&tp->nb_dequeued);
2732
2733         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2734                 rte_pause();
2735
2736         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2737                                 num_to_process);
2738         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2739                         num_to_process);
2740         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2741                 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2742                                 bufs->hard_outputs, bufs->soft_outputs,
2743                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2744
2745         /* Set counter to validate the ordering */
2746         for (j = 0; j < num_to_process; ++j)
2747                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2748
2749         for (j = 0; j < TEST_REPETITIONS; ++j) {
2750                 for (i = 0; i < num_to_process; ++i) {
2751                         if (!loopback)
2752                                 rte_pktmbuf_reset(
2753                                         ops[i]->ldpc_dec.hard_output.data);
2754                         if (hc_out || loopback)
2755                                 mbuf_reset(
2756                                 ops[i]->ldpc_dec.harq_combined_output.data);
2757                 }
2758
2759                 tp->start_time = rte_rdtsc_precise();
2760                 for (enqueued = 0; enqueued < num_to_process;) {
2761                         num_to_enq = burst_sz;
2762
2763                         if (unlikely(num_to_process - enqueued < num_to_enq))
2764                                 num_to_enq = num_to_process - enqueued;
2765
2766                         enq = 0;
2767                         do {
2768                                 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2769                                                 tp->dev_id,
2770                                                 queue_id, &ops[enqueued],
2771                                                 num_to_enq);
2772                         } while (unlikely(num_to_enq != enq));
2773                         enqueued += enq;
2774
2775                         /* Write to thread burst_sz current number of enqueued
2776                          * descriptors. It ensures that proper number of
2777                          * descriptors will be dequeued in callback
2778                          * function - needed for last batch in case where
2779                          * the number of operations is not a multiple of
2780                          * burst size.
2781                          */
2782                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2783
2784                         /* Wait until processing of previous batch is
2785                          * completed
2786                          */
2787                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2788                                         (int16_t) enqueued)
2789                                 rte_pause();
2790                 }
2791                 if (j != TEST_REPETITIONS - 1)
2792                         rte_atomic16_clear(&tp->nb_dequeued);
2793         }
2794
2795         return TEST_SUCCESS;
2796 }
2797
2798 static int
2799 throughput_intr_lcore_dec(void *arg)
2800 {
2801         struct thread_params *tp = arg;
2802         unsigned int enqueued;
2803         const uint16_t queue_id = tp->queue_id;
2804         const uint16_t burst_sz = tp->op_params->burst_sz;
2805         const uint16_t num_to_process = tp->op_params->num_to_process;
2806         struct rte_bbdev_dec_op *ops[num_to_process];
2807         struct test_buffers *bufs = NULL;
2808         struct rte_bbdev_info info;
2809         int ret, i, j;
2810         uint16_t num_to_enq, enq;
2811
2812         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2813                         "BURST_SIZE should be <= %u", MAX_BURST);
2814
2815         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2816                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2817                         tp->dev_id, queue_id);
2818
2819         rte_bbdev_info_get(tp->dev_id, &info);
2820
2821         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2822                         "NUM_OPS cannot exceed %u for this device",
2823                         info.drv.queue_size_lim);
2824
2825         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2826
2827         rte_atomic16_clear(&tp->processing_status);
2828         rte_atomic16_clear(&tp->nb_dequeued);
2829
2830         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2831                 rte_pause();
2832
2833         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2834                                 num_to_process);
2835         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2836                         num_to_process);
2837         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2838                 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2839                                 bufs->hard_outputs, bufs->soft_outputs,
2840                                 tp->op_params->ref_dec_op);
2841
2842         /* Set counter to validate the ordering */
2843         for (j = 0; j < num_to_process; ++j)
2844                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2845
2846         for (j = 0; j < TEST_REPETITIONS; ++j) {
2847                 for (i = 0; i < num_to_process; ++i)
2848                         rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2849
2850                 tp->start_time = rte_rdtsc_precise();
2851                 for (enqueued = 0; enqueued < num_to_process;) {
2852                         num_to_enq = burst_sz;
2853
2854                         if (unlikely(num_to_process - enqueued < num_to_enq))
2855                                 num_to_enq = num_to_process - enqueued;
2856
2857                         enq = 0;
2858                         do {
2859                                 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2860                                                 queue_id, &ops[enqueued],
2861                                                 num_to_enq);
2862                         } while (unlikely(num_to_enq != enq));
2863                         enqueued += enq;
2864
2865                         /* Write to thread burst_sz current number of enqueued
2866                          * descriptors. It ensures that proper number of
2867                          * descriptors will be dequeued in callback
2868                          * function - needed for last batch in case where
2869                          * the number of operations is not a multiple of
2870                          * burst size.
2871                          */
2872                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2873
2874                         /* Wait until processing of previous batch is
2875                          * completed
2876                          */
2877                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2878                                         (int16_t) enqueued)
2879                                 rte_pause();
2880                 }
2881                 if (j != TEST_REPETITIONS - 1)
2882                         rte_atomic16_clear(&tp->nb_dequeued);
2883         }
2884
2885         return TEST_SUCCESS;
2886 }
2887
2888 static int
2889 throughput_intr_lcore_enc(void *arg)
2890 {
2891         struct thread_params *tp = arg;
2892         unsigned int enqueued;
2893         const uint16_t queue_id = tp->queue_id;
2894         const uint16_t burst_sz = tp->op_params->burst_sz;
2895         const uint16_t num_to_process = tp->op_params->num_to_process;
2896         struct rte_bbdev_enc_op *ops[num_to_process];
2897         struct test_buffers *bufs = NULL;
2898         struct rte_bbdev_info info;
2899         int ret, i, j;
2900         uint16_t num_to_enq, enq;
2901
2902         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2903                         "BURST_SIZE should be <= %u", MAX_BURST);
2904
2905         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2906                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2907                         tp->dev_id, queue_id);
2908
2909         rte_bbdev_info_get(tp->dev_id, &info);
2910
2911         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2912                         "NUM_OPS cannot exceed %u for this device",
2913                         info.drv.queue_size_lim);
2914
2915         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2916
2917         rte_atomic16_clear(&tp->processing_status);
2918         rte_atomic16_clear(&tp->nb_dequeued);
2919
2920         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2921                 rte_pause();
2922
2923         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2924                         num_to_process);
2925         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2926                         num_to_process);
2927         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2928                 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2929                                 bufs->hard_outputs, tp->op_params->ref_enc_op);
2930
2931         /* Set counter to validate the ordering */
2932         for (j = 0; j < num_to_process; ++j)
2933                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2934
2935         for (j = 0; j < TEST_REPETITIONS; ++j) {
2936                 for (i = 0; i < num_to_process; ++i)
2937                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2938
2939                 tp->start_time = rte_rdtsc_precise();
2940                 for (enqueued = 0; enqueued < num_to_process;) {
2941                         num_to_enq = burst_sz;
2942
2943                         if (unlikely(num_to_process - enqueued < num_to_enq))
2944                                 num_to_enq = num_to_process - enqueued;
2945
2946                         enq = 0;
2947                         do {
2948                                 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2949                                                 queue_id, &ops[enqueued],
2950                                                 num_to_enq);
2951                         } while (unlikely(enq != num_to_enq));
2952                         enqueued += enq;
2953
2954                         /* Write to thread burst_sz current number of enqueued
2955                          * descriptors. It ensures that proper number of
2956                          * descriptors will be dequeued in callback
2957                          * function - needed for last batch in case where
2958                          * the number of operations is not a multiple of
2959                          * burst size.
2960                          */
2961                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2962
2963                         /* Wait until processing of previous batch is
2964                          * completed
2965                          */
2966                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2967                                         (int16_t) enqueued)
2968                                 rte_pause();
2969                 }
2970                 if (j != TEST_REPETITIONS - 1)
2971                         rte_atomic16_clear(&tp->nb_dequeued);
2972         }
2973
2974         return TEST_SUCCESS;
2975 }
2976
2977
2978 static int
2979 throughput_intr_lcore_ldpc_enc(void *arg)
2980 {
2981         struct thread_params *tp = arg;
2982         unsigned int enqueued;
2983         const uint16_t queue_id = tp->queue_id;
2984         const uint16_t burst_sz = tp->op_params->burst_sz;
2985         const uint16_t num_to_process = tp->op_params->num_to_process;
2986         struct rte_bbdev_enc_op *ops[num_to_process];
2987         struct test_buffers *bufs = NULL;
2988         struct rte_bbdev_info info;
2989         int ret, i, j;
2990         uint16_t num_to_enq, enq;
2991
2992         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2993                         "BURST_SIZE should be <= %u", MAX_BURST);
2994
2995         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2996                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2997                         tp->dev_id, queue_id);
2998
2999         rte_bbdev_info_get(tp->dev_id, &info);
3000
3001         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
3002                         "NUM_OPS cannot exceed %u for this device",
3003                         info.drv.queue_size_lim);
3004
3005         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3006
3007         rte_atomic16_clear(&tp->processing_status);
3008         rte_atomic16_clear(&tp->nb_dequeued);
3009
3010         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3011                 rte_pause();
3012
3013         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
3014                         num_to_process);
3015         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3016                         num_to_process);
3017         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3018                 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
3019                                 bufs->inputs, bufs->hard_outputs,
3020                                 tp->op_params->ref_enc_op);
3021
3022         /* Set counter to validate the ordering */
3023         for (j = 0; j < num_to_process; ++j)
3024                 ops[j]->opaque_data = (void *)(uintptr_t)j;
3025
3026         for (j = 0; j < TEST_REPETITIONS; ++j) {
3027                 for (i = 0; i < num_to_process; ++i)
3028                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
3029
3030                 tp->start_time = rte_rdtsc_precise();
3031                 for (enqueued = 0; enqueued < num_to_process;) {
3032                         num_to_enq = burst_sz;
3033
3034                         if (unlikely(num_to_process - enqueued < num_to_enq))
3035                                 num_to_enq = num_to_process - enqueued;
3036
3037                         enq = 0;
3038                         do {
3039                                 enq += rte_bbdev_enqueue_ldpc_enc_ops(
3040                                                 tp->dev_id,
3041                                                 queue_id, &ops[enqueued],
3042                                                 num_to_enq);
3043                         } while (unlikely(enq != num_to_enq));
3044                         enqueued += enq;
3045
3046                         /* Write to thread burst_sz current number of enqueued
3047                          * descriptors. It ensures that proper number of
3048                          * descriptors will be dequeued in callback
3049                          * function - needed for last batch in case where
3050                          * the number of operations is not a multiple of
3051                          * burst size.
3052                          */
3053                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
3054
3055                         /* Wait until processing of previous batch is
3056                          * completed
3057                          */
3058                         while (rte_atomic16_read(&tp->nb_dequeued) !=
3059                                         (int16_t) enqueued)
3060                                 rte_pause();
3061                 }
3062                 if (j != TEST_REPETITIONS - 1)
3063                         rte_atomic16_clear(&tp->nb_dequeued);
3064         }
3065
3066         return TEST_SUCCESS;
3067 }
3068
3069 static int
3070 throughput_pmd_lcore_dec(void *arg)
3071 {
3072         struct thread_params *tp = arg;
3073         uint16_t enq, deq;
3074         uint64_t total_time = 0, start_time;
3075         const uint16_t queue_id = tp->queue_id;
3076         const uint16_t burst_sz = tp->op_params->burst_sz;
3077         const uint16_t num_ops = tp->op_params->num_to_process;
3078         struct rte_bbdev_dec_op *ops_enq[num_ops];
3079         struct rte_bbdev_dec_op *ops_deq[num_ops];
3080         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3081         struct test_buffers *bufs = NULL;
3082         int i, j, ret;
3083         struct rte_bbdev_info info;
3084         uint16_t num_to_enq;
3085
3086         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3087                         "BURST_SIZE should be <= %u", MAX_BURST);
3088
3089         rte_bbdev_info_get(tp->dev_id, &info);
3090
3091         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3092                         "NUM_OPS cannot exceed %u for this device",
3093                         info.drv.queue_size_lim);
3094
3095         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3096
3097         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3098                 rte_pause();
3099
3100         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3101         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3102
3103         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3104                 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3105                                 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3106
3107         /* Set counter to validate the ordering */
3108         for (j = 0; j < num_ops; ++j)
3109                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3110
3111         for (i = 0; i < TEST_REPETITIONS; ++i) {
3112
3113                 for (j = 0; j < num_ops; ++j)
3114                         mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3115
3116                 start_time = rte_rdtsc_precise();
3117
3118                 for (enq = 0, deq = 0; enq < num_ops;) {
3119                         num_to_enq = burst_sz;
3120
3121                         if (unlikely(num_ops - enq < num_to_enq))
3122                                 num_to_enq = num_ops - enq;
3123
3124                         enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3125                                         queue_id, &ops_enq[enq], num_to_enq);
3126
3127                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3128                                         queue_id, &ops_deq[deq], enq - deq);
3129                 }
3130
3131                 /* dequeue the remaining */
3132                 while (deq < enq) {
3133                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3134                                         queue_id, &ops_deq[deq], enq - deq);
3135                 }
3136
3137                 total_time += rte_rdtsc_precise() - start_time;
3138         }
3139
3140         tp->iter_count = 0;
3141         /* get the max of iter_count for all dequeued ops */
3142         for (i = 0; i < num_ops; ++i) {
3143                 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3144                                 tp->iter_count);
3145         }
3146
3147         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3148                 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3149                                 tp->op_params->vector_mask);
3150                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3151         }
3152
3153         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3154
3155         double tb_len_bits = calc_dec_TB_size(ref_op);
3156
3157         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3158                         ((double)total_time / (double)rte_get_tsc_hz());
3159         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3160                         1000000.0) / ((double)total_time /
3161                         (double)rte_get_tsc_hz());
3162
3163         return TEST_SUCCESS;
3164 }
3165
3166 static int
3167 bler_pmd_lcore_ldpc_dec(void *arg)
3168 {
3169         struct thread_params *tp = arg;
3170         uint16_t enq, deq;
3171         uint64_t total_time = 0, start_time;
3172         const uint16_t queue_id = tp->queue_id;
3173         const uint16_t burst_sz = tp->op_params->burst_sz;
3174         const uint16_t num_ops = tp->op_params->num_to_process;
3175         struct rte_bbdev_dec_op *ops_enq[num_ops];
3176         struct rte_bbdev_dec_op *ops_deq[num_ops];
3177         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3178         struct test_buffers *bufs = NULL;
3179         int i, j, ret;
3180         float parity_bler = 0;
3181         struct rte_bbdev_info info;
3182         uint16_t num_to_enq;
3183         bool extDdr = check_bit(ldpc_cap_flags,
3184                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3185         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3186                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3187         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3188                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3189
3190         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3191                         "BURST_SIZE should be <= %u", MAX_BURST);
3192
3193         rte_bbdev_info_get(tp->dev_id, &info);
3194
3195         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3196                         "NUM_OPS cannot exceed %u for this device",
3197                         info.drv.queue_size_lim);
3198
3199         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3200
3201         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3202                 rte_pause();
3203
3204         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3205         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3206
3207         /* For BLER tests we need to enable early termination */
3208         if (!check_bit(ref_op->ldpc_dec.op_flags,
3209                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3210                 ref_op->ldpc_dec.op_flags +=
3211                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3212         ref_op->ldpc_dec.iter_max = get_iter_max();
3213         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3214
3215         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3216                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3217                                 bufs->hard_outputs, bufs->soft_outputs,
3218                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3219         generate_llr_input(num_ops, bufs->inputs, ref_op);
3220
3221         /* Set counter to validate the ordering */
3222         for (j = 0; j < num_ops; ++j)
3223                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3224
3225         for (i = 0; i < 1; ++i) { /* Could add more iterations */
3226                 for (j = 0; j < num_ops; ++j) {
3227                         if (!loopback)
3228                                 mbuf_reset(
3229                                 ops_enq[j]->ldpc_dec.hard_output.data);
3230                         if (hc_out || loopback)
3231                                 mbuf_reset(
3232                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3233                 }
3234                 if (extDdr) {
3235                         bool preload = i == (TEST_REPETITIONS - 1);
3236                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3237                                         num_ops, preload);
3238                 }
3239                 start_time = rte_rdtsc_precise();
3240
3241                 for (enq = 0, deq = 0; enq < num_ops;) {
3242                         num_to_enq = burst_sz;
3243
3244                         if (unlikely(num_ops - enq < num_to_enq))
3245                                 num_to_enq = num_ops - enq;
3246
3247                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3248                                         queue_id, &ops_enq[enq], num_to_enq);
3249
3250                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3251                                         queue_id, &ops_deq[deq], enq - deq);
3252                 }
3253
3254                 /* dequeue the remaining */
3255                 while (deq < enq) {
3256                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3257                                         queue_id, &ops_deq[deq], enq - deq);
3258                 }
3259
3260                 total_time += rte_rdtsc_precise() - start_time;
3261         }
3262
3263         tp->iter_count = 0;
3264         tp->iter_average = 0;
3265         /* get the max of iter_count for all dequeued ops */
3266         for (i = 0; i < num_ops; ++i) {
3267                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3268                                 tp->iter_count);
3269                 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3270                 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3271                         parity_bler += 1.0;
3272         }
3273
3274         parity_bler /= num_ops; /* This one is based on SYND */
3275         tp->iter_average /= num_ops;
3276         tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3277
3278         if (test_vector.op_type != RTE_BBDEV_OP_NONE
3279                         && tp->bler == 0
3280                         && parity_bler == 0
3281                         && !hc_out) {
3282                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3283                                 tp->op_params->vector_mask);
3284                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3285         }
3286
3287         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3288
3289         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3290         tp->ops_per_sec = ((double)num_ops * 1) /
3291                         ((double)total_time / (double)rte_get_tsc_hz());
3292         tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3293                         1000000.0) / ((double)total_time /
3294                         (double)rte_get_tsc_hz());
3295
3296         return TEST_SUCCESS;
3297 }
3298
3299 static int
3300 throughput_pmd_lcore_ldpc_dec(void *arg)
3301 {
3302         struct thread_params *tp = arg;
3303         uint16_t enq, deq;
3304         uint64_t total_time = 0, start_time;
3305         const uint16_t queue_id = tp->queue_id;
3306         const uint16_t burst_sz = tp->op_params->burst_sz;
3307         const uint16_t num_ops = tp->op_params->num_to_process;
3308         struct rte_bbdev_dec_op *ops_enq[num_ops];
3309         struct rte_bbdev_dec_op *ops_deq[num_ops];
3310         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3311         struct test_buffers *bufs = NULL;
3312         int i, j, ret;
3313         struct rte_bbdev_info info;
3314         uint16_t num_to_enq;
3315         bool extDdr = check_bit(ldpc_cap_flags,
3316                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3317         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3318                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3319         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3320                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3321
3322         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3323                         "BURST_SIZE should be <= %u", MAX_BURST);
3324
3325         rte_bbdev_info_get(tp->dev_id, &info);
3326
3327         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3328                         "NUM_OPS cannot exceed %u for this device",
3329                         info.drv.queue_size_lim);
3330
3331         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3332
3333         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3334                 rte_pause();
3335
3336         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3337         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3338
3339         /* For throughput tests we need to disable early termination */
3340         if (check_bit(ref_op->ldpc_dec.op_flags,
3341                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3342                 ref_op->ldpc_dec.op_flags -=
3343                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3344         ref_op->ldpc_dec.iter_max = get_iter_max();
3345         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3346
3347         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3348                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3349                                 bufs->hard_outputs, bufs->soft_outputs,
3350                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3351
3352         /* Set counter to validate the ordering */
3353         for (j = 0; j < num_ops; ++j)
3354                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3355
3356         for (i = 0; i < TEST_REPETITIONS; ++i) {
3357                 for (j = 0; j < num_ops; ++j) {
3358                         if (!loopback)
3359                                 mbuf_reset(
3360                                 ops_enq[j]->ldpc_dec.hard_output.data);
3361                         if (hc_out || loopback)
3362                                 mbuf_reset(
3363                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3364                 }
3365                 if (extDdr) {
3366                         bool preload = i == (TEST_REPETITIONS - 1);
3367                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3368                                         num_ops, preload);
3369                 }
3370                 start_time = rte_rdtsc_precise();
3371
3372                 for (enq = 0, deq = 0; enq < num_ops;) {
3373                         num_to_enq = burst_sz;
3374
3375                         if (unlikely(num_ops - enq < num_to_enq))
3376                                 num_to_enq = num_ops - enq;
3377
3378                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3379                                         queue_id, &ops_enq[enq], num_to_enq);
3380
3381                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3382                                         queue_id, &ops_deq[deq], enq - deq);
3383                 }
3384
3385                 /* dequeue the remaining */
3386                 while (deq < enq) {
3387                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3388                                         queue_id, &ops_deq[deq], enq - deq);
3389                 }
3390
3391                 total_time += rte_rdtsc_precise() - start_time;
3392         }
3393
3394         tp->iter_count = 0;
3395         /* get the max of iter_count for all dequeued ops */
3396         for (i = 0; i < num_ops; ++i) {
3397                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3398                                 tp->iter_count);
3399         }
3400         if (extDdr) {
3401                 /* Read loopback is not thread safe */
3402                 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3403         }
3404
3405         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3406                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3407                                 tp->op_params->vector_mask);
3408                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3409         }
3410
3411         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3412
3413         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3414
3415         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3416                         ((double)total_time / (double)rte_get_tsc_hz());
3417         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3418                         1000000.0) / ((double)total_time /
3419                         (double)rte_get_tsc_hz());
3420
3421         return TEST_SUCCESS;
3422 }
3423
3424 static int
3425 throughput_pmd_lcore_enc(void *arg)
3426 {
3427         struct thread_params *tp = arg;
3428         uint16_t enq, deq;
3429         uint64_t total_time = 0, start_time;
3430         const uint16_t queue_id = tp->queue_id;
3431         const uint16_t burst_sz = tp->op_params->burst_sz;
3432         const uint16_t num_ops = tp->op_params->num_to_process;
3433         struct rte_bbdev_enc_op *ops_enq[num_ops];
3434         struct rte_bbdev_enc_op *ops_deq[num_ops];
3435         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3436         struct test_buffers *bufs = NULL;
3437         int i, j, ret;
3438         struct rte_bbdev_info info;
3439         uint16_t num_to_enq;
3440
3441         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3442                         "BURST_SIZE should be <= %u", MAX_BURST);
3443
3444         rte_bbdev_info_get(tp->dev_id, &info);
3445
3446         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3447                         "NUM_OPS cannot exceed %u for this device",
3448                         info.drv.queue_size_lim);
3449
3450         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3451
3452         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3453                 rte_pause();
3454
3455         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3456                         num_ops);
3457         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3458                         num_ops);
3459         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3460                 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3461                                 bufs->hard_outputs, ref_op);
3462
3463         /* Set counter to validate the ordering */
3464         for (j = 0; j < num_ops; ++j)
3465                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3466
3467         for (i = 0; i < TEST_REPETITIONS; ++i) {
3468
3469                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3470                         for (j = 0; j < num_ops; ++j)
3471                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3472
3473                 start_time = rte_rdtsc_precise();
3474
3475                 for (enq = 0, deq = 0; enq < num_ops;) {
3476                         num_to_enq = burst_sz;
3477
3478                         if (unlikely(num_ops - enq < num_to_enq))
3479                                 num_to_enq = num_ops - enq;
3480
3481                         enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3482                                         queue_id, &ops_enq[enq], num_to_enq);
3483
3484                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3485                                         queue_id, &ops_deq[deq], enq - deq);
3486                 }
3487
3488                 /* dequeue the remaining */
3489                 while (deq < enq) {
3490                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3491                                         queue_id, &ops_deq[deq], enq - deq);
3492                 }
3493
3494                 total_time += rte_rdtsc_precise() - start_time;
3495         }
3496
3497         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3498                 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3499                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3500         }
3501
3502         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3503
3504         double tb_len_bits = calc_enc_TB_size(ref_op);
3505
3506         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3507                         ((double)total_time / (double)rte_get_tsc_hz());
3508         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3509                         / 1000000.0) / ((double)total_time /
3510                         (double)rte_get_tsc_hz());
3511
3512         return TEST_SUCCESS;
3513 }
3514
3515 static int
3516 throughput_pmd_lcore_ldpc_enc(void *arg)
3517 {
3518         struct thread_params *tp = arg;
3519         uint16_t enq, deq;
3520         uint64_t total_time = 0, start_time;
3521         const uint16_t queue_id = tp->queue_id;
3522         const uint16_t burst_sz = tp->op_params->burst_sz;
3523         const uint16_t num_ops = tp->op_params->num_to_process;
3524         struct rte_bbdev_enc_op *ops_enq[num_ops];
3525         struct rte_bbdev_enc_op *ops_deq[num_ops];
3526         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3527         struct test_buffers *bufs = NULL;
3528         int i, j, ret;
3529         struct rte_bbdev_info info;
3530         uint16_t num_to_enq;
3531
3532         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3533                         "BURST_SIZE should be <= %u", MAX_BURST);
3534
3535         rte_bbdev_info_get(tp->dev_id, &info);
3536
3537         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3538                         "NUM_OPS cannot exceed %u for this device",
3539                         info.drv.queue_size_lim);
3540
3541         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3542
3543         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3544                 rte_pause();
3545
3546         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3547                         num_ops);
3548         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3549                         num_ops);
3550         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3551                 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3552                                 bufs->hard_outputs, ref_op);
3553
3554         /* Set counter to validate the ordering */
3555         for (j = 0; j < num_ops; ++j)
3556                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3557
3558         for (i = 0; i < TEST_REPETITIONS; ++i) {
3559
3560                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3561                         for (j = 0; j < num_ops; ++j)
3562                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3563
3564                 start_time = rte_rdtsc_precise();
3565
3566                 for (enq = 0, deq = 0; enq < num_ops;) {
3567                         num_to_enq = burst_sz;
3568
3569                         if (unlikely(num_ops - enq < num_to_enq))
3570                                 num_to_enq = num_ops - enq;
3571
3572                         enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3573                                         queue_id, &ops_enq[enq], num_to_enq);
3574
3575                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3576                                         queue_id, &ops_deq[deq], enq - deq);
3577                 }
3578
3579                 /* dequeue the remaining */
3580                 while (deq < enq) {
3581                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3582                                         queue_id, &ops_deq[deq], enq - deq);
3583                 }
3584
3585                 total_time += rte_rdtsc_precise() - start_time;
3586         }
3587
3588         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3589                 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3590                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3591         }
3592
3593         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3594
3595         double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3596
3597         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3598                         ((double)total_time / (double)rte_get_tsc_hz());
3599         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3600                         / 1000000.0) / ((double)total_time /
3601                         (double)rte_get_tsc_hz());
3602
3603         return TEST_SUCCESS;
3604 }
3605
3606 static void
3607 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3608 {
3609         unsigned int iter = 0;
3610         double total_mops = 0, total_mbps = 0;
3611
3612         for (iter = 0; iter < used_cores; iter++) {
3613                 printf(
3614                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3615                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3616                         t_params[iter].mbps);
3617                 total_mops += t_params[iter].ops_per_sec;
3618                 total_mbps += t_params[iter].mbps;
3619         }
3620         printf(
3621                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3622                 used_cores, total_mops, total_mbps);
3623 }
3624
3625 /* Aggregate the performance results over the number of cores used */
3626 static void
3627 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3628 {
3629         unsigned int core_idx = 0;
3630         double total_mops = 0, total_mbps = 0;
3631         uint8_t iter_count = 0;
3632
3633         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3634                 printf(
3635                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3636                         t_params[core_idx].lcore_id,
3637                         t_params[core_idx].ops_per_sec,
3638                         t_params[core_idx].mbps,
3639                         t_params[core_idx].iter_count);
3640                 total_mops += t_params[core_idx].ops_per_sec;
3641                 total_mbps += t_params[core_idx].mbps;
3642                 iter_count = RTE_MAX(iter_count,
3643                                 t_params[core_idx].iter_count);
3644         }
3645         printf(
3646                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3647                 used_cores, total_mops, total_mbps, iter_count);
3648 }
3649
3650 /* Aggregate the performance results over the number of cores used */
3651 static void
3652 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3653 {
3654         unsigned int core_idx = 0;
3655         double total_mbps = 0, total_bler = 0, total_iter = 0;
3656         double snr = get_snr();
3657
3658         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3659                 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3660                                 t_params[core_idx].lcore_id,
3661                                 t_params[core_idx].bler * 100,
3662                                 t_params[core_idx].iter_average,
3663                                 t_params[core_idx].mbps,
3664                                 get_vector_filename());
3665                 total_mbps += t_params[core_idx].mbps;
3666                 total_bler += t_params[core_idx].bler;
3667                 total_iter += t_params[core_idx].iter_average;
3668         }
3669         total_bler /= used_cores;
3670         total_iter /= used_cores;
3671
3672         printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3673                         snr, total_bler * 100, total_iter, get_iter_max(),
3674                         total_mbps, get_vector_filename());
3675 }
3676
3677 /*
3678  * Test function that determines BLER wireless performance
3679  */
3680 static int
3681 bler_test(struct active_device *ad,
3682                 struct test_op_params *op_params)
3683 {
3684         int ret;
3685         unsigned int lcore_id, used_cores = 0;
3686         struct thread_params *t_params;
3687         struct rte_bbdev_info info;
3688         lcore_function_t *bler_function;
3689         uint16_t num_lcores;
3690         const char *op_type_str;
3691
3692         rte_bbdev_info_get(ad->dev_id, &info);
3693
3694         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3695         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3696                         test_vector.op_type);
3697
3698         printf("+ ------------------------------------------------------- +\n");
3699         printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3700                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3701                         op_params->num_to_process, op_params->num_lcores,
3702                         op_type_str,
3703                         intr_enabled ? "Interrupt mode" : "PMD mode",
3704                         (double)rte_get_tsc_hz() / 1000000000.0);
3705
3706         /* Set number of lcores */
3707         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3708                         ? ad->nb_queues
3709                         : op_params->num_lcores;
3710
3711         /* Allocate memory for thread parameters structure */
3712         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3713                         RTE_CACHE_LINE_SIZE);
3714         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3715                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3716                                 RTE_CACHE_LINE_SIZE));
3717
3718         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3719                 bler_function = bler_pmd_lcore_ldpc_dec;
3720         else
3721                 return TEST_SKIPPED;
3722
3723         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3724
3725         /* Main core is set at first entry */
3726         t_params[0].dev_id = ad->dev_id;
3727         t_params[0].lcore_id = rte_lcore_id();
3728         t_params[0].op_params = op_params;
3729         t_params[0].queue_id = ad->queue_ids[used_cores++];
3730         t_params[0].iter_count = 0;
3731
3732         RTE_LCORE_FOREACH_WORKER(lcore_id) {
3733                 if (used_cores >= num_lcores)
3734                         break;
3735
3736                 t_params[used_cores].dev_id = ad->dev_id;
3737                 t_params[used_cores].lcore_id = lcore_id;
3738                 t_params[used_cores].op_params = op_params;
3739                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3740                 t_params[used_cores].iter_count = 0;
3741
3742                 rte_eal_remote_launch(bler_function,
3743                                 &t_params[used_cores++], lcore_id);
3744         }
3745
3746         rte_atomic16_set(&op_params->sync, SYNC_START);
3747         ret = bler_function(&t_params[0]);
3748
3749         /* Main core is always used */
3750         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3751                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3752
3753         print_dec_bler(t_params, num_lcores);
3754
3755         /* Return if test failed */
3756         if (ret) {
3757                 rte_free(t_params);
3758                 return ret;
3759         }
3760
3761         /* Function to print something  here*/
3762         rte_free(t_params);
3763         return ret;
3764 }
3765
3766 /*
3767  * Test function that determines how long an enqueue + dequeue of a burst
3768  * takes on available lcores.
3769  */
3770 static int
3771 throughput_test(struct active_device *ad,
3772                 struct test_op_params *op_params)
3773 {
3774         int ret;
3775         unsigned int lcore_id, used_cores = 0;
3776         struct thread_params *t_params, *tp;
3777         struct rte_bbdev_info info;
3778         lcore_function_t *throughput_function;
3779         uint16_t num_lcores;
3780         const char *op_type_str;
3781
3782         rte_bbdev_info_get(ad->dev_id, &info);
3783
3784         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3785         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3786                         test_vector.op_type);
3787
3788         printf("+ ------------------------------------------------------- +\n");
3789         printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3790                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3791                         op_params->num_to_process, op_params->num_lcores,
3792                         op_type_str,
3793                         intr_enabled ? "Interrupt mode" : "PMD mode",
3794                         (double)rte_get_tsc_hz() / 1000000000.0);
3795
3796         /* Set number of lcores */
3797         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3798                         ? ad->nb_queues
3799                         : op_params->num_lcores;
3800
3801         /* Allocate memory for thread parameters structure */
3802         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3803                         RTE_CACHE_LINE_SIZE);
3804         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3805                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3806                                 RTE_CACHE_LINE_SIZE));
3807
3808         if (intr_enabled) {
3809                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3810                         throughput_function = throughput_intr_lcore_dec;
3811                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3812                         throughput_function = throughput_intr_lcore_ldpc_dec;
3813                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3814                         throughput_function = throughput_intr_lcore_enc;
3815                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3816                         throughput_function = throughput_intr_lcore_ldpc_enc;
3817                 else
3818                         throughput_function = throughput_intr_lcore_enc;
3819
3820                 /* Dequeue interrupt callback registration */
3821                 ret = rte_bbdev_callback_register(ad->dev_id,
3822                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3823                                 t_params);
3824                 if (ret < 0) {
3825                         rte_free(t_params);
3826                         return ret;
3827                 }
3828         } else {
3829                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3830                         throughput_function = throughput_pmd_lcore_dec;
3831                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3832                         throughput_function = throughput_pmd_lcore_ldpc_dec;
3833                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3834                         throughput_function = throughput_pmd_lcore_enc;
3835                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3836                         throughput_function = throughput_pmd_lcore_ldpc_enc;
3837                 else
3838                         throughput_function = throughput_pmd_lcore_enc;
3839         }
3840
3841         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3842
3843         /* Main core is set at first entry */
3844         t_params[0].dev_id = ad->dev_id;
3845         t_params[0].lcore_id = rte_lcore_id();
3846         t_params[0].op_params = op_params;
3847         t_params[0].queue_id = ad->queue_ids[used_cores++];
3848         t_params[0].iter_count = 0;
3849
3850         RTE_LCORE_FOREACH_WORKER(lcore_id) {
3851                 if (used_cores >= num_lcores)
3852                         break;
3853
3854                 t_params[used_cores].dev_id = ad->dev_id;
3855                 t_params[used_cores].lcore_id = lcore_id;
3856                 t_params[used_cores].op_params = op_params;
3857                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3858                 t_params[used_cores].iter_count = 0;
3859
3860                 rte_eal_remote_launch(throughput_function,
3861                                 &t_params[used_cores++], lcore_id);
3862         }
3863
3864         rte_atomic16_set(&op_params->sync, SYNC_START);
3865         ret = throughput_function(&t_params[0]);
3866
3867         /* Main core is always used */
3868         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3869                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3870
3871         /* Return if test failed */
3872         if (ret) {
3873                 rte_free(t_params);
3874                 return ret;
3875         }
3876
3877         /* Print throughput if interrupts are disabled and test passed */
3878         if (!intr_enabled) {
3879                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3880                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3881                         print_dec_throughput(t_params, num_lcores);
3882                 else
3883                         print_enc_throughput(t_params, num_lcores);
3884                 rte_free(t_params);
3885                 return ret;
3886         }
3887
3888         /* In interrupt TC we need to wait for the interrupt callback to deqeue
3889          * all pending operations. Skip waiting for queues which reported an
3890          * error using processing_status variable.
3891          * Wait for main lcore operations.
3892          */
3893         tp = &t_params[0];
3894         while ((rte_atomic16_read(&tp->nb_dequeued) <
3895                         op_params->num_to_process) &&
3896                         (rte_atomic16_read(&tp->processing_status) !=
3897                         TEST_FAILED))
3898                 rte_pause();
3899
3900         tp->ops_per_sec /= TEST_REPETITIONS;
3901         tp->mbps /= TEST_REPETITIONS;
3902         ret |= (int)rte_atomic16_read(&tp->processing_status);
3903
3904         /* Wait for worker lcores operations */
3905         for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3906                 tp = &t_params[used_cores];
3907
3908                 while ((rte_atomic16_read(&tp->nb_dequeued) <
3909                                 op_params->num_to_process) &&
3910                                 (rte_atomic16_read(&tp->processing_status) !=
3911                                 TEST_FAILED))
3912                         rte_pause();
3913
3914                 tp->ops_per_sec /= TEST_REPETITIONS;
3915                 tp->mbps /= TEST_REPETITIONS;
3916                 ret |= (int)rte_atomic16_read(&tp->processing_status);
3917         }
3918
3919         /* Print throughput if test passed */
3920         if (!ret) {
3921                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3922                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3923                         print_dec_throughput(t_params, num_lcores);
3924                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3925                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3926                         print_enc_throughput(t_params, num_lcores);
3927         }
3928
3929         rte_free(t_params);
3930         return ret;
3931 }
3932
3933 static int
3934 latency_test_dec(struct rte_mempool *mempool,
3935                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3936                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3937                 const uint16_t num_to_process, uint16_t burst_sz,
3938                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3939 {
3940         int ret = TEST_SUCCESS;
3941         uint16_t i, j, dequeued;
3942         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3943         uint64_t start_time = 0, last_time = 0;
3944
3945         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3946                 uint16_t enq = 0, deq = 0;
3947                 bool first_time = true;
3948                 last_time = 0;
3949
3950                 if (unlikely(num_to_process - dequeued < burst_sz))
3951                         burst_sz = num_to_process - dequeued;
3952
3953                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3954                 TEST_ASSERT_SUCCESS(ret,
3955                                 "rte_bbdev_dec_op_alloc_bulk() failed");
3956                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3957                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3958                                         bufs->inputs,
3959                                         bufs->hard_outputs,
3960                                         bufs->soft_outputs,
3961                                         ref_op);
3962
3963                 /* Set counter to validate the ordering */
3964                 for (j = 0; j < burst_sz; ++j)
3965                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3966
3967                 start_time = rte_rdtsc_precise();
3968
3969                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
3970                                 burst_sz);
3971                 TEST_ASSERT(enq == burst_sz,
3972                                 "Error enqueueing burst, expected %u, got %u",
3973                                 burst_sz, enq);
3974
3975                 /* Dequeue */
3976                 do {
3977                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3978                                         &ops_deq[deq], burst_sz - deq);
3979                         if (likely(first_time && (deq > 0))) {
3980                                 last_time = rte_rdtsc_precise() - start_time;
3981                                 first_time = false;
3982                         }
3983                 } while (unlikely(burst_sz != deq));
3984
3985                 *max_time = RTE_MAX(*max_time, last_time);
3986                 *min_time = RTE_MIN(*min_time, last_time);
3987                 *total_time += last_time;
3988
3989                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3990                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
3991                                         vector_mask);
3992                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3993                 }
3994
3995                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3996                 dequeued += deq;
3997         }
3998
3999         return i;
4000 }
4001
4002 /* Test case for latency/validation for LDPC Decoder */
4003 static int
4004 latency_test_ldpc_dec(struct rte_mempool *mempool,
4005                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
4006                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
4007                 const uint16_t num_to_process, uint16_t burst_sz,
4008                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time,
4009                 bool disable_et)
4010 {
4011         int ret = TEST_SUCCESS;
4012         uint16_t i, j, dequeued;
4013         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4014         uint64_t start_time = 0, last_time = 0;
4015         bool extDdr = ldpc_cap_flags &
4016                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4017
4018         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4019                 uint16_t enq = 0, deq = 0;
4020                 bool first_time = true;
4021                 last_time = 0;
4022
4023                 if (unlikely(num_to_process - dequeued < burst_sz))
4024                         burst_sz = num_to_process - dequeued;
4025
4026                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4027                 TEST_ASSERT_SUCCESS(ret,
4028                                 "rte_bbdev_dec_op_alloc_bulk() failed");
4029
4030                 /* For latency tests we need to disable early termination */
4031                 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags,
4032                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
4033                         ref_op->ldpc_dec.op_flags -=
4034                                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
4035                 ref_op->ldpc_dec.iter_max = get_iter_max();
4036                 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
4037
4038                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4039                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4040                                         bufs->inputs,
4041                                         bufs->hard_outputs,
4042                                         bufs->soft_outputs,
4043                                         bufs->harq_inputs,
4044                                         bufs->harq_outputs,
4045                                         ref_op);
4046
4047                 if (extDdr)
4048                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4049                                         burst_sz, true);
4050
4051                 /* Set counter to validate the ordering */
4052                 for (j = 0; j < burst_sz; ++j)
4053                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4054
4055                 start_time = rte_rdtsc_precise();
4056
4057                 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4058                                 &ops_enq[enq], burst_sz);
4059                 TEST_ASSERT(enq == burst_sz,
4060                                 "Error enqueueing burst, expected %u, got %u",
4061                                 burst_sz, enq);
4062
4063                 /* Dequeue */
4064                 do {
4065                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4066                                         &ops_deq[deq], burst_sz - deq);
4067                         if (likely(first_time && (deq > 0))) {
4068                                 last_time = rte_rdtsc_precise() - start_time;
4069                                 first_time = false;
4070                         }
4071                 } while (unlikely(burst_sz != deq));
4072
4073                 *max_time = RTE_MAX(*max_time, last_time);
4074                 *min_time = RTE_MIN(*min_time, last_time);
4075                 *total_time += last_time;
4076
4077                 if (extDdr)
4078                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4079
4080                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4081                         ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4082                                         vector_mask);
4083                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4084                 }
4085
4086                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4087                 dequeued += deq;
4088         }
4089         return i;
4090 }
4091
4092 static int
4093 latency_test_enc(struct rte_mempool *mempool,
4094                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4095                 uint16_t dev_id, uint16_t queue_id,
4096                 const uint16_t num_to_process, uint16_t burst_sz,
4097                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4098 {
4099         int ret = TEST_SUCCESS;
4100         uint16_t i, j, dequeued;
4101         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4102         uint64_t start_time = 0, last_time = 0;
4103
4104         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4105                 uint16_t enq = 0, deq = 0;
4106                 bool first_time = true;
4107                 last_time = 0;
4108
4109                 if (unlikely(num_to_process - dequeued < burst_sz))
4110                         burst_sz = num_to_process - dequeued;
4111
4112                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4113                 TEST_ASSERT_SUCCESS(ret,
4114                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4115                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4116                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4117                                         bufs->inputs,
4118                                         bufs->hard_outputs,
4119                                         ref_op);
4120
4121                 /* Set counter to validate the ordering */
4122                 for (j = 0; j < burst_sz; ++j)
4123                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4124
4125                 start_time = rte_rdtsc_precise();
4126
4127                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4128                                 burst_sz);
4129                 TEST_ASSERT(enq == burst_sz,
4130                                 "Error enqueueing burst, expected %u, got %u",
4131                                 burst_sz, enq);
4132
4133                 /* Dequeue */
4134                 do {
4135                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4136                                         &ops_deq[deq], burst_sz - deq);
4137                         if (likely(first_time && (deq > 0))) {
4138                                 last_time += rte_rdtsc_precise() - start_time;
4139                                 first_time = false;
4140                         }
4141                 } while (unlikely(burst_sz != deq));
4142
4143                 *max_time = RTE_MAX(*max_time, last_time);
4144                 *min_time = RTE_MIN(*min_time, last_time);
4145                 *total_time += last_time;
4146
4147                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4148                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4149                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4150                 }
4151
4152                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4153                 dequeued += deq;
4154         }
4155
4156         return i;
4157 }
4158
4159 static int
4160 latency_test_ldpc_enc(struct rte_mempool *mempool,
4161                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4162                 uint16_t dev_id, uint16_t queue_id,
4163                 const uint16_t num_to_process, uint16_t burst_sz,
4164                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4165 {
4166         int ret = TEST_SUCCESS;
4167         uint16_t i, j, dequeued;
4168         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4169         uint64_t start_time = 0, last_time = 0;
4170
4171         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4172                 uint16_t enq = 0, deq = 0;
4173                 bool first_time = true;
4174                 last_time = 0;
4175
4176                 if (unlikely(num_to_process - dequeued < burst_sz))
4177                         burst_sz = num_to_process - dequeued;
4178
4179                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4180                 TEST_ASSERT_SUCCESS(ret,
4181                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4182                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4183                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4184                                         bufs->inputs,
4185                                         bufs->hard_outputs,
4186                                         ref_op);
4187
4188                 /* Set counter to validate the ordering */
4189                 for (j = 0; j < burst_sz; ++j)
4190                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4191
4192                 start_time = rte_rdtsc_precise();
4193
4194                 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4195                                 &ops_enq[enq], burst_sz);
4196                 TEST_ASSERT(enq == burst_sz,
4197                                 "Error enqueueing burst, expected %u, got %u",
4198                                 burst_sz, enq);
4199
4200                 /* Dequeue */
4201                 do {
4202                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4203                                         &ops_deq[deq], burst_sz - deq);
4204                         if (likely(first_time && (deq > 0))) {
4205                                 last_time += rte_rdtsc_precise() - start_time;
4206                                 first_time = false;
4207                         }
4208                 } while (unlikely(burst_sz != deq));
4209
4210                 *max_time = RTE_MAX(*max_time, last_time);
4211                 *min_time = RTE_MIN(*min_time, last_time);
4212                 *total_time += last_time;
4213
4214                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4215                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4216                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4217                 }
4218
4219                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4220                 dequeued += deq;
4221         }
4222
4223         return i;
4224 }
4225
4226 /* Common function for running validation and latency test cases */
4227 static int
4228 validation_latency_test(struct active_device *ad,
4229                 struct test_op_params *op_params, bool latency_flag)
4230 {
4231         int iter;
4232         uint16_t burst_sz = op_params->burst_sz;
4233         const uint16_t num_to_process = op_params->num_to_process;
4234         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4235         const uint16_t queue_id = ad->queue_ids[0];
4236         struct test_buffers *bufs = NULL;
4237         struct rte_bbdev_info info;
4238         uint64_t total_time, min_time, max_time;
4239         const char *op_type_str;
4240
4241         total_time = max_time = 0;
4242         min_time = UINT64_MAX;
4243
4244         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4245                         "BURST_SIZE should be <= %u", MAX_BURST);
4246
4247         rte_bbdev_info_get(ad->dev_id, &info);
4248         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4249
4250         op_type_str = rte_bbdev_op_type_str(op_type);
4251         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4252
4253         printf("+ ------------------------------------------------------- +\n");
4254         if (latency_flag)
4255                 printf("== test: latency\ndev:");
4256         else
4257                 printf("== test: validation\ndev:");
4258         printf("%s, burst size: %u, num ops: %u, op type: %s\n",
4259                         info.dev_name, burst_sz, num_to_process, op_type_str);
4260
4261         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4262                 iter = latency_test_dec(op_params->mp, bufs,
4263                                 op_params->ref_dec_op, op_params->vector_mask,
4264                                 ad->dev_id, queue_id, num_to_process,
4265                                 burst_sz, &total_time, &min_time, &max_time);
4266         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4267                 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4268                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4269                                 num_to_process, burst_sz, &total_time,
4270                                 &min_time, &max_time);
4271         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4272                 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4273                                 op_params->ref_dec_op, op_params->vector_mask,
4274                                 ad->dev_id, queue_id, num_to_process,
4275                                 burst_sz, &total_time, &min_time, &max_time,
4276                                 latency_flag);
4277         else /* RTE_BBDEV_OP_TURBO_ENC */
4278                 iter = latency_test_enc(op_params->mp, bufs,
4279                                 op_params->ref_enc_op,
4280                                 ad->dev_id, queue_id,
4281                                 num_to_process, burst_sz, &total_time,
4282                                 &min_time, &max_time);
4283
4284         if (iter <= 0)
4285                 return TEST_FAILED;
4286
4287         printf("Operation latency:\n"
4288                         "\tavg: %lg cycles, %lg us\n"
4289                         "\tmin: %lg cycles, %lg us\n"
4290                         "\tmax: %lg cycles, %lg us\n",
4291                         (double)total_time / (double)iter,
4292                         (double)(total_time * 1000000) / (double)iter /
4293                         (double)rte_get_tsc_hz(), (double)min_time,
4294                         (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4295                         (double)max_time, (double)(max_time * 1000000) /
4296                         (double)rte_get_tsc_hz());
4297
4298         return TEST_SUCCESS;
4299 }
4300
4301 static int
4302 latency_test(struct active_device *ad, struct test_op_params *op_params)
4303 {
4304         return validation_latency_test(ad, op_params, true);
4305 }
4306
4307 static int
4308 validation_test(struct active_device *ad, struct test_op_params *op_params)
4309 {
4310         return validation_latency_test(ad, op_params, false);
4311 }
4312
4313 #ifdef RTE_BBDEV_OFFLOAD_COST
4314 static int
4315 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4316                 struct rte_bbdev_stats *stats)
4317 {
4318         struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4319         struct rte_bbdev_stats *q_stats;
4320
4321         if (queue_id >= dev->data->num_queues)
4322                 return -1;
4323
4324         q_stats = &dev->data->queues[queue_id].queue_stats;
4325
4326         stats->enqueued_count = q_stats->enqueued_count;
4327         stats->dequeued_count = q_stats->dequeued_count;
4328         stats->enqueue_err_count = q_stats->enqueue_err_count;
4329         stats->dequeue_err_count = q_stats->dequeue_err_count;
4330         stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4331
4332         return 0;
4333 }
4334
4335 static int
4336 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4337                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4338                 uint16_t queue_id, const uint16_t num_to_process,
4339                 uint16_t burst_sz, struct test_time_stats *time_st)
4340 {
4341         int i, dequeued, ret;
4342         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4343         uint64_t enq_start_time, deq_start_time;
4344         uint64_t enq_sw_last_time, deq_last_time;
4345         struct rte_bbdev_stats stats;
4346
4347         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4348                 uint16_t enq = 0, deq = 0;
4349
4350                 if (unlikely(num_to_process - dequeued < burst_sz))
4351                         burst_sz = num_to_process - dequeued;
4352
4353                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4354                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4355                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4356                                         bufs->inputs,
4357                                         bufs->hard_outputs,
4358                                         bufs->soft_outputs,
4359                                         ref_op);
4360
4361                 /* Start time meas for enqueue function offload latency */
4362                 enq_start_time = rte_rdtsc_precise();
4363                 do {
4364                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4365                                         &ops_enq[enq], burst_sz - enq);
4366                 } while (unlikely(burst_sz != enq));
4367
4368                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4369                 TEST_ASSERT_SUCCESS(ret,
4370                                 "Failed to get stats for queue (%u) of device (%u)",
4371                                 queue_id, dev_id);
4372
4373                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4374                                 stats.acc_offload_cycles;
4375                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4376                                 enq_sw_last_time);
4377                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4378                                 enq_sw_last_time);
4379                 time_st->enq_sw_total_time += enq_sw_last_time;
4380
4381                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4382                                 stats.acc_offload_cycles);
4383                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4384                                 stats.acc_offload_cycles);
4385                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4386
4387                 /* give time for device to process ops */
4388                 rte_delay_us(200);
4389
4390                 /* Start time meas for dequeue function offload latency */
4391                 deq_start_time = rte_rdtsc_precise();
4392                 /* Dequeue one operation */
4393                 do {
4394                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4395                                         &ops_deq[deq], 1);
4396                 } while (unlikely(deq != 1));
4397
4398                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4399                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4400                                 deq_last_time);
4401                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4402                                 deq_last_time);
4403                 time_st->deq_total_time += deq_last_time;
4404
4405                 /* Dequeue remaining operations if needed*/
4406                 while (burst_sz != deq)
4407                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4408                                         &ops_deq[deq], burst_sz - deq);
4409
4410                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4411                 dequeued += deq;
4412         }
4413
4414         return i;
4415 }
4416
4417 static int
4418 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4419                 struct test_buffers *bufs,
4420                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4421                 uint16_t queue_id, const uint16_t num_to_process,
4422                 uint16_t burst_sz, struct test_time_stats *time_st)
4423 {
4424         int i, dequeued, ret;
4425         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4426         uint64_t enq_start_time, deq_start_time;
4427         uint64_t enq_sw_last_time, deq_last_time;
4428         struct rte_bbdev_stats stats;
4429         bool extDdr = ldpc_cap_flags &
4430                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4431
4432         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4433                 uint16_t enq = 0, deq = 0;
4434
4435                 if (unlikely(num_to_process - dequeued < burst_sz))
4436                         burst_sz = num_to_process - dequeued;
4437
4438                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4439                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4440                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4441                                         bufs->inputs,
4442                                         bufs->hard_outputs,
4443                                         bufs->soft_outputs,
4444                                         bufs->harq_inputs,
4445                                         bufs->harq_outputs,
4446                                         ref_op);
4447
4448                 if (extDdr)
4449                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4450                                         burst_sz, true);
4451
4452                 /* Start time meas for enqueue function offload latency */
4453                 enq_start_time = rte_rdtsc_precise();
4454                 do {
4455                         enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4456                                         &ops_enq[enq], burst_sz - enq);
4457                 } while (unlikely(burst_sz != enq));
4458
4459                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4460                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4461                 TEST_ASSERT_SUCCESS(ret,
4462                                 "Failed to get stats for queue (%u) of device (%u)",
4463                                 queue_id, dev_id);
4464
4465                 enq_sw_last_time -= stats.acc_offload_cycles;
4466                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4467                                 enq_sw_last_time);
4468                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4469                                 enq_sw_last_time);
4470                 time_st->enq_sw_total_time += enq_sw_last_time;
4471
4472                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4473                                 stats.acc_offload_cycles);
4474                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4475                                 stats.acc_offload_cycles);
4476                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4477
4478                 /* give time for device to process ops */
4479                 rte_delay_us(200);
4480
4481                 /* Start time meas for dequeue function offload latency */
4482                 deq_start_time = rte_rdtsc_precise();
4483                 /* Dequeue one operation */
4484                 do {
4485                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4486                                         &ops_deq[deq], 1);
4487                 } while (unlikely(deq != 1));
4488
4489                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4490                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4491                                 deq_last_time);
4492                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4493                                 deq_last_time);
4494                 time_st->deq_total_time += deq_last_time;
4495
4496                 /* Dequeue remaining operations if needed*/
4497                 while (burst_sz != deq)
4498                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4499                                         &ops_deq[deq], burst_sz - deq);
4500
4501                 if (extDdr) {
4502                         /* Read loopback is not thread safe */
4503                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4504                 }
4505
4506                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4507                 dequeued += deq;
4508         }
4509
4510         return i;
4511 }
4512
4513 static int
4514 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4515                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4516                 uint16_t queue_id, const uint16_t num_to_process,
4517                 uint16_t burst_sz, struct test_time_stats *time_st)
4518 {
4519         int i, dequeued, ret;
4520         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4521         uint64_t enq_start_time, deq_start_time;
4522         uint64_t enq_sw_last_time, deq_last_time;
4523         struct rte_bbdev_stats stats;
4524
4525         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4526                 uint16_t enq = 0, deq = 0;
4527
4528                 if (unlikely(num_to_process - dequeued < burst_sz))
4529                         burst_sz = num_to_process - dequeued;
4530
4531                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4532                 TEST_ASSERT_SUCCESS(ret,
4533                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4534                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4535                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4536                                         bufs->inputs,
4537                                         bufs->hard_outputs,
4538                                         ref_op);
4539
4540                 /* Start time meas for enqueue function offload latency */
4541                 enq_start_time = rte_rdtsc_precise();
4542                 do {
4543                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4544                                         &ops_enq[enq], burst_sz - enq);
4545                 } while (unlikely(burst_sz != enq));
4546
4547                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4548
4549                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4550                 TEST_ASSERT_SUCCESS(ret,
4551                                 "Failed to get stats for queue (%u) of device (%u)",
4552                                 queue_id, dev_id);
4553                 enq_sw_last_time -= stats.acc_offload_cycles;
4554                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4555                                 enq_sw_last_time);
4556                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4557                                 enq_sw_last_time);
4558                 time_st->enq_sw_total_time += enq_sw_last_time;
4559
4560                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4561                                 stats.acc_offload_cycles);
4562                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4563                                 stats.acc_offload_cycles);
4564                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4565
4566                 /* give time for device to process ops */
4567                 rte_delay_us(200);
4568
4569                 /* Start time meas for dequeue function offload latency */
4570                 deq_start_time = rte_rdtsc_precise();
4571                 /* Dequeue one operation */
4572                 do {
4573                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4574                                         &ops_deq[deq], 1);
4575                 } while (unlikely(deq != 1));
4576
4577                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4578                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4579                                 deq_last_time);
4580                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4581                                 deq_last_time);
4582                 time_st->deq_total_time += deq_last_time;
4583
4584                 while (burst_sz != deq)
4585                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4586                                         &ops_deq[deq], burst_sz - deq);
4587
4588                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4589                 dequeued += deq;
4590         }
4591
4592         return i;
4593 }
4594
4595 static int
4596 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4597                 struct test_buffers *bufs,
4598                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4599                 uint16_t queue_id, const uint16_t num_to_process,
4600                 uint16_t burst_sz, struct test_time_stats *time_st)
4601 {
4602         int i, dequeued, ret;
4603         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4604         uint64_t enq_start_time, deq_start_time;
4605         uint64_t enq_sw_last_time, deq_last_time;
4606         struct rte_bbdev_stats stats;
4607
4608         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4609                 uint16_t enq = 0, deq = 0;
4610
4611                 if (unlikely(num_to_process - dequeued < burst_sz))
4612                         burst_sz = num_to_process - dequeued;
4613
4614                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4615                 TEST_ASSERT_SUCCESS(ret,
4616                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4617                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4618                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4619                                         bufs->inputs,
4620                                         bufs->hard_outputs,
4621                                         ref_op);
4622
4623                 /* Start time meas for enqueue function offload latency */
4624                 enq_start_time = rte_rdtsc_precise();
4625                 do {
4626                         enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4627                                         &ops_enq[enq], burst_sz - enq);
4628                 } while (unlikely(burst_sz != enq));
4629
4630                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4631                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4632                 TEST_ASSERT_SUCCESS(ret,
4633                                 "Failed to get stats for queue (%u) of device (%u)",
4634                                 queue_id, dev_id);
4635
4636                 enq_sw_last_time -= stats.acc_offload_cycles;
4637                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4638                                 enq_sw_last_time);
4639                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4640                                 enq_sw_last_time);
4641                 time_st->enq_sw_total_time += enq_sw_last_time;
4642
4643                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4644                                 stats.acc_offload_cycles);
4645                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4646                                 stats.acc_offload_cycles);
4647                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4648
4649                 /* give time for device to process ops */
4650                 rte_delay_us(200);
4651
4652                 /* Start time meas for dequeue function offload latency */
4653                 deq_start_time = rte_rdtsc_precise();
4654                 /* Dequeue one operation */
4655                 do {
4656                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4657                                         &ops_deq[deq], 1);
4658                 } while (unlikely(deq != 1));
4659
4660                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4661                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4662                                 deq_last_time);
4663                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4664                                 deq_last_time);
4665                 time_st->deq_total_time += deq_last_time;
4666
4667                 while (burst_sz != deq)
4668                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4669                                         &ops_deq[deq], burst_sz - deq);
4670
4671                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4672                 dequeued += deq;
4673         }
4674
4675         return i;
4676 }
4677 #endif
4678
4679 static int
4680 offload_cost_test(struct active_device *ad,
4681                 struct test_op_params *op_params)
4682 {
4683 #ifndef RTE_BBDEV_OFFLOAD_COST
4684         RTE_SET_USED(ad);
4685         RTE_SET_USED(op_params);
4686         printf("Offload latency test is disabled.\n");
4687         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4688         return TEST_SKIPPED;
4689 #else
4690         int iter;
4691         uint16_t burst_sz = op_params->burst_sz;
4692         const uint16_t num_to_process = op_params->num_to_process;
4693         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4694         const uint16_t queue_id = ad->queue_ids[0];
4695         struct test_buffers *bufs = NULL;
4696         struct rte_bbdev_info info;
4697         const char *op_type_str;
4698         struct test_time_stats time_st;
4699
4700         memset(&time_st, 0, sizeof(struct test_time_stats));
4701         time_st.enq_sw_min_time = UINT64_MAX;
4702         time_st.enq_acc_min_time = UINT64_MAX;
4703         time_st.deq_min_time = UINT64_MAX;
4704
4705         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4706                         "BURST_SIZE should be <= %u", MAX_BURST);
4707
4708         rte_bbdev_info_get(ad->dev_id, &info);
4709         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4710
4711         op_type_str = rte_bbdev_op_type_str(op_type);
4712         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4713
4714         printf("+ ------------------------------------------------------- +\n");
4715         printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4716                         info.dev_name, burst_sz, num_to_process, op_type_str);
4717
4718         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4719                 iter = offload_latency_test_dec(op_params->mp, bufs,
4720                                 op_params->ref_dec_op, ad->dev_id, queue_id,
4721                                 num_to_process, burst_sz, &time_st);
4722         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4723                 iter = offload_latency_test_enc(op_params->mp, bufs,
4724                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4725                                 num_to_process, burst_sz, &time_st);
4726         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4727                 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4728                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4729                                 num_to_process, burst_sz, &time_st);
4730         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4731                 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4732                         op_params->ref_dec_op, ad->dev_id, queue_id,
4733                         num_to_process, burst_sz, &time_st);
4734         else
4735                 iter = offload_latency_test_enc(op_params->mp, bufs,
4736                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4737                                 num_to_process, burst_sz, &time_st);
4738
4739         if (iter <= 0)
4740                 return TEST_FAILED;
4741
4742         printf("Enqueue driver offload cost latency:\n"
4743                         "\tavg: %lg cycles, %lg us\n"
4744                         "\tmin: %lg cycles, %lg us\n"
4745                         "\tmax: %lg cycles, %lg us\n"
4746                         "Enqueue accelerator offload cost latency:\n"
4747                         "\tavg: %lg cycles, %lg us\n"
4748                         "\tmin: %lg cycles, %lg us\n"
4749                         "\tmax: %lg cycles, %lg us\n",
4750                         (double)time_st.enq_sw_total_time / (double)iter,
4751                         (double)(time_st.enq_sw_total_time * 1000000) /
4752                         (double)iter / (double)rte_get_tsc_hz(),
4753                         (double)time_st.enq_sw_min_time,
4754                         (double)(time_st.enq_sw_min_time * 1000000) /
4755                         rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4756                         (double)(time_st.enq_sw_max_time * 1000000) /
4757                         rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4758                         (double)iter,
4759                         (double)(time_st.enq_acc_total_time * 1000000) /
4760                         (double)iter / (double)rte_get_tsc_hz(),
4761                         (double)time_st.enq_acc_min_time,
4762                         (double)(time_st.enq_acc_min_time * 1000000) /
4763                         rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4764                         (double)(time_st.enq_acc_max_time * 1000000) /
4765                         rte_get_tsc_hz());
4766
4767         printf("Dequeue offload cost latency - one op:\n"
4768                         "\tavg: %lg cycles, %lg us\n"
4769                         "\tmin: %lg cycles, %lg us\n"
4770                         "\tmax: %lg cycles, %lg us\n",
4771                         (double)time_st.deq_total_time / (double)iter,
4772                         (double)(time_st.deq_total_time * 1000000) /
4773                         (double)iter / (double)rte_get_tsc_hz(),
4774                         (double)time_st.deq_min_time,
4775                         (double)(time_st.deq_min_time * 1000000) /
4776                         rte_get_tsc_hz(), (double)time_st.deq_max_time,
4777                         (double)(time_st.deq_max_time * 1000000) /
4778                         rte_get_tsc_hz());
4779
4780         return TEST_SUCCESS;
4781 #endif
4782 }
4783
4784 #ifdef RTE_BBDEV_OFFLOAD_COST
4785 static int
4786 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4787                 const uint16_t num_to_process, uint16_t burst_sz,
4788                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4789                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4790 {
4791         int i, deq_total;
4792         struct rte_bbdev_dec_op *ops[MAX_BURST];
4793         uint64_t deq_start_time, deq_last_time;
4794
4795         /* Test deq offload latency from an empty queue */
4796
4797         for (i = 0, deq_total = 0; deq_total < num_to_process;
4798                         ++i, deq_total += burst_sz) {
4799                 deq_start_time = rte_rdtsc_precise();
4800
4801                 if (unlikely(num_to_process - deq_total < burst_sz))
4802                         burst_sz = num_to_process - deq_total;
4803                 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4804                         rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4805                                         burst_sz);
4806                 else
4807                         rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4808                                         burst_sz);
4809
4810                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4811                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4812                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4813                 *deq_total_time += deq_last_time;
4814         }
4815
4816         return i;
4817 }
4818
4819 static int
4820 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4821                 const uint16_t num_to_process, uint16_t burst_sz,
4822                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4823                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4824 {
4825         int i, deq_total;
4826         struct rte_bbdev_enc_op *ops[MAX_BURST];
4827         uint64_t deq_start_time, deq_last_time;
4828
4829         /* Test deq offload latency from an empty queue */
4830         for (i = 0, deq_total = 0; deq_total < num_to_process;
4831                         ++i, deq_total += burst_sz) {
4832                 deq_start_time = rte_rdtsc_precise();
4833
4834                 if (unlikely(num_to_process - deq_total < burst_sz))
4835                         burst_sz = num_to_process - deq_total;
4836                 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4837                         rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4838                                         burst_sz);
4839                 else
4840                         rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4841                                         burst_sz);
4842
4843                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4844                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4845                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4846                 *deq_total_time += deq_last_time;
4847         }
4848
4849         return i;
4850 }
4851
4852 #endif
4853
4854 static int
4855 offload_latency_empty_q_test(struct active_device *ad,
4856                 struct test_op_params *op_params)
4857 {
4858 #ifndef RTE_BBDEV_OFFLOAD_COST
4859         RTE_SET_USED(ad);
4860         RTE_SET_USED(op_params);
4861         printf("Offload latency empty dequeue test is disabled.\n");
4862         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4863         return TEST_SKIPPED;
4864 #else
4865         int iter;
4866         uint64_t deq_total_time, deq_min_time, deq_max_time;
4867         uint16_t burst_sz = op_params->burst_sz;
4868         const uint16_t num_to_process = op_params->num_to_process;
4869         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4870         const uint16_t queue_id = ad->queue_ids[0];
4871         struct rte_bbdev_info info;
4872         const char *op_type_str;
4873
4874         deq_total_time = deq_max_time = 0;
4875         deq_min_time = UINT64_MAX;
4876
4877         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4878                         "BURST_SIZE should be <= %u", MAX_BURST);
4879
4880         rte_bbdev_info_get(ad->dev_id, &info);
4881
4882         op_type_str = rte_bbdev_op_type_str(op_type);
4883         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4884
4885         printf("+ ------------------------------------------------------- +\n");
4886         printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4887                         info.dev_name, burst_sz, num_to_process, op_type_str);
4888
4889         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4890                         op_type == RTE_BBDEV_OP_LDPC_DEC)
4891                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4892                                 num_to_process, burst_sz, &deq_total_time,
4893                                 &deq_min_time, &deq_max_time, op_type);
4894         else
4895                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4896                                 num_to_process, burst_sz, &deq_total_time,
4897                                 &deq_min_time, &deq_max_time, op_type);
4898
4899         if (iter <= 0)
4900                 return TEST_FAILED;
4901
4902         printf("Empty dequeue offload:\n"
4903                         "\tavg: %lg cycles, %lg us\n"
4904                         "\tmin: %lg cycles, %lg us\n"
4905                         "\tmax: %lg cycles, %lg us\n",
4906                         (double)deq_total_time / (double)iter,
4907                         (double)(deq_total_time * 1000000) / (double)iter /
4908                         (double)rte_get_tsc_hz(), (double)deq_min_time,
4909                         (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4910                         (double)deq_max_time, (double)(deq_max_time * 1000000) /
4911                         rte_get_tsc_hz());
4912
4913         return TEST_SUCCESS;
4914 #endif
4915 }
4916
4917 static int
4918 bler_tc(void)
4919 {
4920         return run_test_case(bler_test);
4921 }
4922
4923 static int
4924 throughput_tc(void)
4925 {
4926         return run_test_case(throughput_test);
4927 }
4928
4929 static int
4930 offload_cost_tc(void)
4931 {
4932         return run_test_case(offload_cost_test);
4933 }
4934
4935 static int
4936 offload_latency_empty_q_tc(void)
4937 {
4938         return run_test_case(offload_latency_empty_q_test);
4939 }
4940
4941 static int
4942 latency_tc(void)
4943 {
4944         return run_test_case(latency_test);
4945 }
4946
4947 static int
4948 validation_tc(void)
4949 {
4950         return run_test_case(validation_test);
4951 }
4952
4953 static int
4954 interrupt_tc(void)
4955 {
4956         return run_test_case(throughput_test);
4957 }
4958
4959 static struct unit_test_suite bbdev_bler_testsuite = {
4960         .suite_name = "BBdev BLER Tests",
4961         .setup = testsuite_setup,
4962         .teardown = testsuite_teardown,
4963         .unit_test_cases = {
4964                 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
4965                 TEST_CASES_END() /**< NULL terminate unit test array */
4966         }
4967 };
4968
4969 static struct unit_test_suite bbdev_throughput_testsuite = {
4970         .suite_name = "BBdev Throughput Tests",
4971         .setup = testsuite_setup,
4972         .teardown = testsuite_teardown,
4973         .unit_test_cases = {
4974                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
4975                 TEST_CASES_END() /**< NULL terminate unit test array */
4976         }
4977 };
4978
4979 static struct unit_test_suite bbdev_validation_testsuite = {
4980         .suite_name = "BBdev Validation Tests",
4981         .setup = testsuite_setup,
4982         .teardown = testsuite_teardown,
4983         .unit_test_cases = {
4984                 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc),
4985                 TEST_CASES_END() /**< NULL terminate unit test array */
4986         }
4987 };
4988
4989 static struct unit_test_suite bbdev_latency_testsuite = {
4990         .suite_name = "BBdev Latency Tests",
4991         .setup = testsuite_setup,
4992         .teardown = testsuite_teardown,
4993         .unit_test_cases = {
4994                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4995                 TEST_CASES_END() /**< NULL terminate unit test array */
4996         }
4997 };
4998
4999 static struct unit_test_suite bbdev_offload_cost_testsuite = {
5000         .suite_name = "BBdev Offload Cost Tests",
5001         .setup = testsuite_setup,
5002         .teardown = testsuite_teardown,
5003         .unit_test_cases = {
5004                 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
5005                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
5006                 TEST_CASES_END() /**< NULL terminate unit test array */
5007         }
5008 };
5009
5010 static struct unit_test_suite bbdev_interrupt_testsuite = {
5011         .suite_name = "BBdev Interrupt Tests",
5012         .setup = interrupt_testsuite_setup,
5013         .teardown = testsuite_teardown,
5014         .unit_test_cases = {
5015                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
5016                 TEST_CASES_END() /**< NULL terminate unit test array */
5017         }
5018 };
5019
5020 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
5021 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
5022 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
5023 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
5024 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
5025 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);