app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19 #include <rte_interrupts.h>
  20
  21 #include "main.h"
  22 #include "test_bbdev_vector.h"
  23
  24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  25
  26 #define MAX_QUEUES RTE_MAX_LCORE
  27 #define TEST_REPETITIONS 1000
  28
  29 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
  30 #include <fpga_lte_fec.h>
  31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
  32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
  33 #define VF_UL_4G_QUEUE_VALUE 4
  34 #define VF_DL_4G_QUEUE_VALUE 4
  35 #define UL_4G_BANDWIDTH 3
  36 #define DL_4G_BANDWIDTH 3
  37 #define UL_4G_LOAD_BALANCE 128
  38 #define DL_4G_LOAD_BALANCE 128
  39 #define FLR_4G_TIMEOUT 610
  40 #endif
  41
  42 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
  43 #include <rte_pmd_fpga_5gnr_fec.h>
  44 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
  45 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
  46 #define VF_UL_5G_QUEUE_VALUE 4
  47 #define VF_DL_5G_QUEUE_VALUE 4
  48 #define UL_5G_BANDWIDTH 3
  49 #define DL_5G_BANDWIDTH 3
  50 #define UL_5G_LOAD_BALANCE 128
  51 #define DL_5G_LOAD_BALANCE 128
  52 #define FLR_5G_TIMEOUT 610
  53 #endif
  54
  55 #ifdef RTE_BASEBAND_ACC100
  56 #include <rte_acc100_cfg.h>
  57 #define ACC100PF_DRIVER_NAME   ("intel_acc100_pf")
  58 #define ACC100VF_DRIVER_NAME   ("intel_acc100_vf")
  59 #define ACC100_QMGR_NUM_AQS 16
  60 #define ACC100_QMGR_NUM_QGS 2
  61 #define ACC100_QMGR_AQ_DEPTH 5
  62 #define ACC100_QMGR_INVALID_IDX -1
  63 #define ACC100_QMGR_RR 1
  64 #define ACC100_QOS_GBR 0
  65 #endif
  66
  67 #define OPS_CACHE_SIZE 256U
  68 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  69
  70 #define SYNC_WAIT 0
  71 #define SYNC_START 1
  72 #define INVALID_OPAQUE -1
  73
  74 #define INVALID_QUEUE_ID -1
  75 /* Increment for next code block in external HARQ memory */
  76 #define HARQ_INCR 32768
  77 /* Headroom for filler LLRs insertion in HARQ buffer */
  78 #define FILLER_HEADROOM 1024
  79 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
  80 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
  81 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
  82 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
  83 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
  84 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
  85 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
  86 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
  87 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
  88
  89 static struct test_bbdev_vector test_vector;
  90
  91 /* Switch between PMD and Interrupt for throughput TC */
  92 static bool intr_enabled;
  93
  94 /* LLR arithmetic representation for numerical conversion */
  95 static int ldpc_llr_decimals;
  96 static int ldpc_llr_size;
  97 /* Keep track of the LDPC decoder device capability flag */
  98 static uint32_t ldpc_cap_flags;
  99
 100 /* Represents tested active devices */
 101 static struct active_device {
 102         const char *driver_name;
 103         uint8_t dev_id;
 104         uint16_t supported_ops;
 105         uint16_t queue_ids[MAX_QUEUES];
 106         uint16_t nb_queues;
 107         struct rte_mempool *ops_mempool;
 108         struct rte_mempool *in_mbuf_pool;
 109         struct rte_mempool *hard_out_mbuf_pool;
 110         struct rte_mempool *soft_out_mbuf_pool;
 111         struct rte_mempool *harq_in_mbuf_pool;
 112         struct rte_mempool *harq_out_mbuf_pool;
 113 } active_devs[RTE_BBDEV_MAX_DEVS];
 114
 115 static uint8_t nb_active_devs;
 116
 117 /* Data buffers used by BBDEV ops */
 118 struct test_buffers {
 119         struct rte_bbdev_op_data *inputs;
 120         struct rte_bbdev_op_data *hard_outputs;
 121         struct rte_bbdev_op_data *soft_outputs;
 122         struct rte_bbdev_op_data *harq_inputs;
 123         struct rte_bbdev_op_data *harq_outputs;
 124 };
 125
 126 /* Operation parameters specific for given test case */
 127 struct test_op_params {
 128         struct rte_mempool *mp;
 129         struct rte_bbdev_dec_op *ref_dec_op;
 130         struct rte_bbdev_enc_op *ref_enc_op;
 131         uint16_t burst_sz;
 132         uint16_t num_to_process;
 133         uint16_t num_lcores;
 134         int vector_mask;
 135         rte_atomic16_t sync;
 136         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
 137 };
 138
 139 /* Contains per lcore params */
 140 struct thread_params {
 141         uint8_t dev_id;
 142         uint16_t queue_id;
 143         uint32_t lcore_id;
 144         uint64_t start_time;
 145         double ops_per_sec;
 146         double mbps;
 147         uint8_t iter_count;
 148         double iter_average;
 149         double bler;
 150         rte_atomic16_t nb_dequeued;
 151         rte_atomic16_t processing_status;
 152         rte_atomic16_t burst_sz;
 153         struct test_op_params *op_params;
 154         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
 155         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
 156 };
 157
 158 #ifdef RTE_BBDEV_OFFLOAD_COST
 159 /* Stores time statistics */
 160 struct test_time_stats {
 161         /* Stores software enqueue total working time */
 162         uint64_t enq_sw_total_time;
 163         /* Stores minimum value of software enqueue working time */
 164         uint64_t enq_sw_min_time;
 165         /* Stores maximum value of software enqueue working time */
 166         uint64_t enq_sw_max_time;
 167         /* Stores turbo enqueue total working time */
 168         uint64_t enq_acc_total_time;
 169         /* Stores minimum value of accelerator enqueue working time */
 170         uint64_t enq_acc_min_time;
 171         /* Stores maximum value of accelerator enqueue working time */
 172         uint64_t enq_acc_max_time;
 173         /* Stores dequeue total working time */
 174         uint64_t deq_total_time;
 175         /* Stores minimum value of dequeue working time */
 176         uint64_t deq_min_time;
 177         /* Stores maximum value of dequeue working time */
 178         uint64_t deq_max_time;
 179 };
 180 #endif
 181
 182 typedef int (test_case_function)(struct active_device *ad,
 183                 struct test_op_params *op_params);
 184
 185 static inline void
 186 mbuf_reset(struct rte_mbuf *m)
 187 {
 188         m->pkt_len = 0;
 189
 190         do {
 191                 m->data_len = 0;
 192                 m = m->next;
 193         } while (m != NULL);
 194 }
 195
 196 /* Read flag value 0/1 from bitmap */
 197 static inline bool
 198 check_bit(uint32_t bitmap, uint32_t bitmask)
 199 {
 200         return bitmap & bitmask;
 201 }
 202
 203 static inline void
 204 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 205 {
 206         ad->supported_ops |= (1 << op_type);
 207 }
 208
 209 static inline bool
 210 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 211 {
 212         return ad->supported_ops & (1 << op_type);
 213 }
 214
 215 static inline bool
 216 flags_match(uint32_t flags_req, uint32_t flags_present)
 217 {
 218         return (flags_req & flags_present) == flags_req;
 219 }
 220
 221 static void
 222 clear_soft_out_cap(uint32_t *op_flags)
 223 {
 224         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 225         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 226         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 227 }
 228
 229 static int
 230 check_dev_cap(const struct rte_bbdev_info *dev_info)
 231 {
 232         unsigned int i;
 233         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
 234                 nb_harq_inputs, nb_harq_outputs;
 235         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 236
 237         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 238         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 239         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 240         nb_harq_inputs  = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
 241         nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
 242
 243         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 244                 if (op_cap->type != test_vector.op_type)
 245                         continue;
 246
 247                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 248                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 249                                         &op_cap->cap.turbo_dec;
 250                         /* Ignore lack of soft output capability, just skip
 251                          * checking if soft output is valid.
 252                          */
 253                         if ((test_vector.turbo_dec.op_flags &
 254                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 255                                         !(cap->capability_flags &
 256                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 257                                 printf(
 258                                         "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 259                                         dev_info->dev_name);
 260                                 clear_soft_out_cap(
 261                                         &test_vector.turbo_dec.op_flags);
 262                         }
 263
 264                         if (!flags_match(test_vector.turbo_dec.op_flags,
 265                                         cap->capability_flags))
 266                                 return TEST_FAILED;
 267                         if (nb_inputs > cap->num_buffers_src) {
 268                                 printf("Too many inputs defined: %u, max: %u\n",
 269                                         nb_inputs, cap->num_buffers_src);
 270                                 return TEST_FAILED;
 271                         }
 272                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 273                                         (test_vector.turbo_dec.op_flags &
 274                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 275                                 printf(
 276                                         "Too many soft outputs defined: %u, max: %u\n",
 277                                                 nb_soft_outputs,
 278                                                 cap->num_buffers_soft_out);
 279                                 return TEST_FAILED;
 280                         }
 281                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 282                                 printf(
 283                                         "Too many hard outputs defined: %u, max: %u\n",
 284                                                 nb_hard_outputs,
 285                                                 cap->num_buffers_hard_out);
 286                                 return TEST_FAILED;
 287                         }
 288                         if (intr_enabled && !(cap->capability_flags &
 289                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 290                                 printf(
 291                                         "Dequeue interrupts are not supported!\n");
 292                                 return TEST_FAILED;
 293                         }
 294
 295                         return TEST_SUCCESS;
 296                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 297                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 298                                         &op_cap->cap.turbo_enc;
 299
 300                         if (!flags_match(test_vector.turbo_enc.op_flags,
 301                                         cap->capability_flags))
 302                                 return TEST_FAILED;
 303                         if (nb_inputs > cap->num_buffers_src) {
 304                                 printf("Too many inputs defined: %u, max: %u\n",
 305                                         nb_inputs, cap->num_buffers_src);
 306                                 return TEST_FAILED;
 307                         }
 308                         if (nb_hard_outputs > cap->num_buffers_dst) {
 309                                 printf(
 310                                         "Too many hard outputs defined: %u, max: %u\n",
 311                                         nb_hard_outputs, cap->num_buffers_dst);
 312                                 return TEST_FAILED;
 313                         }
 314                         if (intr_enabled && !(cap->capability_flags &
 315                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 316                                 printf(
 317                                         "Dequeue interrupts are not supported!\n");
 318                                 return TEST_FAILED;
 319                         }
 320
 321                         return TEST_SUCCESS;
 322                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
 323                         const struct rte_bbdev_op_cap_ldpc_enc *cap =
 324                                         &op_cap->cap.ldpc_enc;
 325
 326                         if (!flags_match(test_vector.ldpc_enc.op_flags,
 327                                         cap->capability_flags)){
 328                                 printf("Flag Mismatch\n");
 329                                 return TEST_FAILED;
 330                         }
 331                         if (nb_inputs > cap->num_buffers_src) {
 332                                 printf("Too many inputs defined: %u, max: %u\n",
 333                                         nb_inputs, cap->num_buffers_src);
 334                                 return TEST_FAILED;
 335                         }
 336                         if (nb_hard_outputs > cap->num_buffers_dst) {
 337                                 printf(
 338                                         "Too many hard outputs defined: %u, max: %u\n",
 339                                         nb_hard_outputs, cap->num_buffers_dst);
 340                                 return TEST_FAILED;
 341                         }
 342                         if (intr_enabled && !(cap->capability_flags &
 343                                         RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
 344                                 printf(
 345                                         "Dequeue interrupts are not supported!\n");
 346                                 return TEST_FAILED;
 347                         }
 348
 349                         return TEST_SUCCESS;
 350                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
 351                         const struct rte_bbdev_op_cap_ldpc_dec *cap =
 352                                         &op_cap->cap.ldpc_dec;
 353
 354                         if (!flags_match(test_vector.ldpc_dec.op_flags,
 355                                         cap->capability_flags)){
 356                                 printf("Flag Mismatch\n");
 357                                 return TEST_FAILED;
 358                         }
 359                         if (nb_inputs > cap->num_buffers_src) {
 360                                 printf("Too many inputs defined: %u, max: %u\n",
 361                                         nb_inputs, cap->num_buffers_src);
 362                                 return TEST_FAILED;
 363                         }
 364                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 365                                 printf(
 366                                         "Too many hard outputs defined: %u, max: %u\n",
 367                                         nb_hard_outputs,
 368                                         cap->num_buffers_hard_out);
 369                                 return TEST_FAILED;
 370                         }
 371                         if (nb_harq_inputs > cap->num_buffers_hard_out) {
 372                                 printf(
 373                                         "Too many HARQ inputs defined: %u, max: %u\n",
 374                                         nb_hard_outputs,
 375                                         cap->num_buffers_hard_out);
 376                                 return TEST_FAILED;
 377                         }
 378                         if (nb_harq_outputs > cap->num_buffers_hard_out) {
 379                                 printf(
 380                                         "Too many HARQ outputs defined: %u, max: %u\n",
 381                                         nb_hard_outputs,
 382                                         cap->num_buffers_hard_out);
 383                                 return TEST_FAILED;
 384                         }
 385                         if (intr_enabled && !(cap->capability_flags &
 386                                         RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
 387                                 printf(
 388                                         "Dequeue interrupts are not supported!\n");
 389                                 return TEST_FAILED;
 390                         }
 391                         if (intr_enabled && (test_vector.ldpc_dec.op_flags &
 392                                 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
 393                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
 394                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
 395                                         ))) {
 396                                 printf("Skip loop-back with interrupt\n");
 397                                 return TEST_FAILED;
 398                         }
 399                         return TEST_SUCCESS;
 400                 }
 401         }
 402
 403         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 404                 return TEST_SUCCESS; /* Special case for NULL device */
 405
 406         return TEST_FAILED;
 407 }
 408
 409 /* calculates optimal mempool size not smaller than the val */
 410 static unsigned int
 411 optimal_mempool_size(unsigned int val)
 412 {
 413         return rte_align32pow2(val + 1) - 1;
 414 }
 415
 416 /* allocates mbuf mempool for inputs and outputs */
 417 static struct rte_mempool *
 418 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 419                 int socket_id, unsigned int mbuf_pool_size,
 420                 const char *op_type_str)
 421 {
 422         unsigned int i;
 423         uint32_t max_seg_sz = 0;
 424         char pool_name[RTE_MEMPOOL_NAMESIZE];
 425
 426         /* find max input segment size */
 427         for (i = 0; i < entries->nb_segments; ++i)
 428                 if (entries->segments[i].length > max_seg_sz)
 429                         max_seg_sz = entries->segments[i].length;
 430
 431         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 432                         dev_id);
 433         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 434                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
 435                                         + FILLER_HEADROOM,
 436                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 437 }
 438
 439 static int
 440 create_mempools(struct active_device *ad, int socket_id,
 441                 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
 442 {
 443         struct rte_mempool *mp;
 444         unsigned int ops_pool_size, mbuf_pool_size = 0;
 445         char pool_name[RTE_MEMPOOL_NAMESIZE];
 446         const char *op_type_str;
 447         enum rte_bbdev_op_type op_type = org_op_type;
 448
 449         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 450         struct op_data_entries *hard_out =
 451                         &test_vector.entries[DATA_HARD_OUTPUT];
 452         struct op_data_entries *soft_out =
 453                         &test_vector.entries[DATA_SOFT_OUTPUT];
 454         struct op_data_entries *harq_in =
 455                         &test_vector.entries[DATA_HARQ_INPUT];
 456         struct op_data_entries *harq_out =
 457                         &test_vector.entries[DATA_HARQ_OUTPUT];
 458
 459         /* allocate ops mempool */
 460         ops_pool_size = optimal_mempool_size(RTE_MAX(
 461                         /* Ops used plus 1 reference op */
 462                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 463                         /* Minimal cache size plus 1 reference op */
 464                         (unsigned int)(1.5 * rte_lcore_count() *
 465                                         OPS_CACHE_SIZE + 1)),
 466                         OPS_POOL_SIZE_MIN));
 467
 468         if (org_op_type == RTE_BBDEV_OP_NONE)
 469                 op_type = RTE_BBDEV_OP_TURBO_ENC;
 470
 471         op_type_str = rte_bbdev_op_type_str(op_type);
 472         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 473
 474         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 475                         ad->dev_id);
 476         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 477                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 478         TEST_ASSERT_NOT_NULL(mp,
 479                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 480                         ops_pool_size,
 481                         ad->dev_id,
 482                         socket_id);
 483         ad->ops_mempool = mp;
 484
 485         /* Do not create inputs and outputs mbufs for BaseBand Null Device */
 486         if (org_op_type == RTE_BBDEV_OP_NONE)
 487                 return TEST_SUCCESS;
 488
 489         /* Inputs */
 490         if (in->nb_segments > 0) {
 491                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 492                                 in->nb_segments);
 493                 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
 494                                 mbuf_pool_size, "in");
 495                 TEST_ASSERT_NOT_NULL(mp,
 496                                 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 497                                 mbuf_pool_size,
 498                                 ad->dev_id,
 499                                 socket_id);
 500                 ad->in_mbuf_pool = mp;
 501         }
 502
 503         /* Hard outputs */
 504         if (hard_out->nb_segments > 0) {
 505                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 506                                 hard_out->nb_segments);
 507                 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
 508                                 mbuf_pool_size,
 509                                 "hard_out");
 510                 TEST_ASSERT_NOT_NULL(mp,
 511                                 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 512                                 mbuf_pool_size,
 513                                 ad->dev_id,
 514                                 socket_id);
 515                 ad->hard_out_mbuf_pool = mp;
 516         }
 517
 518         /* Soft outputs */
 519         if (soft_out->nb_segments > 0) {
 520                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 521                                 soft_out->nb_segments);
 522                 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
 523                                 mbuf_pool_size,
 524                                 "soft_out");
 525                 TEST_ASSERT_NOT_NULL(mp,
 526                                 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 527                                 mbuf_pool_size,
 528                                 ad->dev_id,
 529                                 socket_id);
 530                 ad->soft_out_mbuf_pool = mp;
 531         }
 532
 533         /* HARQ inputs */
 534         if (harq_in->nb_segments > 0) {
 535                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 536                                 harq_in->nb_segments);
 537                 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
 538                                 mbuf_pool_size,
 539                                 "harq_in");
 540                 TEST_ASSERT_NOT_NULL(mp,
 541                                 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
 542                                 mbuf_pool_size,
 543                                 ad->dev_id,
 544                                 socket_id);
 545                 ad->harq_in_mbuf_pool = mp;
 546         }
 547
 548         /* HARQ outputs */
 549         if (harq_out->nb_segments > 0) {
 550                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 551                                 harq_out->nb_segments);
 552                 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
 553                                 mbuf_pool_size,
 554                                 "harq_out");
 555                 TEST_ASSERT_NOT_NULL(mp,
 556                                 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
 557                                 mbuf_pool_size,
 558                                 ad->dev_id,
 559                                 socket_id);
 560                 ad->harq_out_mbuf_pool = mp;
 561         }
 562
 563         return TEST_SUCCESS;
 564 }
 565
 566 static int
 567 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 568                 struct test_bbdev_vector *vector)
 569 {
 570         int ret;
 571         unsigned int queue_id;
 572         struct rte_bbdev_queue_conf qconf;
 573         struct active_device *ad = &active_devs[nb_active_devs];
 574         unsigned int nb_queues;
 575         enum rte_bbdev_op_type op_type = vector->op_type;
 576
 577 /* Configure fpga lte fec with PF & VF values
 578  * if '-i' flag is set and using fpga device
 579  */
 580 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
 581         if ((get_init_device() == true) &&
 582                 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
 583                 struct rte_fpga_lte_fec_conf conf;
 584                 unsigned int i;
 585
 586                 printf("Configure FPGA LTE FEC Driver %s with default values\n",
 587                                 info->drv.driver_name);
 588
 589                 /* clear default configuration before initialization */
 590                 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf));
 591
 592                 /* Set PF mode :
 593                  * true if PF is used for data plane
 594                  * false for VFs
 595                  */
 596                 conf.pf_mode_en = true;
 597
 598                 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
 599                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 600                         conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
 601                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 602                         conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
 603                 }
 604
 605                 /* UL bandwidth. Needed for schedule algorithm */
 606                 conf.ul_bandwidth = UL_4G_BANDWIDTH;
 607                 /* DL bandwidth */
 608                 conf.dl_bandwidth = DL_4G_BANDWIDTH;
 609
 610                 /* UL & DL load Balance Factor to 64 */
 611                 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
 612                 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
 613
 614                 /**< FLR timeout value */
 615                 conf.flr_time_out = FLR_4G_TIMEOUT;
 616
 617                 /* setup FPGA PF with configuration information */
 618                 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf);
 619                 TEST_ASSERT_SUCCESS(ret,
 620                                 "Failed to configure 4G FPGA PF for bbdev %s",
 621                                 info->dev_name);
 622         }
 623 #endif
 624 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
 625         if ((get_init_device() == true) &&
 626                 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
 627                 struct rte_fpga_5gnr_fec_conf conf;
 628                 unsigned int i;
 629
 630                 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
 631                                 info->drv.driver_name);
 632
 633                 /* clear default configuration before initialization */
 634                 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
 635
 636                 /* Set PF mode :
 637                  * true if PF is used for data plane
 638                  * false for VFs
 639                  */
 640                 conf.pf_mode_en = true;
 641
 642                 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
 643                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 644                         conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
 645                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 646                         conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
 647                 }
 648
 649                 /* UL bandwidth. Needed for schedule algorithm */
 650                 conf.ul_bandwidth = UL_5G_BANDWIDTH;
 651                 /* DL bandwidth */
 652                 conf.dl_bandwidth = DL_5G_BANDWIDTH;
 653
 654                 /* UL & DL load Balance Factor to 64 */
 655                 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
 656                 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
 657
 658                 /**< FLR timeout value */
 659                 conf.flr_time_out = FLR_5G_TIMEOUT;
 660
 661                 /* setup FPGA PF with configuration information */
 662                 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf);
 663                 TEST_ASSERT_SUCCESS(ret,
 664                                 "Failed to configure 5G FPGA PF for bbdev %s",
 665                                 info->dev_name);
 666         }
 667 #endif
 668 #ifdef RTE_BASEBAND_ACC100
 669         if ((get_init_device() == true) &&
 670                 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) {
 671                 struct rte_acc100_conf conf;
 672                 unsigned int i;
 673
 674                 printf("Configure ACC100 FEC Driver %s with default values\n",
 675                                 info->drv.driver_name);
 676
 677                 /* clear default configuration before initialization */
 678                 memset(&conf, 0, sizeof(struct rte_acc100_conf));
 679
 680                 /* Always set in PF mode for built-in configuration */
 681                 conf.pf_mode_en = true;
 682                 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) {
 683                         conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 684                         conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 685                         conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR;
 686                         conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 687                         conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 688                         conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR;
 689                         conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 690                         conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 691                         conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR;
 692                         conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 693                         conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 694                         conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR;
 695                 }
 696
 697                 conf.input_pos_llr_1_bit = true;
 698                 conf.output_pos_llr_1_bit = true;
 699                 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */
 700
 701                 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
 702                 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 703                 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 704                 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 705                 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
 706                 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 707                 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 708                 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 709                 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
 710                 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 711                 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 712                 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 713                 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
 714                 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 715                 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 716                 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 717
 718                 /* setup PF with configuration information */
 719                 ret = rte_acc100_configure(info->dev_name, &conf);
 720                 TEST_ASSERT_SUCCESS(ret,
 721                                 "Failed to configure ACC100 PF for bbdev %s",
 722                                 info->dev_name);
 723         }
 724 #endif
 725         /* Let's refresh this now this is configured */
 726         rte_bbdev_info_get(dev_id, info);
 727         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 728         nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
 729
 730         /* setup device */
 731         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 732         if (ret < 0) {
 733                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 734                                 dev_id, nb_queues, info->socket_id, ret);
 735                 return TEST_FAILED;
 736         }
 737
 738         /* configure interrupts if needed */
 739         if (intr_enabled) {
 740                 ret = rte_bbdev_intr_enable(dev_id);
 741                 if (ret < 0) {
 742                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 743                                         ret);
 744                         return TEST_FAILED;
 745                 }
 746         }
 747
 748         /* setup device queues */
 749         qconf.socket = info->socket_id;
 750         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 751         qconf.priority = 0;
 752         qconf.deferred_start = 0;
 753         qconf.op_type = op_type;
 754
 755         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 756                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 757                 if (ret != 0) {
 758                         printf(
 759                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 760                                         queue_id, qconf.priority, dev_id);
 761                         qconf.priority++;
 762                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 763                                         &qconf);
 764                 }
 765                 if (ret != 0) {
 766                         printf("All queues on dev %u allocated: %u\n",
 767                                         dev_id, queue_id);
 768                         break;
 769                 }
 770                 ad->queue_ids[queue_id] = queue_id;
 771         }
 772         TEST_ASSERT(queue_id != 0,
 773                         "ERROR Failed to configure any queues on dev %u",
 774                         dev_id);
 775         ad->nb_queues = queue_id;
 776
 777         set_avail_op(ad, op_type);
 778
 779         return TEST_SUCCESS;
 780 }
 781
 782 static int
 783 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 784                 struct test_bbdev_vector *vector)
 785 {
 786         int ret;
 787
 788         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 789         active_devs[nb_active_devs].dev_id = dev_id;
 790
 791         ret = add_bbdev_dev(dev_id, info, vector);
 792         if (ret == TEST_SUCCESS)
 793                 ++nb_active_devs;
 794         return ret;
 795 }
 796
 797 static uint8_t
 798 populate_active_devices(void)
 799 {
 800         int ret;
 801         uint8_t dev_id;
 802         uint8_t nb_devs_added = 0;
 803         struct rte_bbdev_info info;
 804
 805         RTE_BBDEV_FOREACH(dev_id) {
 806                 rte_bbdev_info_get(dev_id, &info);
 807
 808                 if (check_dev_cap(&info)) {
 809                         printf(
 810                                 "Device %d (%s) does not support specified capabilities\n",
 811                                         dev_id, info.dev_name);
 812                         continue;
 813                 }
 814
 815                 ret = add_active_device(dev_id, &info, &test_vector);
 816                 if (ret != 0) {
 817                         printf("Adding active bbdev %s skipped\n",
 818                                         info.dev_name);
 819                         continue;
 820                 }
 821                 nb_devs_added++;
 822         }
 823
 824         return nb_devs_added;
 825 }
 826
 827 static int
 828 read_test_vector(void)
 829 {
 830         int ret;
 831
 832         memset(&test_vector, 0, sizeof(test_vector));
 833         printf("Test vector file = %s\n", get_vector_filename());
 834         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 835         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 836                         get_vector_filename());
 837
 838         return TEST_SUCCESS;
 839 }
 840
 841 static int
 842 testsuite_setup(void)
 843 {
 844         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 845
 846         if (populate_active_devices() == 0) {
 847                 printf("No suitable devices found!\n");
 848                 return TEST_SKIPPED;
 849         }
 850
 851         return TEST_SUCCESS;
 852 }
 853
 854 static int
 855 interrupt_testsuite_setup(void)
 856 {
 857         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 858
 859         /* Enable interrupts */
 860         intr_enabled = true;
 861
 862         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 863         if (populate_active_devices() == 0 ||
 864                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 865                 intr_enabled = false;
 866                 printf("No suitable devices found!\n");
 867                 return TEST_SKIPPED;
 868         }
 869
 870         return TEST_SUCCESS;
 871 }
 872
 873 static void
 874 testsuite_teardown(void)
 875 {
 876         uint8_t dev_id;
 877
 878         /* Unconfigure devices */
 879         RTE_BBDEV_FOREACH(dev_id)
 880                 rte_bbdev_close(dev_id);
 881
 882         /* Clear active devices structs. */
 883         memset(active_devs, 0, sizeof(active_devs));
 884         nb_active_devs = 0;
 885
 886         /* Disable interrupts */
 887         intr_enabled = false;
 888 }
 889
 890 static int
 891 ut_setup(void)
 892 {
 893         uint8_t i, dev_id;
 894
 895         for (i = 0; i < nb_active_devs; i++) {
 896                 dev_id = active_devs[i].dev_id;
 897                 /* reset bbdev stats */
 898                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 899                                 "Failed to reset stats of bbdev %u", dev_id);
 900                 /* start the device */
 901                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 902                                 "Failed to start bbdev %u", dev_id);
 903         }
 904
 905         return TEST_SUCCESS;
 906 }
 907
 908 static void
 909 ut_teardown(void)
 910 {
 911         uint8_t i, dev_id;
 912         struct rte_bbdev_stats stats;
 913
 914         for (i = 0; i < nb_active_devs; i++) {
 915                 dev_id = active_devs[i].dev_id;
 916                 /* read stats and print */
 917                 rte_bbdev_stats_get(dev_id, &stats);
 918                 /* Stop the device */
 919                 rte_bbdev_stop(dev_id);
 920         }
 921 }
 922
 923 static int
 924 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 925                 struct op_data_entries *ref_entries,
 926                 struct rte_mempool *mbuf_pool, const uint16_t n,
 927                 enum op_data_type op_type, uint16_t min_alignment)
 928 {
 929         int ret;
 930         unsigned int i, j;
 931         bool large_input = false;
 932
 933         for (i = 0; i < n; ++i) {
 934                 char *data;
 935                 struct op_data_buf *seg = &ref_entries->segments[0];
 936                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 937                 TEST_ASSERT_NOT_NULL(m_head,
 938                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 939                                 op_type, n * ref_entries->nb_segments,
 940                                 mbuf_pool->size);
 941
 942                 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
 943                         /*
 944                          * Special case when DPDK mbuf cannot handle
 945                          * the required input size
 946                          */
 947                         printf("Warning: Larger input size than DPDK mbuf %d\n",
 948                                         seg->length);
 949                         large_input = true;
 950                 }
 951                 bufs[i].data = m_head;
 952                 bufs[i].offset = 0;
 953                 bufs[i].length = 0;
 954
 955                 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
 956                         if ((op_type == DATA_INPUT) && large_input) {
 957                                 /* Allocate a fake overused mbuf */
 958                                 data = rte_malloc(NULL, seg->length, 0);
 959                                 memcpy(data, seg->addr, seg->length);
 960                                 m_head->buf_addr = data;
 961                                 m_head->buf_iova = rte_malloc_virt2iova(data);
 962                                 m_head->data_off = 0;
 963                                 m_head->data_len = seg->length;
 964                         } else {
 965                                 data = rte_pktmbuf_append(m_head, seg->length);
 966                                 TEST_ASSERT_NOT_NULL(data,
 967                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 968                                         seg->length, op_type);
 969
 970                                 TEST_ASSERT(data == RTE_PTR_ALIGN(
 971                                                 data, min_alignment),
 972                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 973                                         data, min_alignment);
 974                                 rte_memcpy(data, seg->addr, seg->length);
 975                         }
 976
 977                         bufs[i].length += seg->length;
 978
 979                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 980                                 struct rte_mbuf *m_tail =
 981                                                 rte_pktmbuf_alloc(mbuf_pool);
 982                                 TEST_ASSERT_NOT_NULL(m_tail,
 983                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 984                                                 op_type,
 985                                                 n * ref_entries->nb_segments,
 986                                                 mbuf_pool->size);
 987                                 seg += 1;
 988
 989                                 data = rte_pktmbuf_append(m_tail, seg->length);
 990                                 TEST_ASSERT_NOT_NULL(data,
 991                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 992                                                 seg->length, op_type);
 993
 994                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 995                                                 min_alignment),
 996                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 997                                                 data, min_alignment);
 998                                 rte_memcpy(data, seg->addr, seg->length);
 999                                 bufs[i].length += seg->length;
1000
1001                                 ret = rte_pktmbuf_chain(m_head, m_tail);
1002                                 TEST_ASSERT_SUCCESS(ret,
1003                                                 "Couldn't chain mbufs from %d data type mbuf pool",
1004                                                 op_type);
1005                         }
1006                 } else {
1007
1008                         /* allocate chained-mbuf for output buffer */
1009                         for (j = 1; j < ref_entries->nb_segments; ++j) {
1010                                 struct rte_mbuf *m_tail =
1011                                                 rte_pktmbuf_alloc(mbuf_pool);
1012                                 TEST_ASSERT_NOT_NULL(m_tail,
1013                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1014                                                 op_type,
1015                                                 n * ref_entries->nb_segments,
1016                                                 mbuf_pool->size);
1017
1018                                 ret = rte_pktmbuf_chain(m_head, m_tail);
1019                                 TEST_ASSERT_SUCCESS(ret,
1020                                                 "Couldn't chain mbufs from %d data type mbuf pool",
1021                                                 op_type);
1022                         }
1023                 }
1024         }
1025
1026         return 0;
1027 }
1028
1029 static int
1030 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
1031                 const int socket)
1032 {
1033         int i;
1034
1035         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
1036         if (*buffers == NULL) {
1037                 printf("WARNING: Failed to allocate op_data on socket %d\n",
1038                                 socket);
1039                 /* try to allocate memory on other detected sockets */
1040                 for (i = 0; i < socket; i++) {
1041                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
1042                         if (*buffers != NULL)
1043                                 break;
1044                 }
1045         }
1046
1047         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
1048 }
1049
1050 static void
1051 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
1052                 const uint16_t n, const int8_t max_llr_modulus)
1053 {
1054         uint16_t i, byte_idx;
1055
1056         for (i = 0; i < n; ++i) {
1057                 struct rte_mbuf *m = input_ops[i].data;
1058                 while (m != NULL) {
1059                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1060                                         input_ops[i].offset);
1061                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1062                                         ++byte_idx)
1063                                 llr[byte_idx] = round((double)max_llr_modulus *
1064                                                 llr[byte_idx] / INT8_MAX);
1065
1066                         m = m->next;
1067                 }
1068         }
1069 }
1070
1071 /*
1072  * We may have to insert filler bits
1073  * when they are required by the HARQ assumption
1074  */
1075 static void
1076 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1077                 const uint16_t n, struct test_op_params *op_params)
1078 {
1079         struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1080
1081         if (input_ops == NULL)
1082                 return;
1083         /* No need to add filler if not required by device */
1084         if (!(ldpc_cap_flags &
1085                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1086                 return;
1087         /* No need to add filler for loopback operation */
1088         if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1089                 return;
1090
1091         uint16_t i, j, parity_offset;
1092         for (i = 0; i < n; ++i) {
1093                 struct rte_mbuf *m = input_ops[i].data;
1094                 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1095                                 input_ops[i].offset);
1096                 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1097                                 * dec.z_c - dec.n_filler;
1098                 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1099                 m->data_len = new_hin_size;
1100                 input_ops[i].length = new_hin_size;
1101                 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1102                                 j--)
1103                         llr[j] = llr[j - dec.n_filler];
1104                 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1105                 for (j = 0; j < dec.n_filler; j++)
1106                         llr[parity_offset + j] = llr_max_pre_scaling;
1107         }
1108 }
1109
1110 static void
1111 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1112                 const uint16_t n, const int8_t llr_size,
1113                 const int8_t llr_decimals)
1114 {
1115         if (input_ops == NULL)
1116                 return;
1117
1118         uint16_t i, byte_idx;
1119
1120         int16_t llr_max, llr_min, llr_tmp;
1121         llr_max = (1 << (llr_size - 1)) - 1;
1122         llr_min = -llr_max;
1123         for (i = 0; i < n; ++i) {
1124                 struct rte_mbuf *m = input_ops[i].data;
1125                 while (m != NULL) {
1126                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1127                                         input_ops[i].offset);
1128                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1129                                         ++byte_idx) {
1130
1131                                 llr_tmp = llr[byte_idx];
1132                                 if (llr_decimals == 4)
1133                                         llr_tmp *= 8;
1134                                 else if (llr_decimals == 2)
1135                                         llr_tmp *= 2;
1136                                 else if (llr_decimals == 0)
1137                                         llr_tmp /= 2;
1138                                 llr_tmp = RTE_MIN(llr_max,
1139                                                 RTE_MAX(llr_min, llr_tmp));
1140                                 llr[byte_idx] = (int8_t) llr_tmp;
1141                         }
1142
1143                         m = m->next;
1144                 }
1145         }
1146 }
1147
1148
1149
1150 static int
1151 fill_queue_buffers(struct test_op_params *op_params,
1152                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1153                 struct rte_mempool *soft_out_mp,
1154                 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1155                 uint16_t queue_id,
1156                 const struct rte_bbdev_op_cap *capabilities,
1157                 uint16_t min_alignment, const int socket_id)
1158 {
1159         int ret;
1160         enum op_data_type type;
1161         const uint16_t n = op_params->num_to_process;
1162
1163         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1164                 in_mp,
1165                 soft_out_mp,
1166                 hard_out_mp,
1167                 harq_in_mp,
1168                 harq_out_mp,
1169         };
1170
1171         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1172                 &op_params->q_bufs[socket_id][queue_id].inputs,
1173                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1174                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1175                 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1176                 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1177         };
1178
1179         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1180                 struct op_data_entries *ref_entries =
1181                                 &test_vector.entries[type];
1182                 if (ref_entries->nb_segments == 0)
1183                         continue;
1184
1185                 ret = allocate_buffers_on_socket(queue_ops[type],
1186                                 n * sizeof(struct rte_bbdev_op_data),
1187                                 socket_id);
1188                 TEST_ASSERT_SUCCESS(ret,
1189                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
1190
1191                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1192                                 mbuf_pools[type], n, type, min_alignment);
1193                 TEST_ASSERT_SUCCESS(ret,
1194                                 "Couldn't init rte_bbdev_op_data structs");
1195         }
1196
1197         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1198                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1199                         capabilities->cap.turbo_dec.max_llr_modulus);
1200
1201         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1202                 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1203                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1204                 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1205                                 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1206                 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1207                                 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1208                 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1209                 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1210                 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1211                 if (!loopback && !llr_comp)
1212                         ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1213                                         ldpc_llr_size, ldpc_llr_decimals);
1214                 if (!loopback && !harq_comp)
1215                         ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1216                                         ldpc_llr_size, ldpc_llr_decimals);
1217                 if (!loopback)
1218                         ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1219                                         op_params);
1220         }
1221
1222         return 0;
1223 }
1224
1225 static void
1226 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1227 {
1228         unsigned int i, j;
1229
1230         rte_mempool_free(ad->ops_mempool);
1231         rte_mempool_free(ad->in_mbuf_pool);
1232         rte_mempool_free(ad->hard_out_mbuf_pool);
1233         rte_mempool_free(ad->soft_out_mbuf_pool);
1234         rte_mempool_free(ad->harq_in_mbuf_pool);
1235         rte_mempool_free(ad->harq_out_mbuf_pool);
1236
1237         for (i = 0; i < rte_lcore_count(); ++i) {
1238                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1239                         rte_free(op_params->q_bufs[j][i].inputs);
1240                         rte_free(op_params->q_bufs[j][i].hard_outputs);
1241                         rte_free(op_params->q_bufs[j][i].soft_outputs);
1242                         rte_free(op_params->q_bufs[j][i].harq_inputs);
1243                         rte_free(op_params->q_bufs[j][i].harq_outputs);
1244                 }
1245         }
1246 }
1247
1248 static void
1249 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1250                 unsigned int start_idx,
1251                 struct rte_bbdev_op_data *inputs,
1252                 struct rte_bbdev_op_data *hard_outputs,
1253                 struct rte_bbdev_op_data *soft_outputs,
1254                 struct rte_bbdev_dec_op *ref_op)
1255 {
1256         unsigned int i;
1257         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1258
1259         for (i = 0; i < n; ++i) {
1260                 if (turbo_dec->code_block_mode == 0) {
1261                         ops[i]->turbo_dec.tb_params.ea =
1262                                         turbo_dec->tb_params.ea;
1263                         ops[i]->turbo_dec.tb_params.eb =
1264                                         turbo_dec->tb_params.eb;
1265                         ops[i]->turbo_dec.tb_params.k_pos =
1266                                         turbo_dec->tb_params.k_pos;
1267                         ops[i]->turbo_dec.tb_params.k_neg =
1268                                         turbo_dec->tb_params.k_neg;
1269                         ops[i]->turbo_dec.tb_params.c =
1270                                         turbo_dec->tb_params.c;
1271                         ops[i]->turbo_dec.tb_params.c_neg =
1272                                         turbo_dec->tb_params.c_neg;
1273                         ops[i]->turbo_dec.tb_params.cab =
1274                                         turbo_dec->tb_params.cab;
1275                         ops[i]->turbo_dec.tb_params.r =
1276                                         turbo_dec->tb_params.r;
1277                 } else {
1278                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1279                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1280                 }
1281
1282                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1283                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1284                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1285                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1286                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1287                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1288                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1289
1290                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1291                 ops[i]->turbo_dec.input = inputs[start_idx + i];
1292                 if (soft_outputs != NULL)
1293                         ops[i]->turbo_dec.soft_output =
1294                                 soft_outputs[start_idx + i];
1295         }
1296 }
1297
1298 static void
1299 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1300                 unsigned int start_idx,
1301                 struct rte_bbdev_op_data *inputs,
1302                 struct rte_bbdev_op_data *outputs,
1303                 struct rte_bbdev_enc_op *ref_op)
1304 {
1305         unsigned int i;
1306         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1307         for (i = 0; i < n; ++i) {
1308                 if (turbo_enc->code_block_mode == 0) {
1309                         ops[i]->turbo_enc.tb_params.ea =
1310                                         turbo_enc->tb_params.ea;
1311                         ops[i]->turbo_enc.tb_params.eb =
1312                                         turbo_enc->tb_params.eb;
1313                         ops[i]->turbo_enc.tb_params.k_pos =
1314                                         turbo_enc->tb_params.k_pos;
1315                         ops[i]->turbo_enc.tb_params.k_neg =
1316                                         turbo_enc->tb_params.k_neg;
1317                         ops[i]->turbo_enc.tb_params.c =
1318                                         turbo_enc->tb_params.c;
1319                         ops[i]->turbo_enc.tb_params.c_neg =
1320                                         turbo_enc->tb_params.c_neg;
1321                         ops[i]->turbo_enc.tb_params.cab =
1322                                         turbo_enc->tb_params.cab;
1323                         ops[i]->turbo_enc.tb_params.ncb_pos =
1324                                         turbo_enc->tb_params.ncb_pos;
1325                         ops[i]->turbo_enc.tb_params.ncb_neg =
1326                                         turbo_enc->tb_params.ncb_neg;
1327                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1328                 } else {
1329                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1330                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1331                         ops[i]->turbo_enc.cb_params.ncb =
1332                                         turbo_enc->cb_params.ncb;
1333                 }
1334                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1335                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1336                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1337
1338                 ops[i]->turbo_enc.output = outputs[start_idx + i];
1339                 ops[i]->turbo_enc.input = inputs[start_idx + i];
1340         }
1341 }
1342
1343
1344 /* Returns a random number drawn from a normal distribution
1345  * with mean of 0 and variance of 1
1346  * Marsaglia algorithm
1347  */
1348 static double
1349 randn(int n)
1350 {
1351         double S, Z, U1, U2, u, v, fac;
1352
1353         do {
1354                 U1 = (double)rand() / RAND_MAX;
1355                 U2 = (double)rand() / RAND_MAX;
1356                 u = 2. * U1 - 1.;
1357                 v = 2. * U2 - 1.;
1358                 S = u * u + v * v;
1359         } while (S >= 1 || S == 0);
1360         fac = sqrt(-2. * log(S) / S);
1361         Z = (n % 2) ? u * fac : v * fac;
1362         return Z;
1363 }
1364
1365 static inline double
1366 maxstar(double A, double B)
1367 {
1368         if (fabs(A - B) > 5)
1369                 return RTE_MAX(A, B);
1370         else
1371                 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1372 }
1373
1374 /*
1375  * Generate Qm LLRS for Qm==8
1376  * Modulation, AWGN and LLR estimation from max log development
1377  */
1378 static void
1379 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1380 {
1381         int qm = 8;
1382         int qam = 256;
1383         int m, k;
1384         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1385         /* 5.1.4 of TS38.211 */
1386         const double symbols_I[256] = {
1387                         5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1388                         5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1389                         11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1390                         15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1391                         15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1392                         1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1393                         1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1394                         15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1395                         13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1396                         -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1397                         -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1398                         -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1399                         -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1400                         -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1401                         -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1402                         -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1403                         -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1404                         -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1405                         -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1406                         -13, -13, -15, -15, -13, -13, -15, -15};
1407         const double symbols_Q[256] = {
1408                         5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1409                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1410                         15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1411                         11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1412                         15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1413                         -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1414                         -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1415                         -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1416                         -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1417                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1418                         7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1419                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1420                         13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1421                         3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1422                         13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1423                         -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1424                         -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1425                         -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1426                         -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1427                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1428         /* Average constellation point energy */
1429         N0 *= 170.0;
1430         for (k = 0; k < qm; k++)
1431                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1432         /* 5.1.4 of TS38.211 */
1433         I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1434                         (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1435         Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1436                         (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1437         /* AWGN channel */
1438         I += sqrt(N0 / 2) * randn(0);
1439         Q += sqrt(N0 / 2) * randn(1);
1440         /*
1441          * Calculate the log of the probability that each of
1442          * the constellation points was transmitted
1443          */
1444         for (m = 0; m < qam; m++)
1445                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1446                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1447         /* Calculate an LLR for each of the k_64QAM bits in the set */
1448         for (k = 0; k < qm; k++) {
1449                 p0 = -999999;
1450                 p1 = -999999;
1451                 /* For each constellation point */
1452                 for (m = 0; m < qam; m++) {
1453                         if ((m >> (qm - k - 1)) & 1)
1454                                 p1 = maxstar(p1, log_syml_prob[m]);
1455                         else
1456                                 p0 = maxstar(p0, log_syml_prob[m]);
1457                 }
1458                 /* Calculate the LLR */
1459                 llr_ = p0 - p1;
1460                 llr_ *= (1 << ldpc_llr_decimals);
1461                 llr_ = round(llr_);
1462                 if (llr_ > llr_max)
1463                         llr_ = llr_max;
1464                 if (llr_ < -llr_max)
1465                         llr_ = -llr_max;
1466                 llrs[qm * i + k] = (int8_t) llr_;
1467         }
1468 }
1469
1470
1471 /*
1472  * Generate Qm LLRS for Qm==6
1473  * Modulation, AWGN and LLR estimation from max log development
1474  */
1475 static void
1476 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1477 {
1478         int qm = 6;
1479         int qam = 64;
1480         int m, k;
1481         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1482         /* 5.1.4 of TS38.211 */
1483         const double symbols_I[64] = {
1484                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1485                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1486                         -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1487                         -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1488                         -5, -5, -7, -7, -5, -5, -7, -7};
1489         const double symbols_Q[64] = {
1490                         3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1491                         -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1492                         -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1493                         5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1494                         -3, -1, -3, -1, -5, -7, -5, -7};
1495         /* Average constellation point energy */
1496         N0 *= 42.0;
1497         for (k = 0; k < qm; k++)
1498                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1499         /* 5.1.4 of TS38.211 */
1500         I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1501         Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1502         /* AWGN channel */
1503         I += sqrt(N0 / 2) * randn(0);
1504         Q += sqrt(N0 / 2) * randn(1);
1505         /*
1506          * Calculate the log of the probability that each of
1507          * the constellation points was transmitted
1508          */
1509         for (m = 0; m < qam; m++)
1510                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1511                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1512         /* Calculate an LLR for each of the k_64QAM bits in the set */
1513         for (k = 0; k < qm; k++) {
1514                 p0 = -999999;
1515                 p1 = -999999;
1516                 /* For each constellation point */
1517                 for (m = 0; m < qam; m++) {
1518                         if ((m >> (qm - k - 1)) & 1)
1519                                 p1 = maxstar(p1, log_syml_prob[m]);
1520                         else
1521                                 p0 = maxstar(p0, log_syml_prob[m]);
1522                 }
1523                 /* Calculate the LLR */
1524                 llr_ = p0 - p1;
1525                 llr_ *= (1 << ldpc_llr_decimals);
1526                 llr_ = round(llr_);
1527                 if (llr_ > llr_max)
1528                         llr_ = llr_max;
1529                 if (llr_ < -llr_max)
1530                         llr_ = -llr_max;
1531                 llrs[qm * i + k] = (int8_t) llr_;
1532         }
1533 }
1534
1535 /*
1536  * Generate Qm LLRS for Qm==4
1537  * Modulation, AWGN and LLR estimation from max log development
1538  */
1539 static void
1540 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1541 {
1542         int qm = 4;
1543         int qam = 16;
1544         int m, k;
1545         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1546         /* 5.1.4 of TS38.211 */
1547         const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1548                         -1, -1, -3, -3, -1, -1, -3, -3};
1549         const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1550                         1, 3, 1, 3, -1, -3, -1, -3};
1551         /* Average constellation point energy */
1552         N0 *= 10.0;
1553         for (k = 0; k < qm; k++)
1554                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1555         /* 5.1.4 of TS38.211 */
1556         I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1557         Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1558         /* AWGN channel */
1559         I += sqrt(N0 / 2) * randn(0);
1560         Q += sqrt(N0 / 2) * randn(1);
1561         /*
1562          * Calculate the log of the probability that each of
1563          * the constellation points was transmitted
1564          */
1565         for (m = 0; m < qam; m++)
1566                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1567                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1568         /* Calculate an LLR for each of the k_64QAM bits in the set */
1569         for (k = 0; k < qm; k++) {
1570                 p0 = -999999;
1571                 p1 = -999999;
1572                 /* For each constellation point */
1573                 for (m = 0; m < qam; m++) {
1574                         if ((m >> (qm - k - 1)) & 1)
1575                                 p1 = maxstar(p1, log_syml_prob[m]);
1576                         else
1577                                 p0 = maxstar(p0, log_syml_prob[m]);
1578                 }
1579                 /* Calculate the LLR */
1580                 llr_ = p0 - p1;
1581                 llr_ *= (1 << ldpc_llr_decimals);
1582                 llr_ = round(llr_);
1583                 if (llr_ > llr_max)
1584                         llr_ = llr_max;
1585                 if (llr_ < -llr_max)
1586                         llr_ = -llr_max;
1587                 llrs[qm * i + k] = (int8_t) llr_;
1588         }
1589 }
1590
1591 static void
1592 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1593 {
1594         double b, b1, n;
1595         double coeff = 2.0 * sqrt(N0);
1596
1597         /* Ignore in vectors rare quasi null LLRs not to be saturated */
1598         if (llrs[j] < 8 && llrs[j] > -8)
1599                 return;
1600
1601         /* Note don't change sign here */
1602         n = randn(j % 2);
1603         b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1604                         + coeff * n) / N0;
1605         b = b1 * (1 << ldpc_llr_decimals);
1606         b = round(b);
1607         if (b > llr_max)
1608                 b = llr_max;
1609         if (b < -llr_max)
1610                 b = -llr_max;
1611         llrs[j] = (int8_t) b;
1612 }
1613
1614 /* Generate LLR for a given SNR */
1615 static void
1616 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1617                 struct rte_bbdev_dec_op *ref_op)
1618 {
1619         struct rte_mbuf *m;
1620         uint16_t qm;
1621         uint32_t i, j, e, range;
1622         double N0, llr_max;
1623
1624         e = ref_op->ldpc_dec.cb_params.e;
1625         qm = ref_op->ldpc_dec.q_m;
1626         llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1627         range = e / qm;
1628         N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1629
1630         for (i = 0; i < n; ++i) {
1631                 m = inputs[i].data;
1632                 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1633                 if (qm == 8) {
1634                         for (j = 0; j < range; ++j)
1635                                 gen_qm8_llr(llrs, j, N0, llr_max);
1636                 } else if (qm == 6) {
1637                         for (j = 0; j < range; ++j)
1638                                 gen_qm6_llr(llrs, j, N0, llr_max);
1639                 } else if (qm == 4) {
1640                         for (j = 0; j < range; ++j)
1641                                 gen_qm4_llr(llrs, j, N0, llr_max);
1642                 } else {
1643                         for (j = 0; j < e; ++j)
1644                                 gen_qm2_llr(llrs, j, N0, llr_max);
1645                 }
1646         }
1647 }
1648
1649 static void
1650 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1651                 unsigned int start_idx,
1652                 struct rte_bbdev_op_data *inputs,
1653                 struct rte_bbdev_op_data *hard_outputs,
1654                 struct rte_bbdev_op_data *soft_outputs,
1655                 struct rte_bbdev_op_data *harq_inputs,
1656                 struct rte_bbdev_op_data *harq_outputs,
1657                 struct rte_bbdev_dec_op *ref_op)
1658 {
1659         unsigned int i;
1660         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1661
1662         for (i = 0; i < n; ++i) {
1663                 if (ldpc_dec->code_block_mode == 0) {
1664                         ops[i]->ldpc_dec.tb_params.ea =
1665                                         ldpc_dec->tb_params.ea;
1666                         ops[i]->ldpc_dec.tb_params.eb =
1667                                         ldpc_dec->tb_params.eb;
1668                         ops[i]->ldpc_dec.tb_params.c =
1669                                         ldpc_dec->tb_params.c;
1670                         ops[i]->ldpc_dec.tb_params.cab =
1671                                         ldpc_dec->tb_params.cab;
1672                         ops[i]->ldpc_dec.tb_params.r =
1673                                         ldpc_dec->tb_params.r;
1674                 } else {
1675                         ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1676                 }
1677
1678                 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1679                 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1680                 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1681                 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1682                 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1683                 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1684                 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1685                 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1686                 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1687
1688                 if (hard_outputs != NULL)
1689                         ops[i]->ldpc_dec.hard_output =
1690                                         hard_outputs[start_idx + i];
1691                 if (inputs != NULL)
1692                         ops[i]->ldpc_dec.input =
1693                                         inputs[start_idx + i];
1694                 if (soft_outputs != NULL)
1695                         ops[i]->ldpc_dec.soft_output =
1696                                         soft_outputs[start_idx + i];
1697                 if (harq_inputs != NULL)
1698                         ops[i]->ldpc_dec.harq_combined_input =
1699                                         harq_inputs[start_idx + i];
1700                 if (harq_outputs != NULL)
1701                         ops[i]->ldpc_dec.harq_combined_output =
1702                                         harq_outputs[start_idx + i];
1703         }
1704 }
1705
1706
1707 static void
1708 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1709                 unsigned int start_idx,
1710                 struct rte_bbdev_op_data *inputs,
1711                 struct rte_bbdev_op_data *outputs,
1712                 struct rte_bbdev_enc_op *ref_op)
1713 {
1714         unsigned int i;
1715         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1716         for (i = 0; i < n; ++i) {
1717                 if (ldpc_enc->code_block_mode == 0) {
1718                         ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1719                         ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1720                         ops[i]->ldpc_enc.tb_params.cab =
1721                                         ldpc_enc->tb_params.cab;
1722                         ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1723                         ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1724                 } else {
1725                         ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1726                 }
1727                 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1728                 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1729                 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1730                 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1731                 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1732                 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1733                 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1734                 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1735                 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1736                 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1737         }
1738 }
1739
1740 static int
1741 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1742                 unsigned int order_idx, const int expected_status)
1743 {
1744         int status = op->status;
1745         /* ignore parity mismatch false alarms for long iterations */
1746         if (get_iter_max() >= 10) {
1747                 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1748                                 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1749                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1750                         status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1751                 }
1752                 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1753                                 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1754                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1755                         status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1756                 }
1757         }
1758
1759         TEST_ASSERT(status == expected_status,
1760                         "op_status (%d) != expected_status (%d)",
1761                         op->status, expected_status);
1762
1763         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1764                         "Ordering error, expected %p, got %p",
1765                         (void *)(uintptr_t)order_idx, op->opaque_data);
1766
1767         return TEST_SUCCESS;
1768 }
1769
1770 static int
1771 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1772                 unsigned int order_idx, const int expected_status)
1773 {
1774         TEST_ASSERT(op->status == expected_status,
1775                         "op_status (%d) != expected_status (%d)",
1776                         op->status, expected_status);
1777
1778         if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1779                 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1780                                 "Ordering error, expected %p, got %p",
1781                                 (void *)(uintptr_t)order_idx, op->opaque_data);
1782
1783         return TEST_SUCCESS;
1784 }
1785
1786 static inline int
1787 validate_op_chain(struct rte_bbdev_op_data *op,
1788                 struct op_data_entries *orig_op)
1789 {
1790         uint8_t i;
1791         struct rte_mbuf *m = op->data;
1792         uint8_t nb_dst_segments = orig_op->nb_segments;
1793         uint32_t total_data_size = 0;
1794
1795         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1796                         "Number of segments differ in original (%u) and filled (%u) op",
1797                         nb_dst_segments, m->nb_segs);
1798
1799         /* Validate each mbuf segment length */
1800         for (i = 0; i < nb_dst_segments; ++i) {
1801                 /* Apply offset to the first mbuf segment */
1802                 uint16_t offset = (i == 0) ? op->offset : 0;
1803                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1804                 total_data_size += orig_op->segments[i].length;
1805
1806                 TEST_ASSERT(orig_op->segments[i].length == data_len,
1807                                 "Length of segment differ in original (%u) and filled (%u) op",
1808                                 orig_op->segments[i].length, data_len);
1809                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1810                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1811                                 data_len,
1812                                 "Output buffers (CB=%u) are not equal", i);
1813                 m = m->next;
1814         }
1815
1816         /* Validate total mbuf pkt length */
1817         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1818         TEST_ASSERT(total_data_size == pkt_len,
1819                         "Length of data differ in original (%u) and filled (%u) op",
1820                         total_data_size, pkt_len);
1821
1822         return TEST_SUCCESS;
1823 }
1824
1825 /*
1826  * Compute K0 for a given configuration for HARQ output length computation
1827  * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1828  */
1829 static inline uint16_t
1830 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1831 {
1832         if (rv_index == 0)
1833                 return 0;
1834         uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1835         if (n_cb == n) {
1836                 if (rv_index == 1)
1837                         return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1838                 else if (rv_index == 2)
1839                         return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1840                 else
1841                         return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1842         }
1843         /* LBRM case - includes a division by N */
1844         if (rv_index == 1)
1845                 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1846                                 / n) * z_c;
1847         else if (rv_index == 2)
1848                 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1849                                 / n) * z_c;
1850         else
1851                 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1852                                 / n) * z_c;
1853 }
1854
1855 /* HARQ output length including the Filler bits */
1856 static inline uint16_t
1857 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1858 {
1859         uint16_t k0 = 0;
1860         uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1861         k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1862         /* Compute RM out size and number of rows */
1863         uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1864                         * ops_ld->z_c - ops_ld->n_filler;
1865         uint16_t deRmOutSize = RTE_MIN(
1866                         k0 + ops_ld->cb_params.e +
1867                         ((k0 > parity_offset) ?
1868                                         0 : ops_ld->n_filler),
1869                                         ops_ld->n_cb);
1870         uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1871                         / ops_ld->z_c);
1872         uint16_t harq_output_len = numRows * ops_ld->z_c;
1873         return harq_output_len;
1874 }
1875
1876 static inline int
1877 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1878                 struct op_data_entries *orig_op,
1879                 struct rte_bbdev_op_ldpc_dec *ops_ld)
1880 {
1881         uint8_t i;
1882         uint32_t j, jj, k;
1883         struct rte_mbuf *m = op->data;
1884         uint8_t nb_dst_segments = orig_op->nb_segments;
1885         uint32_t total_data_size = 0;
1886         int8_t *harq_orig, *harq_out, abs_harq_origin;
1887         uint32_t byte_error = 0, cum_error = 0, error;
1888         int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1889         int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1890         uint16_t parity_offset;
1891
1892         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1893                         "Number of segments differ in original (%u) and filled (%u) op",
1894                         nb_dst_segments, m->nb_segs);
1895
1896         /* Validate each mbuf segment length */
1897         for (i = 0; i < nb_dst_segments; ++i) {
1898                 /* Apply offset to the first mbuf segment */
1899                 uint16_t offset = (i == 0) ? op->offset : 0;
1900                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1901                 total_data_size += orig_op->segments[i].length;
1902
1903                 TEST_ASSERT(orig_op->segments[i].length <
1904                                 (uint32_t)(data_len + 64),
1905                                 "Length of segment differ in original (%u) and filled (%u) op",
1906                                 orig_op->segments[i].length, data_len);
1907                 harq_orig = (int8_t *) orig_op->segments[i].addr;
1908                 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1909
1910                 if (!(ldpc_cap_flags &
1911                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1912                                 ) || (ops_ld->op_flags &
1913                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1914                         data_len -= ops_ld->z_c;
1915                         parity_offset = data_len;
1916                 } else {
1917                         /* Compute RM out size and number of rows */
1918                         parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1919                                         * ops_ld->z_c - ops_ld->n_filler;
1920                         uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1921                                         ops_ld->n_filler;
1922                         if (data_len > deRmOutSize)
1923                                 data_len = deRmOutSize;
1924                         if (data_len > orig_op->segments[i].length)
1925                                 data_len = orig_op->segments[i].length;
1926                 }
1927                 /*
1928                  * HARQ output can have minor differences
1929                  * due to integer representation and related scaling
1930                  */
1931                 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1932                         if (j == parity_offset) {
1933                                 /* Special Handling of the filler bits */
1934                                 for (k = 0; k < ops_ld->n_filler; k++) {
1935                                         if (harq_out[jj] !=
1936                                                         llr_max_pre_scaling) {
1937                                                 printf("HARQ Filler issue %d: %d %d\n",
1938                                                         jj, harq_out[jj],
1939                                                         llr_max);
1940                                                 byte_error++;
1941                                         }
1942                                         jj++;
1943                                 }
1944                         }
1945                         if (!(ops_ld->op_flags &
1946                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1947                                 if (ldpc_llr_decimals > 1)
1948                                         harq_out[jj] = (harq_out[jj] + 1)
1949                                                 >> (ldpc_llr_decimals - 1);
1950                                 /* Saturated to S7 */
1951                                 if (harq_orig[j] > llr_max)
1952                                         harq_orig[j] = llr_max;
1953                                 if (harq_orig[j] < -llr_max)
1954                                         harq_orig[j] = -llr_max;
1955                         }
1956                         if (harq_orig[j] != harq_out[jj]) {
1957                                 error = (harq_orig[j] > harq_out[jj]) ?
1958                                                 harq_orig[j] - harq_out[jj] :
1959                                                 harq_out[jj] - harq_orig[j];
1960                                 abs_harq_origin = harq_orig[j] > 0 ?
1961                                                         harq_orig[j] :
1962                                                         -harq_orig[j];
1963                                 /* Residual quantization error */
1964                                 if ((error > 8 && (abs_harq_origin <
1965                                                 (llr_max - 16))) ||
1966                                                 (error > 16)) {
1967                                         printf("HARQ mismatch %d: exp %d act %d => %d\n",
1968                                                         j, harq_orig[j],
1969                                                         harq_out[jj], error);
1970                                         byte_error++;
1971                                         cum_error += error;
1972                                 }
1973                         }
1974                 }
1975                 m = m->next;
1976         }
1977
1978         if (byte_error)
1979                 TEST_ASSERT(byte_error <= 1,
1980                                 "HARQ output mismatch (%d) %d",
1981                                 byte_error, cum_error);
1982
1983         /* Validate total mbuf pkt length */
1984         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1985         TEST_ASSERT(total_data_size < pkt_len + 64,
1986                         "Length of data differ in original (%u) and filled (%u) op",
1987                         total_data_size, pkt_len);
1988
1989         return TEST_SUCCESS;
1990 }
1991
1992 static int
1993 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1994                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1995 {
1996         unsigned int i;
1997         int ret;
1998         struct op_data_entries *hard_data_orig =
1999                         &test_vector.entries[DATA_HARD_OUTPUT];
2000         struct op_data_entries *soft_data_orig =
2001                         &test_vector.entries[DATA_SOFT_OUTPUT];
2002         struct rte_bbdev_op_turbo_dec *ops_td;
2003         struct rte_bbdev_op_data *hard_output;
2004         struct rte_bbdev_op_data *soft_output;
2005         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
2006
2007         for (i = 0; i < n; ++i) {
2008                 ops_td = &ops[i]->turbo_dec;
2009                 hard_output = &ops_td->hard_output;
2010                 soft_output = &ops_td->soft_output;
2011
2012                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2013                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2014                                         "Returned iter_count (%d) > expected iter_count (%d)",
2015                                         ops_td->iter_count, ref_td->iter_count);
2016                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2017                 TEST_ASSERT_SUCCESS(ret,
2018                                 "Checking status and ordering for decoder failed");
2019
2020                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2021                                 hard_data_orig),
2022                                 "Hard output buffers (CB=%u) are not equal",
2023                                 i);
2024
2025                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
2026                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2027                                         soft_data_orig),
2028                                         "Soft output buffers (CB=%u) are not equal",
2029                                         i);
2030         }
2031
2032         return TEST_SUCCESS;
2033 }
2034
2035 /* Check Number of code blocks errors */
2036 static int
2037 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
2038 {
2039         unsigned int i;
2040         struct op_data_entries *hard_data_orig =
2041                         &test_vector.entries[DATA_HARD_OUTPUT];
2042         struct rte_bbdev_op_ldpc_dec *ops_td;
2043         struct rte_bbdev_op_data *hard_output;
2044         int errors = 0;
2045         struct rte_mbuf *m;
2046
2047         for (i = 0; i < n; ++i) {
2048                 ops_td = &ops[i]->ldpc_dec;
2049                 hard_output = &ops_td->hard_output;
2050                 m = hard_output->data;
2051                 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
2052                                 hard_data_orig->segments[0].addr,
2053                                 hard_data_orig->segments[0].length))
2054                         errors++;
2055         }
2056         return errors;
2057 }
2058
2059 static int
2060 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2061                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2062 {
2063         unsigned int i;
2064         int ret;
2065         struct op_data_entries *hard_data_orig =
2066                         &test_vector.entries[DATA_HARD_OUTPUT];
2067         struct op_data_entries *soft_data_orig =
2068                         &test_vector.entries[DATA_SOFT_OUTPUT];
2069         struct op_data_entries *harq_data_orig =
2070                                 &test_vector.entries[DATA_HARQ_OUTPUT];
2071         struct rte_bbdev_op_ldpc_dec *ops_td;
2072         struct rte_bbdev_op_data *hard_output;
2073         struct rte_bbdev_op_data *harq_output;
2074         struct rte_bbdev_op_data *soft_output;
2075         struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2076
2077         for (i = 0; i < n; ++i) {
2078                 ops_td = &ops[i]->ldpc_dec;
2079                 hard_output = &ops_td->hard_output;
2080                 harq_output = &ops_td->harq_combined_output;
2081                 soft_output = &ops_td->soft_output;
2082
2083                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2084                 TEST_ASSERT_SUCCESS(ret,
2085                                 "Checking status and ordering for decoder failed");
2086                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2087                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2088                                         "Returned iter_count (%d) > expected iter_count (%d)",
2089                                         ops_td->iter_count, ref_td->iter_count);
2090                 /*
2091                  * We can ignore output data when the decoding failed to
2092                  * converge or for loop-back cases
2093                  */
2094                 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2095                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2096                                 ) && (
2097                                 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2098                                                 )) == 0)
2099                         TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2100                                         hard_data_orig),
2101                                         "Hard output buffers (CB=%u) are not equal",
2102                                         i);
2103
2104                 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2105                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2106                                         soft_data_orig),
2107                                         "Soft output buffers (CB=%u) are not equal",
2108                                         i);
2109                 if (ref_op->ldpc_dec.op_flags &
2110                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2111                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2112                                         harq_data_orig, ops_td),
2113                                         "HARQ output buffers (CB=%u) are not equal",
2114                                         i);
2115                 }
2116                 if (ref_op->ldpc_dec.op_flags &
2117                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2118                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2119                                         harq_data_orig, ops_td),
2120                                         "HARQ output buffers (CB=%u) are not equal",
2121                                         i);
2122
2123         }
2124
2125         return TEST_SUCCESS;
2126 }
2127
2128
2129 static int
2130 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2131                 struct rte_bbdev_enc_op *ref_op)
2132 {
2133         unsigned int i;
2134         int ret;
2135         struct op_data_entries *hard_data_orig =
2136                         &test_vector.entries[DATA_HARD_OUTPUT];
2137
2138         for (i = 0; i < n; ++i) {
2139                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2140                 TEST_ASSERT_SUCCESS(ret,
2141                                 "Checking status and ordering for encoder failed");
2142                 TEST_ASSERT_SUCCESS(validate_op_chain(
2143                                 &ops[i]->turbo_enc.output,
2144                                 hard_data_orig),
2145                                 "Output buffers (CB=%u) are not equal",
2146                                 i);
2147         }
2148
2149         return TEST_SUCCESS;
2150 }
2151
2152 static int
2153 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2154                 struct rte_bbdev_enc_op *ref_op)
2155 {
2156         unsigned int i;
2157         int ret;
2158         struct op_data_entries *hard_data_orig =
2159                         &test_vector.entries[DATA_HARD_OUTPUT];
2160
2161         for (i = 0; i < n; ++i) {
2162                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2163                 TEST_ASSERT_SUCCESS(ret,
2164                                 "Checking status and ordering for encoder failed");
2165                 TEST_ASSERT_SUCCESS(validate_op_chain(
2166                                 &ops[i]->ldpc_enc.output,
2167                                 hard_data_orig),
2168                                 "Output buffers (CB=%u) are not equal",
2169                                 i);
2170         }
2171
2172         return TEST_SUCCESS;
2173 }
2174
2175 static void
2176 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2177 {
2178         unsigned int i;
2179         struct op_data_entries *entry;
2180
2181         op->turbo_dec = test_vector.turbo_dec;
2182         entry = &test_vector.entries[DATA_INPUT];
2183         for (i = 0; i < entry->nb_segments; ++i)
2184                 op->turbo_dec.input.length +=
2185                                 entry->segments[i].length;
2186 }
2187
2188 static void
2189 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2190 {
2191         unsigned int i;
2192         struct op_data_entries *entry;
2193
2194         op->ldpc_dec = test_vector.ldpc_dec;
2195         entry = &test_vector.entries[DATA_INPUT];
2196         for (i = 0; i < entry->nb_segments; ++i)
2197                 op->ldpc_dec.input.length +=
2198                                 entry->segments[i].length;
2199         if (test_vector.ldpc_dec.op_flags &
2200                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2201                 entry = &test_vector.entries[DATA_HARQ_INPUT];
2202                 for (i = 0; i < entry->nb_segments; ++i)
2203                         op->ldpc_dec.harq_combined_input.length +=
2204                                 entry->segments[i].length;
2205         }
2206 }
2207
2208
2209 static void
2210 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2211 {
2212         unsigned int i;
2213         struct op_data_entries *entry;
2214
2215         op->turbo_enc = test_vector.turbo_enc;
2216         entry = &test_vector.entries[DATA_INPUT];
2217         for (i = 0; i < entry->nb_segments; ++i)
2218                 op->turbo_enc.input.length +=
2219                                 entry->segments[i].length;
2220 }
2221
2222 static void
2223 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2224 {
2225         unsigned int i;
2226         struct op_data_entries *entry;
2227
2228         op->ldpc_enc = test_vector.ldpc_enc;
2229         entry = &test_vector.entries[DATA_INPUT];
2230         for (i = 0; i < entry->nb_segments; ++i)
2231                 op->ldpc_enc.input.length +=
2232                                 entry->segments[i].length;
2233 }
2234
2235 static uint32_t
2236 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2237 {
2238         uint8_t i;
2239         uint32_t c, r, tb_size = 0;
2240
2241         if (op->turbo_dec.code_block_mode) {
2242                 tb_size = op->turbo_dec.tb_params.k_neg;
2243         } else {
2244                 c = op->turbo_dec.tb_params.c;
2245                 r = op->turbo_dec.tb_params.r;
2246                 for (i = 0; i < c-r; i++)
2247                         tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2248                                 op->turbo_dec.tb_params.k_neg :
2249                                 op->turbo_dec.tb_params.k_pos;
2250         }
2251         return tb_size;
2252 }
2253
2254 static uint32_t
2255 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2256 {
2257         uint8_t i;
2258         uint32_t c, r, tb_size = 0;
2259         uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2260
2261         if (op->ldpc_dec.code_block_mode) {
2262                 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2263         } else {
2264                 c = op->ldpc_dec.tb_params.c;
2265                 r = op->ldpc_dec.tb_params.r;
2266                 for (i = 0; i < c-r; i++)
2267                         tb_size += sys_cols * op->ldpc_dec.z_c
2268                                         - op->ldpc_dec.n_filler;
2269         }
2270         return tb_size;
2271 }
2272
2273 static uint32_t
2274 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2275 {
2276         uint8_t i;
2277         uint32_t c, r, tb_size = 0;
2278
2279         if (op->turbo_enc.code_block_mode) {
2280                 tb_size = op->turbo_enc.tb_params.k_neg;
2281         } else {
2282                 c = op->turbo_enc.tb_params.c;
2283                 r = op->turbo_enc.tb_params.r;
2284                 for (i = 0; i < c-r; i++)
2285                         tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2286                                 op->turbo_enc.tb_params.k_neg :
2287                                 op->turbo_enc.tb_params.k_pos;
2288         }
2289         return tb_size;
2290 }
2291
2292 static uint32_t
2293 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2294 {
2295         uint8_t i;
2296         uint32_t c, r, tb_size = 0;
2297         uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2298
2299         if (op->turbo_enc.code_block_mode) {
2300                 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2301         } else {
2302                 c = op->turbo_enc.tb_params.c;
2303                 r = op->turbo_enc.tb_params.r;
2304                 for (i = 0; i < c-r; i++)
2305                         tb_size += sys_cols * op->ldpc_enc.z_c
2306                                         - op->ldpc_enc.n_filler;
2307         }
2308         return tb_size;
2309 }
2310
2311
2312 static int
2313 init_test_op_params(struct test_op_params *op_params,
2314                 enum rte_bbdev_op_type op_type, const int expected_status,
2315                 const int vector_mask, struct rte_mempool *ops_mp,
2316                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2317 {
2318         int ret = 0;
2319         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2320                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2321                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2322                                 &op_params->ref_dec_op, 1);
2323         else
2324                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2325                                 &op_params->ref_enc_op, 1);
2326
2327         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2328
2329         op_params->mp = ops_mp;
2330         op_params->burst_sz = burst_sz;
2331         op_params->num_to_process = num_to_process;
2332         op_params->num_lcores = num_lcores;
2333         op_params->vector_mask = vector_mask;
2334         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2335                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2336                 op_params->ref_dec_op->status = expected_status;
2337         else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2338                         || op_type == RTE_BBDEV_OP_LDPC_ENC)
2339                 op_params->ref_enc_op->status = expected_status;
2340         return 0;
2341 }
2342
2343 static int
2344 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2345                 struct test_op_params *op_params)
2346 {
2347         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2348         unsigned int i;
2349         struct active_device *ad;
2350         unsigned int burst_sz = get_burst_sz();
2351         enum rte_bbdev_op_type op_type = test_vector.op_type;
2352         const struct rte_bbdev_op_cap *capabilities = NULL;
2353
2354         ad = &active_devs[dev_id];
2355
2356         /* Check if device supports op_type */
2357         if (!is_avail_op(ad, test_vector.op_type))
2358                 return TEST_SUCCESS;
2359
2360         struct rte_bbdev_info info;
2361         rte_bbdev_info_get(ad->dev_id, &info);
2362         socket_id = GET_SOCKET(info.socket_id);
2363
2364         f_ret = create_mempools(ad, socket_id, op_type,
2365                         get_num_ops());
2366         if (f_ret != TEST_SUCCESS) {
2367                 printf("Couldn't create mempools");
2368                 goto fail;
2369         }
2370         if (op_type == RTE_BBDEV_OP_NONE)
2371                 op_type = RTE_BBDEV_OP_TURBO_ENC;
2372
2373         f_ret = init_test_op_params(op_params, test_vector.op_type,
2374                         test_vector.expected_status,
2375                         test_vector.mask,
2376                         ad->ops_mempool,
2377                         burst_sz,
2378                         get_num_ops(),
2379                         get_num_lcores());
2380         if (f_ret != TEST_SUCCESS) {
2381                 printf("Couldn't init test op params");
2382                 goto fail;
2383         }
2384
2385
2386         /* Find capabilities */
2387         const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2388         for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2389                 if (cap->type == test_vector.op_type) {
2390                         capabilities = cap;
2391                         break;
2392                 }
2393                 cap++;
2394         }
2395         TEST_ASSERT_NOT_NULL(capabilities,
2396                         "Couldn't find capabilities");
2397
2398         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2399                 create_reference_dec_op(op_params->ref_dec_op);
2400         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2401                 create_reference_enc_op(op_params->ref_enc_op);
2402         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2403                 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2404         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2405                 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2406
2407         for (i = 0; i < ad->nb_queues; ++i) {
2408                 f_ret = fill_queue_buffers(op_params,
2409                                 ad->in_mbuf_pool,
2410                                 ad->hard_out_mbuf_pool,
2411                                 ad->soft_out_mbuf_pool,
2412                                 ad->harq_in_mbuf_pool,
2413                                 ad->harq_out_mbuf_pool,
2414                                 ad->queue_ids[i],
2415                                 capabilities,
2416                                 info.drv.min_alignment,
2417                                 socket_id);
2418                 if (f_ret != TEST_SUCCESS) {
2419                         printf("Couldn't init queue buffers");
2420                         goto fail;
2421                 }
2422         }
2423
2424         /* Run test case function */
2425         t_ret = test_case_func(ad, op_params);
2426
2427         /* Free active device resources and return */
2428         free_buffers(ad, op_params);
2429         return t_ret;
2430
2431 fail:
2432         free_buffers(ad, op_params);
2433         return TEST_FAILED;
2434 }
2435
2436 /* Run given test function per active device per supported op type
2437  * per burst size.
2438  */
2439 static int
2440 run_test_case(test_case_function *test_case_func)
2441 {
2442         int ret = 0;
2443         uint8_t dev;
2444
2445         /* Alloc op_params */
2446         struct test_op_params *op_params = rte_zmalloc(NULL,
2447                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2448         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2449                         RTE_ALIGN(sizeof(struct test_op_params),
2450                                 RTE_CACHE_LINE_SIZE));
2451
2452         /* For each device run test case function */
2453         for (dev = 0; dev < nb_active_devs; ++dev)
2454                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2455
2456         rte_free(op_params);
2457
2458         return ret;
2459 }
2460
2461
2462 /* Push back the HARQ output from DDR to host */
2463 static void
2464 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2465                 struct rte_bbdev_dec_op **ops,
2466                 const uint16_t n)
2467 {
2468         uint16_t j;
2469         int save_status, ret;
2470         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2471         struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2472         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2473         bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2474         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2475         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2476         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2477         for (j = 0; j < n; ++j) {
2478                 if ((loopback && mem_out) || hc_out) {
2479                         save_status = ops[j]->status;
2480                         ops[j]->ldpc_dec.op_flags =
2481                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2482                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2483                         if (h_comp)
2484                                 ops[j]->ldpc_dec.op_flags +=
2485                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2486                         ops[j]->ldpc_dec.harq_combined_input.offset =
2487                                         harq_offset;
2488                         ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2489                         harq_offset += HARQ_INCR;
2490                         if (!loopback)
2491                                 ops[j]->ldpc_dec.harq_combined_input.length =
2492                                 ops[j]->ldpc_dec.harq_combined_output.length;
2493                         rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2494                                         &ops[j], 1);
2495                         ret = 0;
2496                         while (ret == 0)
2497                                 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2498                                                 dev_id, queue_id,
2499                                                 &ops_deq[j], 1);
2500                         ops[j]->ldpc_dec.op_flags = flags;
2501                         ops[j]->status = save_status;
2502                 }
2503         }
2504 }
2505
2506 /*
2507  * Push back the HARQ output from HW DDR to Host
2508  * Preload HARQ memory input and adjust HARQ offset
2509  */
2510 static void
2511 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2512                 struct rte_bbdev_dec_op **ops, const uint16_t n,
2513                 bool preload)
2514 {
2515         uint16_t j;
2516         int ret;
2517         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2518         struct rte_bbdev_op_data save_hc_in, save_hc_out;
2519         struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2520         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2521         bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2522         bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2523         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2524         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2525         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2526         for (j = 0; j < n; ++j) {
2527                 if ((mem_in || hc_in) && preload) {
2528                         save_hc_in = ops[j]->ldpc_dec.harq_combined_input;
2529                         save_hc_out = ops[j]->ldpc_dec.harq_combined_output;
2530                         ops[j]->ldpc_dec.op_flags =
2531                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2532                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2533                         if (h_comp)
2534                                 ops[j]->ldpc_dec.op_flags +=
2535                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2536                         ops[j]->ldpc_dec.harq_combined_output.offset =
2537                                         harq_offset;
2538                         ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2539                         rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2540                                         &ops[j], 1);
2541                         ret = 0;
2542                         while (ret == 0)
2543                                 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2544                                         dev_id, queue_id, &ops_deq[j], 1);
2545                         ops[j]->ldpc_dec.op_flags = flags;
2546                         ops[j]->ldpc_dec.harq_combined_input = save_hc_in;
2547                         ops[j]->ldpc_dec.harq_combined_output = save_hc_out;
2548                 }
2549                 /* Adjust HARQ offset when we reach external DDR */
2550                 if (mem_in || hc_in)
2551                         ops[j]->ldpc_dec.harq_combined_input.offset
2552                                 = harq_offset;
2553                 if (mem_out || hc_out)
2554                         ops[j]->ldpc_dec.harq_combined_output.offset
2555                                 = harq_offset;
2556                 harq_offset += HARQ_INCR;
2557         }
2558 }
2559
2560 static void
2561 dequeue_event_callback(uint16_t dev_id,
2562                 enum rte_bbdev_event_type event, void *cb_arg,
2563                 void *ret_param)
2564 {
2565         int ret;
2566         uint16_t i;
2567         uint64_t total_time;
2568         uint16_t deq, burst_sz, num_ops;
2569         uint16_t queue_id = *(uint16_t *) ret_param;
2570         struct rte_bbdev_info info;
2571         double tb_len_bits;
2572         struct thread_params *tp = cb_arg;
2573
2574         /* Find matching thread params using queue_id */
2575         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2576                 if (tp->queue_id == queue_id)
2577                         break;
2578
2579         if (i == MAX_QUEUES) {
2580                 printf("%s: Queue_id from interrupt details was not found!\n",
2581                                 __func__);
2582                 return;
2583         }
2584
2585         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2586                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2587                 printf(
2588                         "Dequeue interrupt handler called for incorrect event!\n");
2589                 return;
2590         }
2591
2592         burst_sz = rte_atomic16_read(&tp->burst_sz);
2593         num_ops = tp->op_params->num_to_process;
2594
2595         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2596                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2597                                 &tp->dec_ops[
2598                                         rte_atomic16_read(&tp->nb_dequeued)],
2599                                 burst_sz);
2600         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2601                 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2602                                 &tp->dec_ops[
2603                                         rte_atomic16_read(&tp->nb_dequeued)],
2604                                 burst_sz);
2605         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2606                 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2607                                 &tp->enc_ops[
2608                                         rte_atomic16_read(&tp->nb_dequeued)],
2609                                 burst_sz);
2610         else /*RTE_BBDEV_OP_TURBO_ENC*/
2611                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2612                                 &tp->enc_ops[
2613                                         rte_atomic16_read(&tp->nb_dequeued)],
2614                                 burst_sz);
2615
2616         if (deq < burst_sz) {
2617                 printf(
2618                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2619                         burst_sz, deq);
2620                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2621                 return;
2622         }
2623
2624         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2625                 rte_atomic16_add(&tp->nb_dequeued, deq);
2626                 return;
2627         }
2628
2629         total_time = rte_rdtsc_precise() - tp->start_time;
2630
2631         rte_bbdev_info_get(dev_id, &info);
2632
2633         ret = TEST_SUCCESS;
2634
2635         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2636                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2637                 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2638                                 tp->op_params->vector_mask);
2639                 /* get the max of iter_count for all dequeued ops */
2640                 for (i = 0; i < num_ops; ++i)
2641                         tp->iter_count = RTE_MAX(
2642                                         tp->dec_ops[i]->turbo_dec.iter_count,
2643                                         tp->iter_count);
2644                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2645         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2646                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2647                 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2648                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2649         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2650                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2651                 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2652                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2653         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2654                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2655                 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2656                                 tp->op_params->vector_mask);
2657                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2658         }
2659
2660         if (ret) {
2661                 printf("Buffers validation failed\n");
2662                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2663         }
2664
2665         switch (test_vector.op_type) {
2666         case RTE_BBDEV_OP_TURBO_DEC:
2667                 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2668                 break;
2669         case RTE_BBDEV_OP_TURBO_ENC:
2670                 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2671                 break;
2672         case RTE_BBDEV_OP_LDPC_DEC:
2673                 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2674                 break;
2675         case RTE_BBDEV_OP_LDPC_ENC:
2676                 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2677                 break;
2678         case RTE_BBDEV_OP_NONE:
2679                 tb_len_bits = 0.0;
2680                 break;
2681         default:
2682                 printf("Unknown op type: %d\n", test_vector.op_type);
2683                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2684                 return;
2685         }
2686
2687         tp->ops_per_sec += ((double)num_ops) /
2688                         ((double)total_time / (double)rte_get_tsc_hz());
2689         tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2690                         ((double)total_time / (double)rte_get_tsc_hz());
2691
2692         rte_atomic16_add(&tp->nb_dequeued, deq);
2693 }
2694
2695 static int
2696 throughput_intr_lcore_ldpc_dec(void *arg)
2697 {
2698         struct thread_params *tp = arg;
2699         unsigned int enqueued;
2700         const uint16_t queue_id = tp->queue_id;
2701         const uint16_t burst_sz = tp->op_params->burst_sz;
2702         const uint16_t num_to_process = tp->op_params->num_to_process;
2703         struct rte_bbdev_dec_op *ops[num_to_process];
2704         struct test_buffers *bufs = NULL;
2705         struct rte_bbdev_info info;
2706         int ret, i, j;
2707         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2708         uint16_t num_to_enq, enq;
2709
2710         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2711                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2712         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2713                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2714
2715         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2716                         "BURST_SIZE should be <= %u", MAX_BURST);
2717
2718         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2719                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2720                         tp->dev_id, queue_id);
2721
2722         rte_bbdev_info_get(tp->dev_id, &info);
2723
2724         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2725                         "NUM_OPS cannot exceed %u for this device",
2726                         info.drv.queue_size_lim);
2727
2728         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2729
2730         rte_atomic16_clear(&tp->processing_status);
2731         rte_atomic16_clear(&tp->nb_dequeued);
2732
2733         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2734                 rte_pause();
2735
2736         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2737                                 num_to_process);
2738         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2739                         num_to_process);
2740         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2741                 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2742                                 bufs->hard_outputs, bufs->soft_outputs,
2743                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2744
2745         /* Set counter to validate the ordering */
2746         for (j = 0; j < num_to_process; ++j)
2747                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2748
2749         for (j = 0; j < TEST_REPETITIONS; ++j) {
2750                 for (i = 0; i < num_to_process; ++i) {
2751                         if (!loopback)
2752                                 rte_pktmbuf_reset(
2753                                         ops[i]->ldpc_dec.hard_output.data);
2754                         if (hc_out || loopback)
2755                                 mbuf_reset(
2756                                 ops[i]->ldpc_dec.harq_combined_output.data);
2757                 }
2758
2759                 tp->start_time = rte_rdtsc_precise();
2760                 for (enqueued = 0; enqueued < num_to_process;) {
2761                         num_to_enq = burst_sz;
2762
2763                         if (unlikely(num_to_process - enqueued < num_to_enq))
2764                                 num_to_enq = num_to_process - enqueued;
2765
2766                         enq = 0;
2767                         do {
2768                                 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2769                                                 tp->dev_id,
2770                                                 queue_id, &ops[enqueued],
2771                                                 num_to_enq);
2772                         } while (unlikely(num_to_enq != enq));
2773                         enqueued += enq;
2774
2775                         /* Write to thread burst_sz current number of enqueued
2776                          * descriptors. It ensures that proper number of
2777                          * descriptors will be dequeued in callback
2778                          * function - needed for last batch in case where
2779                          * the number of operations is not a multiple of
2780                          * burst size.
2781                          */
2782                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2783
2784                         /* Wait until processing of previous batch is
2785                          * completed
2786                          */
2787                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2788                                         (int16_t) enqueued)
2789                                 rte_pause();
2790                 }
2791                 if (j != TEST_REPETITIONS - 1)
2792                         rte_atomic16_clear(&tp->nb_dequeued);
2793         }
2794
2795         return TEST_SUCCESS;
2796 }
2797
2798 static int
2799 throughput_intr_lcore_dec(void *arg)
2800 {
2801         struct thread_params *tp = arg;
2802         unsigned int enqueued;
2803         const uint16_t queue_id = tp->queue_id;
2804         const uint16_t burst_sz = tp->op_params->burst_sz;
2805         const uint16_t num_to_process = tp->op_params->num_to_process;
2806         struct rte_bbdev_dec_op *ops[num_to_process];
2807         struct test_buffers *bufs = NULL;
2808         struct rte_bbdev_info info;
2809         int ret, i, j;
2810         uint16_t num_to_enq, enq;
2811
2812         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2813                         "BURST_SIZE should be <= %u", MAX_BURST);
2814
2815         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2816                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2817                         tp->dev_id, queue_id);
2818
2819         rte_bbdev_info_get(tp->dev_id, &info);
2820
2821         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2822                         "NUM_OPS cannot exceed %u for this device",
2823                         info.drv.queue_size_lim);
2824
2825         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2826
2827         rte_atomic16_clear(&tp->processing_status);
2828         rte_atomic16_clear(&tp->nb_dequeued);
2829
2830         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2831                 rte_pause();
2832
2833         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2834                                 num_to_process);
2835         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2836                         num_to_process);
2837         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2838                 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2839                                 bufs->hard_outputs, bufs->soft_outputs,
2840                                 tp->op_params->ref_dec_op);
2841
2842         /* Set counter to validate the ordering */
2843         for (j = 0; j < num_to_process; ++j)
2844                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2845
2846         for (j = 0; j < TEST_REPETITIONS; ++j) {
2847                 for (i = 0; i < num_to_process; ++i)
2848                         rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2849
2850                 tp->start_time = rte_rdtsc_precise();
2851                 for (enqueued = 0; enqueued < num_to_process;) {
2852                         num_to_enq = burst_sz;
2853
2854                         if (unlikely(num_to_process - enqueued < num_to_enq))
2855                                 num_to_enq = num_to_process - enqueued;
2856
2857                         enq = 0;
2858                         do {
2859                                 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2860                                                 queue_id, &ops[enqueued],
2861                                                 num_to_enq);
2862                         } while (unlikely(num_to_enq != enq));
2863                         enqueued += enq;
2864
2865                         /* Write to thread burst_sz current number of enqueued
2866                          * descriptors. It ensures that proper number of
2867                          * descriptors will be dequeued in callback
2868                          * function - needed for last batch in case where
2869                          * the number of operations is not a multiple of
2870                          * burst size.
2871                          */
2872                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2873
2874                         /* Wait until processing of previous batch is
2875                          * completed
2876                          */
2877                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2878                                         (int16_t) enqueued)
2879                                 rte_pause();
2880                 }
2881                 if (j != TEST_REPETITIONS - 1)
2882                         rte_atomic16_clear(&tp->nb_dequeued);
2883         }
2884
2885         return TEST_SUCCESS;
2886 }
2887
2888 static int
2889 throughput_intr_lcore_enc(void *arg)
2890 {
2891         struct thread_params *tp = arg;
2892         unsigned int enqueued;
2893         const uint16_t queue_id = tp->queue_id;
2894         const uint16_t burst_sz = tp->op_params->burst_sz;
2895         const uint16_t num_to_process = tp->op_params->num_to_process;
2896         struct rte_bbdev_enc_op *ops[num_to_process];
2897         struct test_buffers *bufs = NULL;
2898         struct rte_bbdev_info info;
2899         int ret, i, j;
2900         uint16_t num_to_enq, enq;
2901
2902         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2903                         "BURST_SIZE should be <= %u", MAX_BURST);
2904
2905         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2906                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2907                         tp->dev_id, queue_id);
2908
2909         rte_bbdev_info_get(tp->dev_id, &info);
2910
2911         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2912                         "NUM_OPS cannot exceed %u for this device",
2913                         info.drv.queue_size_lim);
2914
2915         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2916
2917         rte_atomic16_clear(&tp->processing_status);
2918         rte_atomic16_clear(&tp->nb_dequeued);
2919
2920         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2921                 rte_pause();
2922
2923         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2924                         num_to_process);
2925         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2926                         num_to_process);
2927         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2928                 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2929                                 bufs->hard_outputs, tp->op_params->ref_enc_op);
2930
2931         /* Set counter to validate the ordering */
2932         for (j = 0; j < num_to_process; ++j)
2933                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2934
2935         for (j = 0; j < TEST_REPETITIONS; ++j) {
2936                 for (i = 0; i < num_to_process; ++i)
2937                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2938
2939                 tp->start_time = rte_rdtsc_precise();
2940                 for (enqueued = 0; enqueued < num_to_process;) {
2941                         num_to_enq = burst_sz;
2942
2943                         if (unlikely(num_to_process - enqueued < num_to_enq))
2944                                 num_to_enq = num_to_process - enqueued;
2945
2946                         enq = 0;
2947                         do {
2948                                 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2949                                                 queue_id, &ops[enqueued],
2950                                                 num_to_enq);
2951                         } while (unlikely(enq != num_to_enq));
2952                         enqueued += enq;
2953
2954                         /* Write to thread burst_sz current number of enqueued
2955                          * descriptors. It ensures that proper number of
2956                          * descriptors will be dequeued in callback
2957                          * function - needed for last batch in case where
2958                          * the number of operations is not a multiple of
2959                          * burst size.
2960                          */
2961                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2962
2963                         /* Wait until processing of previous batch is
2964                          * completed
2965                          */
2966                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2967                                         (int16_t) enqueued)
2968                                 rte_pause();
2969                 }
2970                 if (j != TEST_REPETITIONS - 1)
2971                         rte_atomic16_clear(&tp->nb_dequeued);
2972         }
2973
2974         return TEST_SUCCESS;
2975 }
2976
2977
2978 static int
2979 throughput_intr_lcore_ldpc_enc(void *arg)
2980 {
2981         struct thread_params *tp = arg;
2982         unsigned int enqueued;
2983         const uint16_t queue_id = tp->queue_id;
2984         const uint16_t burst_sz = tp->op_params->burst_sz;
2985         const uint16_t num_to_process = tp->op_params->num_to_process;
2986         struct rte_bbdev_enc_op *ops[num_to_process];
2987         struct test_buffers *bufs = NULL;
2988         struct rte_bbdev_info info;
2989         int ret, i, j;
2990         uint16_t num_to_enq, enq;
2991
2992         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2993                         "BURST_SIZE should be <= %u", MAX_BURST);
2994
2995         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2996                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2997                         tp->dev_id, queue_id);
2998
2999         rte_bbdev_info_get(tp->dev_id, &info);
3000
3001         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
3002                         "NUM_OPS cannot exceed %u for this device",
3003                         info.drv.queue_size_lim);
3004
3005         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3006
3007         rte_atomic16_clear(&tp->processing_status);
3008         rte_atomic16_clear(&tp->nb_dequeued);
3009
3010         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3011                 rte_pause();
3012
3013         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
3014                         num_to_process);
3015         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3016                         num_to_process);
3017         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3018                 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
3019                                 bufs->inputs, bufs->hard_outputs,
3020                                 tp->op_params->ref_enc_op);
3021
3022         /* Set counter to validate the ordering */
3023         for (j = 0; j < num_to_process; ++j)
3024                 ops[j]->opaque_data = (void *)(uintptr_t)j;
3025
3026         for (j = 0; j < TEST_REPETITIONS; ++j) {
3027                 for (i = 0; i < num_to_process; ++i)
3028                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
3029
3030                 tp->start_time = rte_rdtsc_precise();
3031                 for (enqueued = 0; enqueued < num_to_process;) {
3032                         num_to_enq = burst_sz;
3033
3034                         if (unlikely(num_to_process - enqueued < num_to_enq))
3035                                 num_to_enq = num_to_process - enqueued;
3036
3037                         enq = 0;
3038                         do {
3039                                 enq += rte_bbdev_enqueue_ldpc_enc_ops(
3040                                                 tp->dev_id,
3041                                                 queue_id, &ops[enqueued],
3042                                                 num_to_enq);
3043                         } while (unlikely(enq != num_to_enq));
3044                         enqueued += enq;
3045
3046                         /* Write to thread burst_sz current number of enqueued
3047                          * descriptors. It ensures that proper number of
3048                          * descriptors will be dequeued in callback
3049                          * function - needed for last batch in case where
3050                          * the number of operations is not a multiple of
3051                          * burst size.
3052                          */
3053                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
3054
3055                         /* Wait until processing of previous batch is
3056                          * completed
3057                          */
3058                         while (rte_atomic16_read(&tp->nb_dequeued) !=
3059                                         (int16_t) enqueued)
3060                                 rte_pause();
3061                 }
3062                 if (j != TEST_REPETITIONS - 1)
3063                         rte_atomic16_clear(&tp->nb_dequeued);
3064         }
3065
3066         return TEST_SUCCESS;
3067 }
3068
3069 static int
3070 throughput_pmd_lcore_dec(void *arg)
3071 {
3072         struct thread_params *tp = arg;
3073         uint16_t enq, deq;
3074         uint64_t total_time = 0, start_time;
3075         const uint16_t queue_id = tp->queue_id;
3076         const uint16_t burst_sz = tp->op_params->burst_sz;
3077         const uint16_t num_ops = tp->op_params->num_to_process;
3078         struct rte_bbdev_dec_op *ops_enq[num_ops];
3079         struct rte_bbdev_dec_op *ops_deq[num_ops];
3080         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3081         struct test_buffers *bufs = NULL;
3082         int i, j, ret;
3083         struct rte_bbdev_info info;
3084         uint16_t num_to_enq;
3085
3086         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3087                         "BURST_SIZE should be <= %u", MAX_BURST);
3088
3089         rte_bbdev_info_get(tp->dev_id, &info);
3090
3091         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3092                         "NUM_OPS cannot exceed %u for this device",
3093                         info.drv.queue_size_lim);
3094
3095         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3096
3097         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3098                 rte_pause();
3099
3100         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3101         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3102
3103         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3104                 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3105                                 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3106
3107         /* Set counter to validate the ordering */
3108         for (j = 0; j < num_ops; ++j)
3109                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3110
3111         for (i = 0; i < TEST_REPETITIONS; ++i) {
3112
3113                 for (j = 0; j < num_ops; ++j)
3114                         mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3115
3116                 start_time = rte_rdtsc_precise();
3117
3118                 for (enq = 0, deq = 0; enq < num_ops;) {
3119                         num_to_enq = burst_sz;
3120
3121                         if (unlikely(num_ops - enq < num_to_enq))
3122                                 num_to_enq = num_ops - enq;
3123
3124                         enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3125                                         queue_id, &ops_enq[enq], num_to_enq);
3126
3127                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3128                                         queue_id, &ops_deq[deq], enq - deq);
3129                 }
3130
3131                 /* dequeue the remaining */
3132                 while (deq < enq) {
3133                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3134                                         queue_id, &ops_deq[deq], enq - deq);
3135                 }
3136
3137                 total_time += rte_rdtsc_precise() - start_time;
3138         }
3139
3140         tp->iter_count = 0;
3141         /* get the max of iter_count for all dequeued ops */
3142         for (i = 0; i < num_ops; ++i) {
3143                 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3144                                 tp->iter_count);
3145         }
3146
3147         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3148                 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3149                                 tp->op_params->vector_mask);
3150                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3151         }
3152
3153         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3154
3155         double tb_len_bits = calc_dec_TB_size(ref_op);
3156
3157         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3158                         ((double)total_time / (double)rte_get_tsc_hz());
3159         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3160                         1000000.0) / ((double)total_time /
3161                         (double)rte_get_tsc_hz());
3162
3163         return TEST_SUCCESS;
3164 }
3165
3166 static int
3167 bler_pmd_lcore_ldpc_dec(void *arg)
3168 {
3169         struct thread_params *tp = arg;
3170         uint16_t enq, deq;
3171         uint64_t total_time = 0, start_time;
3172         const uint16_t queue_id = tp->queue_id;
3173         const uint16_t burst_sz = tp->op_params->burst_sz;
3174         const uint16_t num_ops = tp->op_params->num_to_process;
3175         struct rte_bbdev_dec_op *ops_enq[num_ops];
3176         struct rte_bbdev_dec_op *ops_deq[num_ops];
3177         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3178         struct test_buffers *bufs = NULL;
3179         int i, j, ret;
3180         float parity_bler = 0;
3181         struct rte_bbdev_info info;
3182         uint16_t num_to_enq;
3183         bool extDdr = check_bit(ldpc_cap_flags,
3184                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3185         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3186                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3187         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3188                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3189
3190         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3191                         "BURST_SIZE should be <= %u", MAX_BURST);
3192
3193         rte_bbdev_info_get(tp->dev_id, &info);
3194
3195         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3196                         "NUM_OPS cannot exceed %u for this device",
3197                         info.drv.queue_size_lim);
3198
3199         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3200
3201         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3202                 rte_pause();
3203
3204         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3205         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3206
3207         /* For BLER tests we need to enable early termination */
3208         if (!check_bit(ref_op->ldpc_dec.op_flags,
3209                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3210                 ref_op->ldpc_dec.op_flags +=
3211                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3212         ref_op->ldpc_dec.iter_max = get_iter_max();
3213         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3214
3215         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3216                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3217                                 bufs->hard_outputs, bufs->soft_outputs,
3218                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3219         generate_llr_input(num_ops, bufs->inputs, ref_op);
3220
3221         /* Set counter to validate the ordering */
3222         for (j = 0; j < num_ops; ++j)
3223                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3224
3225         for (i = 0; i < 1; ++i) { /* Could add more iterations */
3226                 for (j = 0; j < num_ops; ++j) {
3227                         if (!loopback)
3228                                 mbuf_reset(
3229                                 ops_enq[j]->ldpc_dec.hard_output.data);
3230                         if (hc_out || loopback)
3231                                 mbuf_reset(
3232                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3233                 }
3234                 if (extDdr) {
3235                         bool preload = i == (TEST_REPETITIONS - 1);
3236                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3237                                         num_ops, preload);
3238                 }
3239                 start_time = rte_rdtsc_precise();
3240
3241                 for (enq = 0, deq = 0; enq < num_ops;) {
3242                         num_to_enq = burst_sz;
3243
3244                         if (unlikely(num_ops - enq < num_to_enq))
3245                                 num_to_enq = num_ops - enq;
3246
3247                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3248                                         queue_id, &ops_enq[enq], num_to_enq);
3249
3250                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3251                                         queue_id, &ops_deq[deq], enq - deq);
3252                 }
3253
3254                 /* dequeue the remaining */
3255                 while (deq < enq) {
3256                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3257                                         queue_id, &ops_deq[deq], enq - deq);
3258                 }
3259
3260                 total_time += rte_rdtsc_precise() - start_time;
3261         }
3262
3263         tp->iter_count = 0;
3264         tp->iter_average = 0;
3265         /* get the max of iter_count for all dequeued ops */
3266         for (i = 0; i < num_ops; ++i) {
3267                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3268                                 tp->iter_count);
3269                 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3270                 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3271                         parity_bler += 1.0;
3272         }
3273
3274         parity_bler /= num_ops; /* This one is based on SYND */
3275         tp->iter_average /= num_ops;
3276         tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3277
3278         if (test_vector.op_type != RTE_BBDEV_OP_NONE
3279                         && tp->bler == 0
3280                         && parity_bler == 0
3281                         && !hc_out) {
3282                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3283                                 tp->op_params->vector_mask);
3284                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3285         }
3286
3287         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3288
3289         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3290         tp->ops_per_sec = ((double)num_ops * 1) /
3291                         ((double)total_time / (double)rte_get_tsc_hz());
3292         tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3293                         1000000.0) / ((double)total_time /
3294                         (double)rte_get_tsc_hz());
3295
3296         return TEST_SUCCESS;
3297 }
3298
3299 static int
3300 throughput_pmd_lcore_ldpc_dec(void *arg)
3301 {
3302         struct thread_params *tp = arg;
3303         uint16_t enq, deq;
3304         uint64_t total_time = 0, start_time;
3305         const uint16_t queue_id = tp->queue_id;
3306         const uint16_t burst_sz = tp->op_params->burst_sz;
3307         const uint16_t num_ops = tp->op_params->num_to_process;
3308         struct rte_bbdev_dec_op *ops_enq[num_ops];
3309         struct rte_bbdev_dec_op *ops_deq[num_ops];
3310         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3311         struct test_buffers *bufs = NULL;
3312         int i, j, ret;
3313         struct rte_bbdev_info info;
3314         uint16_t num_to_enq;
3315         bool extDdr = check_bit(ldpc_cap_flags,
3316                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3317         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3318                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3319         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3320                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3321
3322         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3323                         "BURST_SIZE should be <= %u", MAX_BURST);
3324
3325         rte_bbdev_info_get(tp->dev_id, &info);
3326
3327         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3328                         "NUM_OPS cannot exceed %u for this device",
3329                         info.drv.queue_size_lim);
3330
3331         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3332
3333         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3334                 rte_pause();
3335
3336         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3337         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3338
3339         /* For throughput tests we need to disable early termination */
3340         if (check_bit(ref_op->ldpc_dec.op_flags,
3341                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3342                 ref_op->ldpc_dec.op_flags -=
3343                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3344         ref_op->ldpc_dec.iter_max = get_iter_max();
3345         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3346
3347         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3348                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3349                                 bufs->hard_outputs, bufs->soft_outputs,
3350                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3351
3352         /* Set counter to validate the ordering */
3353         for (j = 0; j < num_ops; ++j)
3354                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3355
3356         for (i = 0; i < TEST_REPETITIONS; ++i) {
3357                 for (j = 0; j < num_ops; ++j) {
3358                         if (!loopback)
3359                                 mbuf_reset(
3360                                 ops_enq[j]->ldpc_dec.hard_output.data);
3361                         if (hc_out || loopback)
3362                                 mbuf_reset(
3363                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3364                 }
3365                 if (extDdr) {
3366                         bool preload = i == (TEST_REPETITIONS - 1);
3367                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3368                                         num_ops, preload);
3369                 }
3370                 start_time = rte_rdtsc_precise();
3371
3372                 for (enq = 0, deq = 0; enq < num_ops;) {
3373                         num_to_enq = burst_sz;
3374
3375                         if (unlikely(num_ops - enq < num_to_enq))
3376                                 num_to_enq = num_ops - enq;
3377
3378                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3379                                         queue_id, &ops_enq[enq], num_to_enq);
3380
3381                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3382                                         queue_id, &ops_deq[deq], enq - deq);
3383                 }
3384
3385                 /* dequeue the remaining */
3386                 while (deq < enq) {
3387                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3388                                         queue_id, &ops_deq[deq], enq - deq);
3389                 }
3390
3391                 total_time += rte_rdtsc_precise() - start_time;
3392         }
3393
3394         tp->iter_count = 0;
3395         /* get the max of iter_count for all dequeued ops */
3396         for (i = 0; i < num_ops; ++i) {
3397                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3398                                 tp->iter_count);
3399         }
3400         if (extDdr) {
3401                 /* Read loopback is not thread safe */
3402                 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3403         }
3404
3405         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3406                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3407                                 tp->op_params->vector_mask);
3408                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3409         }
3410
3411         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3412
3413         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3414
3415         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3416                         ((double)total_time / (double)rte_get_tsc_hz());
3417         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3418                         1000000.0) / ((double)total_time /
3419                         (double)rte_get_tsc_hz());
3420
3421         return TEST_SUCCESS;
3422 }
3423
3424 static int
3425 throughput_pmd_lcore_enc(void *arg)
3426 {
3427         struct thread_params *tp = arg;
3428         uint16_t enq, deq;
3429         uint64_t total_time = 0, start_time;
3430         const uint16_t queue_id = tp->queue_id;
3431         const uint16_t burst_sz = tp->op_params->burst_sz;
3432         const uint16_t num_ops = tp->op_params->num_to_process;
3433         struct rte_bbdev_enc_op *ops_enq[num_ops];
3434         struct rte_bbdev_enc_op *ops_deq[num_ops];
3435         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3436         struct test_buffers *bufs = NULL;
3437         int i, j, ret;
3438         struct rte_bbdev_info info;
3439         uint16_t num_to_enq;
3440
3441         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3442                         "BURST_SIZE should be <= %u", MAX_BURST);
3443
3444         rte_bbdev_info_get(tp->dev_id, &info);
3445
3446         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3447                         "NUM_OPS cannot exceed %u for this device",
3448                         info.drv.queue_size_lim);
3449
3450         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3451
3452         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3453                 rte_pause();
3454
3455         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3456                         num_ops);
3457         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3458                         num_ops);
3459         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3460                 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3461                                 bufs->hard_outputs, ref_op);
3462
3463         /* Set counter to validate the ordering */
3464         for (j = 0; j < num_ops; ++j)
3465                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3466
3467         for (i = 0; i < TEST_REPETITIONS; ++i) {
3468
3469                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3470                         for (j = 0; j < num_ops; ++j)
3471                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3472
3473                 start_time = rte_rdtsc_precise();
3474
3475                 for (enq = 0, deq = 0; enq < num_ops;) {
3476                         num_to_enq = burst_sz;
3477
3478                         if (unlikely(num_ops - enq < num_to_enq))
3479                                 num_to_enq = num_ops - enq;
3480
3481                         enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3482                                         queue_id, &ops_enq[enq], num_to_enq);
3483
3484                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3485                                         queue_id, &ops_deq[deq], enq - deq);
3486                 }
3487
3488                 /* dequeue the remaining */
3489                 while (deq < enq) {
3490                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3491                                         queue_id, &ops_deq[deq], enq - deq);
3492                 }
3493
3494                 total_time += rte_rdtsc_precise() - start_time;
3495         }
3496
3497         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3498                 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3499                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3500         }
3501
3502         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3503
3504         double tb_len_bits = calc_enc_TB_size(ref_op);
3505
3506         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3507                         ((double)total_time / (double)rte_get_tsc_hz());
3508         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3509                         / 1000000.0) / ((double)total_time /
3510                         (double)rte_get_tsc_hz());
3511
3512         return TEST_SUCCESS;
3513 }
3514
3515 static int
3516 throughput_pmd_lcore_ldpc_enc(void *arg)
3517 {
3518         struct thread_params *tp = arg;
3519         uint16_t enq, deq;
3520         uint64_t total_time = 0, start_time;
3521         const uint16_t queue_id = tp->queue_id;
3522         const uint16_t burst_sz = tp->op_params->burst_sz;
3523         const uint16_t num_ops = tp->op_params->num_to_process;
3524         struct rte_bbdev_enc_op *ops_enq[num_ops];
3525         struct rte_bbdev_enc_op *ops_deq[num_ops];
3526         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3527         struct test_buffers *bufs = NULL;
3528         int i, j, ret;
3529         struct rte_bbdev_info info;
3530         uint16_t num_to_enq;
3531
3532         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3533                         "BURST_SIZE should be <= %u", MAX_BURST);
3534
3535         rte_bbdev_info_get(tp->dev_id, &info);
3536
3537         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3538                         "NUM_OPS cannot exceed %u for this device",
3539                         info.drv.queue_size_lim);
3540
3541         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3542
3543         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3544                 rte_pause();
3545
3546         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3547                         num_ops);
3548         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3549                         num_ops);
3550         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3551                 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3552                                 bufs->hard_outputs, ref_op);
3553
3554         /* Set counter to validate the ordering */
3555         for (j = 0; j < num_ops; ++j)
3556                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3557
3558         for (i = 0; i < TEST_REPETITIONS; ++i) {
3559
3560                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3561                         for (j = 0; j < num_ops; ++j)
3562                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3563
3564                 start_time = rte_rdtsc_precise();
3565
3566                 for (enq = 0, deq = 0; enq < num_ops;) {
3567                         num_to_enq = burst_sz;
3568
3569                         if (unlikely(num_ops - enq < num_to_enq))
3570                                 num_to_enq = num_ops - enq;
3571
3572                         enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3573                                         queue_id, &ops_enq[enq], num_to_enq);
3574
3575                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3576                                         queue_id, &ops_deq[deq], enq - deq);
3577                 }
3578
3579                 /* dequeue the remaining */
3580                 while (deq < enq) {
3581                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3582                                         queue_id, &ops_deq[deq], enq - deq);
3583                 }
3584
3585                 total_time += rte_rdtsc_precise() - start_time;
3586         }
3587
3588         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3589                 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3590                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3591         }
3592
3593         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3594
3595         double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3596
3597         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3598                         ((double)total_time / (double)rte_get_tsc_hz());
3599         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3600                         / 1000000.0) / ((double)total_time /
3601                         (double)rte_get_tsc_hz());
3602
3603         return TEST_SUCCESS;
3604 }
3605
3606 static void
3607 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3608 {
3609         unsigned int iter = 0;
3610         double total_mops = 0, total_mbps = 0;
3611
3612         for (iter = 0; iter < used_cores; iter++) {
3613                 printf(
3614                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3615                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3616                         t_params[iter].mbps);
3617                 total_mops += t_params[iter].ops_per_sec;
3618                 total_mbps += t_params[iter].mbps;
3619         }
3620         printf(
3621                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3622                 used_cores, total_mops, total_mbps);
3623 }
3624
3625 /* Aggregate the performance results over the number of cores used */
3626 static void
3627 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3628 {
3629         unsigned int core_idx = 0;
3630         double total_mops = 0, total_mbps = 0;
3631         uint8_t iter_count = 0;
3632
3633         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3634                 printf(
3635                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3636                         t_params[core_idx].lcore_id,
3637                         t_params[core_idx].ops_per_sec,
3638                         t_params[core_idx].mbps,
3639                         t_params[core_idx].iter_count);
3640                 total_mops += t_params[core_idx].ops_per_sec;
3641                 total_mbps += t_params[core_idx].mbps;
3642                 iter_count = RTE_MAX(iter_count,
3643                                 t_params[core_idx].iter_count);
3644         }
3645         printf(
3646                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3647                 used_cores, total_mops, total_mbps, iter_count);
3648 }
3649
3650 /* Aggregate the performance results over the number of cores used */
3651 static void
3652 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3653 {
3654         unsigned int core_idx = 0;
3655         double total_mbps = 0, total_bler = 0, total_iter = 0;
3656         double snr = get_snr();
3657
3658         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3659                 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3660                                 t_params[core_idx].lcore_id,
3661                                 t_params[core_idx].bler * 100,
3662                                 t_params[core_idx].iter_average,
3663                                 t_params[core_idx].mbps,
3664                                 get_vector_filename());
3665                 total_mbps += t_params[core_idx].mbps;
3666                 total_bler += t_params[core_idx].bler;
3667                 total_iter += t_params[core_idx].iter_average;
3668         }
3669         total_bler /= used_cores;
3670         total_iter /= used_cores;
3671
3672         printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3673                         snr, total_bler * 100, total_iter, get_iter_max(),
3674                         total_mbps, get_vector_filename());
3675 }
3676
3677 /*
3678  * Test function that determines BLER wireless performance
3679  */
3680 static int
3681 bler_test(struct active_device *ad,
3682                 struct test_op_params *op_params)
3683 {
3684         int ret;
3685         unsigned int lcore_id, used_cores = 0;
3686         struct thread_params *t_params;
3687         struct rte_bbdev_info info;
3688         lcore_function_t *bler_function;
3689         uint16_t num_lcores;
3690         const char *op_type_str;
3691
3692         rte_bbdev_info_get(ad->dev_id, &info);
3693
3694         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3695         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3696                         test_vector.op_type);
3697
3698         printf("+ ------------------------------------------------------- +\n");
3699         printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3700                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3701                         op_params->num_to_process, op_params->num_lcores,
3702                         op_type_str,
3703                         intr_enabled ? "Interrupt mode" : "PMD mode",
3704                         (double)rte_get_tsc_hz() / 1000000000.0);
3705
3706         /* Set number of lcores */
3707         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3708                         ? ad->nb_queues
3709                         : op_params->num_lcores;
3710
3711         /* Allocate memory for thread parameters structure */
3712         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3713                         RTE_CACHE_LINE_SIZE);
3714         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3715                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3716                                 RTE_CACHE_LINE_SIZE));
3717
3718         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3719                 bler_function = bler_pmd_lcore_ldpc_dec;
3720         else
3721                 return TEST_SKIPPED;
3722
3723         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3724
3725         /* Main core is set at first entry */
3726         t_params[0].dev_id = ad->dev_id;
3727         t_params[0].lcore_id = rte_lcore_id();
3728         t_params[0].op_params = op_params;
3729         t_params[0].queue_id = ad->queue_ids[used_cores++];
3730         t_params[0].iter_count = 0;
3731
3732         RTE_LCORE_FOREACH_WORKER(lcore_id) {
3733                 if (used_cores >= num_lcores)
3734                         break;
3735
3736                 t_params[used_cores].dev_id = ad->dev_id;
3737                 t_params[used_cores].lcore_id = lcore_id;
3738                 t_params[used_cores].op_params = op_params;
3739                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3740                 t_params[used_cores].iter_count = 0;
3741
3742                 rte_eal_remote_launch(bler_function,
3743                                 &t_params[used_cores++], lcore_id);
3744         }
3745
3746         rte_atomic16_set(&op_params->sync, SYNC_START);
3747         ret = bler_function(&t_params[0]);
3748
3749         /* Main core is always used */
3750         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3751                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3752
3753         print_dec_bler(t_params, num_lcores);
3754
3755         /* Return if test failed */
3756         if (ret) {
3757                 rte_free(t_params);
3758                 return ret;
3759         }
3760
3761         /* Function to print something  here*/
3762         rte_free(t_params);
3763         return ret;
3764 }
3765
3766 /*
3767  * Test function that determines how long an enqueue + dequeue of a burst
3768  * takes on available lcores.
3769  */
3770 static int
3771 throughput_test(struct active_device *ad,
3772                 struct test_op_params *op_params)
3773 {
3774         int ret;
3775         unsigned int lcore_id, used_cores = 0;
3776         struct thread_params *t_params, *tp;
3777         struct rte_bbdev_info info;
3778         lcore_function_t *throughput_function;
3779         uint16_t num_lcores;
3780         const char *op_type_str;
3781
3782         rte_bbdev_info_get(ad->dev_id, &info);
3783
3784         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3785         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3786                         test_vector.op_type);
3787
3788         printf("+ ------------------------------------------------------- +\n");
3789         printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3790                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3791                         op_params->num_to_process, op_params->num_lcores,
3792                         op_type_str,
3793                         intr_enabled ? "Interrupt mode" : "PMD mode",
3794                         (double)rte_get_tsc_hz() / 1000000000.0);
3795
3796         /* Set number of lcores */
3797         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3798                         ? ad->nb_queues
3799                         : op_params->num_lcores;
3800
3801         /* Allocate memory for thread parameters structure */
3802         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3803                         RTE_CACHE_LINE_SIZE);
3804         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3805                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3806                                 RTE_CACHE_LINE_SIZE));
3807
3808         if (intr_enabled) {
3809                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3810                         throughput_function = throughput_intr_lcore_dec;
3811                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3812                         throughput_function = throughput_intr_lcore_ldpc_dec;
3813                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3814                         throughput_function = throughput_intr_lcore_enc;
3815                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3816                         throughput_function = throughput_intr_lcore_ldpc_enc;
3817                 else
3818                         throughput_function = throughput_intr_lcore_enc;
3819
3820                 /* Dequeue interrupt callback registration */
3821                 ret = rte_bbdev_callback_register(ad->dev_id,
3822                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3823                                 t_params);
3824                 if (ret < 0) {
3825                         rte_free(t_params);
3826                         return ret;
3827                 }
3828         } else {
3829                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3830                         throughput_function = throughput_pmd_lcore_dec;
3831                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3832                         throughput_function = throughput_pmd_lcore_ldpc_dec;
3833                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3834                         throughput_function = throughput_pmd_lcore_enc;
3835                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3836                         throughput_function = throughput_pmd_lcore_ldpc_enc;
3837                 else
3838                         throughput_function = throughput_pmd_lcore_enc;
3839         }
3840
3841         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3842
3843         /* Main core is set at first entry */
3844         t_params[0].dev_id = ad->dev_id;
3845         t_params[0].lcore_id = rte_lcore_id();
3846         t_params[0].op_params = op_params;
3847         t_params[0].queue_id = ad->queue_ids[used_cores++];
3848         t_params[0].iter_count = 0;
3849
3850         RTE_LCORE_FOREACH_WORKER(lcore_id) {
3851                 if (used_cores >= num_lcores)
3852                         break;
3853
3854                 t_params[used_cores].dev_id = ad->dev_id;
3855                 t_params[used_cores].lcore_id = lcore_id;
3856                 t_params[used_cores].op_params = op_params;
3857                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3858                 t_params[used_cores].iter_count = 0;
3859
3860                 rte_eal_remote_launch(throughput_function,
3861                                 &t_params[used_cores++], lcore_id);
3862         }
3863
3864         rte_atomic16_set(&op_params->sync, SYNC_START);
3865         ret = throughput_function(&t_params[0]);
3866
3867         /* Main core is always used */
3868         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3869                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3870
3871         /* Return if test failed */
3872         if (ret) {
3873                 rte_free(t_params);
3874                 return ret;
3875         }
3876
3877         /* Print throughput if interrupts are disabled and test passed */
3878         if (!intr_enabled) {
3879                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3880                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3881                         print_dec_throughput(t_params, num_lcores);
3882                 else
3883                         print_enc_throughput(t_params, num_lcores);
3884                 rte_free(t_params);
3885                 return ret;
3886         }
3887
3888         /* In interrupt TC we need to wait for the interrupt callback to deqeue
3889          * all pending operations. Skip waiting for queues which reported an
3890          * error using processing_status variable.
3891          * Wait for main lcore operations.
3892          */
3893         tp = &t_params[0];
3894         while ((rte_atomic16_read(&tp->nb_dequeued) <
3895                         op_params->num_to_process) &&
3896                         (rte_atomic16_read(&tp->processing_status) !=
3897                         TEST_FAILED))
3898                 rte_pause();
3899
3900         tp->ops_per_sec /= TEST_REPETITIONS;
3901         tp->mbps /= TEST_REPETITIONS;
3902         ret |= (int)rte_atomic16_read(&tp->processing_status);
3903
3904         /* Wait for worker lcores operations */
3905         for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3906                 tp = &t_params[used_cores];
3907
3908                 while ((rte_atomic16_read(&tp->nb_dequeued) <
3909                                 op_params->num_to_process) &&
3910                                 (rte_atomic16_read(&tp->processing_status) !=
3911                                 TEST_FAILED))
3912                         rte_pause();
3913
3914                 tp->ops_per_sec /= TEST_REPETITIONS;
3915                 tp->mbps /= TEST_REPETITIONS;
3916                 ret |= (int)rte_atomic16_read(&tp->processing_status);
3917         }
3918
3919         /* Print throughput if test passed */
3920         if (!ret) {
3921                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3922                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3923                         print_dec_throughput(t_params, num_lcores);
3924                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3925                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3926                         print_enc_throughput(t_params, num_lcores);
3927         }
3928
3929         rte_free(t_params);
3930         return ret;
3931 }
3932
3933 static int
3934 latency_test_dec(struct rte_mempool *mempool,
3935                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3936                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3937                 const uint16_t num_to_process, uint16_t burst_sz,
3938                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3939 {
3940         int ret = TEST_SUCCESS;
3941         uint16_t i, j, dequeued;
3942         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3943         uint64_t start_time = 0, last_time = 0;
3944
3945         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3946                 uint16_t enq = 0, deq = 0;
3947                 bool first_time = true;
3948                 last_time = 0;
3949
3950                 if (unlikely(num_to_process - dequeued < burst_sz))
3951                         burst_sz = num_to_process - dequeued;
3952
3953                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3954                 TEST_ASSERT_SUCCESS(ret,
3955                                 "rte_bbdev_dec_op_alloc_bulk() failed");
3956                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3957                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3958                                         bufs->inputs,
3959                                         bufs->hard_outputs,
3960                                         bufs->soft_outputs,
3961                                         ref_op);
3962
3963                 /* Set counter to validate the ordering */
3964                 for (j = 0; j < burst_sz; ++j)
3965                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3966
3967                 start_time = rte_rdtsc_precise();
3968
3969                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
3970                                 burst_sz);
3971                 TEST_ASSERT(enq == burst_sz,
3972                                 "Error enqueueing burst, expected %u, got %u",
3973                                 burst_sz, enq);
3974
3975                 /* Dequeue */
3976                 do {
3977                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3978                                         &ops_deq[deq], burst_sz - deq);
3979                         if (likely(first_time && (deq > 0))) {
3980                                 last_time = rte_rdtsc_precise() - start_time;
3981                                 first_time = false;
3982                         }
3983                 } while (unlikely(burst_sz != deq));
3984
3985                 *max_time = RTE_MAX(*max_time, last_time);
3986                 *min_time = RTE_MIN(*min_time, last_time);
3987                 *total_time += last_time;
3988
3989                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3990                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
3991                                         vector_mask);
3992                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3993                 }
3994
3995                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3996                 dequeued += deq;
3997         }
3998
3999         return i;
4000 }
4001
4002 static int
4003 latency_test_ldpc_dec(struct rte_mempool *mempool,
4004                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
4005                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
4006                 const uint16_t num_to_process, uint16_t burst_sz,
4007                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4008 {
4009         int ret = TEST_SUCCESS;
4010         uint16_t i, j, dequeued;
4011         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4012         uint64_t start_time = 0, last_time = 0;
4013         bool extDdr = ldpc_cap_flags &
4014                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4015
4016         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4017                 uint16_t enq = 0, deq = 0;
4018                 bool first_time = true;
4019                 last_time = 0;
4020
4021                 if (unlikely(num_to_process - dequeued < burst_sz))
4022                         burst_sz = num_to_process - dequeued;
4023
4024                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4025                 TEST_ASSERT_SUCCESS(ret,
4026                                 "rte_bbdev_dec_op_alloc_bulk() failed");
4027
4028                 /* For latency tests we need to disable early termination */
4029                 if (check_bit(ref_op->ldpc_dec.op_flags,
4030                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
4031                         ref_op->ldpc_dec.op_flags -=
4032                                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
4033                 ref_op->ldpc_dec.iter_max = get_iter_max();
4034                 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
4035
4036                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4037                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4038                                         bufs->inputs,
4039                                         bufs->hard_outputs,
4040                                         bufs->soft_outputs,
4041                                         bufs->harq_inputs,
4042                                         bufs->harq_outputs,
4043                                         ref_op);
4044
4045                 if (extDdr)
4046                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4047                                         burst_sz, true);
4048
4049                 /* Set counter to validate the ordering */
4050                 for (j = 0; j < burst_sz; ++j)
4051                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4052
4053                 start_time = rte_rdtsc_precise();
4054
4055                 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4056                                 &ops_enq[enq], burst_sz);
4057                 TEST_ASSERT(enq == burst_sz,
4058                                 "Error enqueueing burst, expected %u, got %u",
4059                                 burst_sz, enq);
4060
4061                 /* Dequeue */
4062                 do {
4063                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4064                                         &ops_deq[deq], burst_sz - deq);
4065                         if (likely(first_time && (deq > 0))) {
4066                                 last_time = rte_rdtsc_precise() - start_time;
4067                                 first_time = false;
4068                         }
4069                 } while (unlikely(burst_sz != deq));
4070
4071                 *max_time = RTE_MAX(*max_time, last_time);
4072                 *min_time = RTE_MIN(*min_time, last_time);
4073                 *total_time += last_time;
4074
4075                 if (extDdr)
4076                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4077
4078                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4079                         ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4080                                         vector_mask);
4081                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4082                 }
4083
4084                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4085                 dequeued += deq;
4086         }
4087         return i;
4088 }
4089
4090 static int
4091 latency_test_enc(struct rte_mempool *mempool,
4092                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4093                 uint16_t dev_id, uint16_t queue_id,
4094                 const uint16_t num_to_process, uint16_t burst_sz,
4095                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4096 {
4097         int ret = TEST_SUCCESS;
4098         uint16_t i, j, dequeued;
4099         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4100         uint64_t start_time = 0, last_time = 0;
4101
4102         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4103                 uint16_t enq = 0, deq = 0;
4104                 bool first_time = true;
4105                 last_time = 0;
4106
4107                 if (unlikely(num_to_process - dequeued < burst_sz))
4108                         burst_sz = num_to_process - dequeued;
4109
4110                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4111                 TEST_ASSERT_SUCCESS(ret,
4112                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4113                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4114                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4115                                         bufs->inputs,
4116                                         bufs->hard_outputs,
4117                                         ref_op);
4118
4119                 /* Set counter to validate the ordering */
4120                 for (j = 0; j < burst_sz; ++j)
4121                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4122
4123                 start_time = rte_rdtsc_precise();
4124
4125                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4126                                 burst_sz);
4127                 TEST_ASSERT(enq == burst_sz,
4128                                 "Error enqueueing burst, expected %u, got %u",
4129                                 burst_sz, enq);
4130
4131                 /* Dequeue */
4132                 do {
4133                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4134                                         &ops_deq[deq], burst_sz - deq);
4135                         if (likely(first_time && (deq > 0))) {
4136                                 last_time += rte_rdtsc_precise() - start_time;
4137                                 first_time = false;
4138                         }
4139                 } while (unlikely(burst_sz != deq));
4140
4141                 *max_time = RTE_MAX(*max_time, last_time);
4142                 *min_time = RTE_MIN(*min_time, last_time);
4143                 *total_time += last_time;
4144
4145                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4146                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4147                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4148                 }
4149
4150                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4151                 dequeued += deq;
4152         }
4153
4154         return i;
4155 }
4156
4157 static int
4158 latency_test_ldpc_enc(struct rte_mempool *mempool,
4159                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4160                 uint16_t dev_id, uint16_t queue_id,
4161                 const uint16_t num_to_process, uint16_t burst_sz,
4162                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4163 {
4164         int ret = TEST_SUCCESS;
4165         uint16_t i, j, dequeued;
4166         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4167         uint64_t start_time = 0, last_time = 0;
4168
4169         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4170                 uint16_t enq = 0, deq = 0;
4171                 bool first_time = true;
4172                 last_time = 0;
4173
4174                 if (unlikely(num_to_process - dequeued < burst_sz))
4175                         burst_sz = num_to_process - dequeued;
4176
4177                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4178                 TEST_ASSERT_SUCCESS(ret,
4179                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4180                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4181                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4182                                         bufs->inputs,
4183                                         bufs->hard_outputs,
4184                                         ref_op);
4185
4186                 /* Set counter to validate the ordering */
4187                 for (j = 0; j < burst_sz; ++j)
4188                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4189
4190                 start_time = rte_rdtsc_precise();
4191
4192                 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4193                                 &ops_enq[enq], burst_sz);
4194                 TEST_ASSERT(enq == burst_sz,
4195                                 "Error enqueueing burst, expected %u, got %u",
4196                                 burst_sz, enq);
4197
4198                 /* Dequeue */
4199                 do {
4200                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4201                                         &ops_deq[deq], burst_sz - deq);
4202                         if (likely(first_time && (deq > 0))) {
4203                                 last_time += rte_rdtsc_precise() - start_time;
4204                                 first_time = false;
4205                         }
4206                 } while (unlikely(burst_sz != deq));
4207
4208                 *max_time = RTE_MAX(*max_time, last_time);
4209                 *min_time = RTE_MIN(*min_time, last_time);
4210                 *total_time += last_time;
4211
4212                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4213                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4214                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4215                 }
4216
4217                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4218                 dequeued += deq;
4219         }
4220
4221         return i;
4222 }
4223
4224 static int
4225 latency_test(struct active_device *ad,
4226                 struct test_op_params *op_params)
4227 {
4228         int iter;
4229         uint16_t burst_sz = op_params->burst_sz;
4230         const uint16_t num_to_process = op_params->num_to_process;
4231         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4232         const uint16_t queue_id = ad->queue_ids[0];
4233         struct test_buffers *bufs = NULL;
4234         struct rte_bbdev_info info;
4235         uint64_t total_time, min_time, max_time;
4236         const char *op_type_str;
4237
4238         total_time = max_time = 0;
4239         min_time = UINT64_MAX;
4240
4241         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4242                         "BURST_SIZE should be <= %u", MAX_BURST);
4243
4244         rte_bbdev_info_get(ad->dev_id, &info);
4245         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4246
4247         op_type_str = rte_bbdev_op_type_str(op_type);
4248         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4249
4250         printf("+ ------------------------------------------------------- +\n");
4251         printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4252                         info.dev_name, burst_sz, num_to_process, op_type_str);
4253
4254         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4255                 iter = latency_test_dec(op_params->mp, bufs,
4256                                 op_params->ref_dec_op, op_params->vector_mask,
4257                                 ad->dev_id, queue_id, num_to_process,
4258                                 burst_sz, &total_time, &min_time, &max_time);
4259         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4260                 iter = latency_test_enc(op_params->mp, bufs,
4261                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4262                                 num_to_process, burst_sz, &total_time,
4263                                 &min_time, &max_time);
4264         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4265                 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4266                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4267                                 num_to_process, burst_sz, &total_time,
4268                                 &min_time, &max_time);
4269         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4270                 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4271                                 op_params->ref_dec_op, op_params->vector_mask,
4272                                 ad->dev_id, queue_id, num_to_process,
4273                                 burst_sz, &total_time, &min_time, &max_time);
4274         else
4275                 iter = latency_test_enc(op_params->mp, bufs,
4276                                         op_params->ref_enc_op,
4277                                         ad->dev_id, queue_id,
4278                                         num_to_process, burst_sz, &total_time,
4279                                         &min_time, &max_time);
4280
4281         if (iter <= 0)
4282                 return TEST_FAILED;
4283
4284         printf("Operation latency:\n"
4285                         "\tavg: %lg cycles, %lg us\n"
4286                         "\tmin: %lg cycles, %lg us\n"
4287                         "\tmax: %lg cycles, %lg us\n",
4288                         (double)total_time / (double)iter,
4289                         (double)(total_time * 1000000) / (double)iter /
4290                         (double)rte_get_tsc_hz(), (double)min_time,
4291                         (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4292                         (double)max_time, (double)(max_time * 1000000) /
4293                         (double)rte_get_tsc_hz());
4294
4295         return TEST_SUCCESS;
4296 }
4297
4298 #ifdef RTE_BBDEV_OFFLOAD_COST
4299 static int
4300 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4301                 struct rte_bbdev_stats *stats)
4302 {
4303         struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4304         struct rte_bbdev_stats *q_stats;
4305
4306         if (queue_id >= dev->data->num_queues)
4307                 return -1;
4308
4309         q_stats = &dev->data->queues[queue_id].queue_stats;
4310
4311         stats->enqueued_count = q_stats->enqueued_count;
4312         stats->dequeued_count = q_stats->dequeued_count;
4313         stats->enqueue_err_count = q_stats->enqueue_err_count;
4314         stats->dequeue_err_count = q_stats->dequeue_err_count;
4315         stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4316
4317         return 0;
4318 }
4319
4320 static int
4321 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4322                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4323                 uint16_t queue_id, const uint16_t num_to_process,
4324                 uint16_t burst_sz, struct test_time_stats *time_st)
4325 {
4326         int i, dequeued, ret;
4327         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4328         uint64_t enq_start_time, deq_start_time;
4329         uint64_t enq_sw_last_time, deq_last_time;
4330         struct rte_bbdev_stats stats;
4331
4332         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4333                 uint16_t enq = 0, deq = 0;
4334
4335                 if (unlikely(num_to_process - dequeued < burst_sz))
4336                         burst_sz = num_to_process - dequeued;
4337
4338                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4339                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4340                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4341                                         bufs->inputs,
4342                                         bufs->hard_outputs,
4343                                         bufs->soft_outputs,
4344                                         ref_op);
4345
4346                 /* Start time meas for enqueue function offload latency */
4347                 enq_start_time = rte_rdtsc_precise();
4348                 do {
4349                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4350                                         &ops_enq[enq], burst_sz - enq);
4351                 } while (unlikely(burst_sz != enq));
4352
4353                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4354                 TEST_ASSERT_SUCCESS(ret,
4355                                 "Failed to get stats for queue (%u) of device (%u)",
4356                                 queue_id, dev_id);
4357
4358                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4359                                 stats.acc_offload_cycles;
4360                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4361                                 enq_sw_last_time);
4362                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4363                                 enq_sw_last_time);
4364                 time_st->enq_sw_total_time += enq_sw_last_time;
4365
4366                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4367                                 stats.acc_offload_cycles);
4368                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4369                                 stats.acc_offload_cycles);
4370                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4371
4372                 /* give time for device to process ops */
4373                 rte_delay_us(200);
4374
4375                 /* Start time meas for dequeue function offload latency */
4376                 deq_start_time = rte_rdtsc_precise();
4377                 /* Dequeue one operation */
4378                 do {
4379                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4380                                         &ops_deq[deq], 1);
4381                 } while (unlikely(deq != 1));
4382
4383                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4384                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4385                                 deq_last_time);
4386                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4387                                 deq_last_time);
4388                 time_st->deq_total_time += deq_last_time;
4389
4390                 /* Dequeue remaining operations if needed*/
4391                 while (burst_sz != deq)
4392                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4393                                         &ops_deq[deq], burst_sz - deq);
4394
4395                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4396                 dequeued += deq;
4397         }
4398
4399         return i;
4400 }
4401
4402 static int
4403 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4404                 struct test_buffers *bufs,
4405                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4406                 uint16_t queue_id, const uint16_t num_to_process,
4407                 uint16_t burst_sz, struct test_time_stats *time_st)
4408 {
4409         int i, dequeued, ret;
4410         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4411         uint64_t enq_start_time, deq_start_time;
4412         uint64_t enq_sw_last_time, deq_last_time;
4413         struct rte_bbdev_stats stats;
4414         bool extDdr = ldpc_cap_flags &
4415                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4416
4417         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4418                 uint16_t enq = 0, deq = 0;
4419
4420                 if (unlikely(num_to_process - dequeued < burst_sz))
4421                         burst_sz = num_to_process - dequeued;
4422
4423                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4424                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4425                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4426                                         bufs->inputs,
4427                                         bufs->hard_outputs,
4428                                         bufs->soft_outputs,
4429                                         bufs->harq_inputs,
4430                                         bufs->harq_outputs,
4431                                         ref_op);
4432
4433                 if (extDdr)
4434                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4435                                         burst_sz, true);
4436
4437                 /* Start time meas for enqueue function offload latency */
4438                 enq_start_time = rte_rdtsc_precise();
4439                 do {
4440                         enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4441                                         &ops_enq[enq], burst_sz - enq);
4442                 } while (unlikely(burst_sz != enq));
4443
4444                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4445                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4446                 TEST_ASSERT_SUCCESS(ret,
4447                                 "Failed to get stats for queue (%u) of device (%u)",
4448                                 queue_id, dev_id);
4449
4450                 enq_sw_last_time -= stats.acc_offload_cycles;
4451                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4452                                 enq_sw_last_time);
4453                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4454                                 enq_sw_last_time);
4455                 time_st->enq_sw_total_time += enq_sw_last_time;
4456
4457                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4458                                 stats.acc_offload_cycles);
4459                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4460                                 stats.acc_offload_cycles);
4461                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4462
4463                 /* give time for device to process ops */
4464                 rte_delay_us(200);
4465
4466                 /* Start time meas for dequeue function offload latency */
4467                 deq_start_time = rte_rdtsc_precise();
4468                 /* Dequeue one operation */
4469                 do {
4470                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4471                                         &ops_deq[deq], 1);
4472                 } while (unlikely(deq != 1));
4473
4474                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4475                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4476                                 deq_last_time);
4477                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4478                                 deq_last_time);
4479                 time_st->deq_total_time += deq_last_time;
4480
4481                 /* Dequeue remaining operations if needed*/
4482                 while (burst_sz != deq)
4483                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4484                                         &ops_deq[deq], burst_sz - deq);
4485
4486                 if (extDdr) {
4487                         /* Read loopback is not thread safe */
4488                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4489                 }
4490
4491                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4492                 dequeued += deq;
4493         }
4494
4495         return i;
4496 }
4497
4498 static int
4499 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4500                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4501                 uint16_t queue_id, const uint16_t num_to_process,
4502                 uint16_t burst_sz, struct test_time_stats *time_st)
4503 {
4504         int i, dequeued, ret;
4505         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4506         uint64_t enq_start_time, deq_start_time;
4507         uint64_t enq_sw_last_time, deq_last_time;
4508         struct rte_bbdev_stats stats;
4509
4510         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4511                 uint16_t enq = 0, deq = 0;
4512
4513                 if (unlikely(num_to_process - dequeued < burst_sz))
4514                         burst_sz = num_to_process - dequeued;
4515
4516                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4517                 TEST_ASSERT_SUCCESS(ret,
4518                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4519                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4520                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4521                                         bufs->inputs,
4522                                         bufs->hard_outputs,
4523                                         ref_op);
4524
4525                 /* Start time meas for enqueue function offload latency */
4526                 enq_start_time = rte_rdtsc_precise();
4527                 do {
4528                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4529                                         &ops_enq[enq], burst_sz - enq);
4530                 } while (unlikely(burst_sz != enq));
4531
4532                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4533
4534                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4535                 TEST_ASSERT_SUCCESS(ret,
4536                                 "Failed to get stats for queue (%u) of device (%u)",
4537                                 queue_id, dev_id);
4538                 enq_sw_last_time -= stats.acc_offload_cycles;
4539                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4540                                 enq_sw_last_time);
4541                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4542                                 enq_sw_last_time);
4543                 time_st->enq_sw_total_time += enq_sw_last_time;
4544
4545                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4546                                 stats.acc_offload_cycles);
4547                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4548                                 stats.acc_offload_cycles);
4549                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4550
4551                 /* give time for device to process ops */
4552                 rte_delay_us(200);
4553
4554                 /* Start time meas for dequeue function offload latency */
4555                 deq_start_time = rte_rdtsc_precise();
4556                 /* Dequeue one operation */
4557                 do {
4558                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4559                                         &ops_deq[deq], 1);
4560                 } while (unlikely(deq != 1));
4561
4562                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4563                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4564                                 deq_last_time);
4565                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4566                                 deq_last_time);
4567                 time_st->deq_total_time += deq_last_time;
4568
4569                 while (burst_sz != deq)
4570                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4571                                         &ops_deq[deq], burst_sz - deq);
4572
4573                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4574                 dequeued += deq;
4575         }
4576
4577         return i;
4578 }
4579
4580 static int
4581 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4582                 struct test_buffers *bufs,
4583                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4584                 uint16_t queue_id, const uint16_t num_to_process,
4585                 uint16_t burst_sz, struct test_time_stats *time_st)
4586 {
4587         int i, dequeued, ret;
4588         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4589         uint64_t enq_start_time, deq_start_time;
4590         uint64_t enq_sw_last_time, deq_last_time;
4591         struct rte_bbdev_stats stats;
4592
4593         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4594                 uint16_t enq = 0, deq = 0;
4595
4596                 if (unlikely(num_to_process - dequeued < burst_sz))
4597                         burst_sz = num_to_process - dequeued;
4598
4599                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4600                 TEST_ASSERT_SUCCESS(ret,
4601                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4602                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4603                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4604                                         bufs->inputs,
4605                                         bufs->hard_outputs,
4606                                         ref_op);
4607
4608                 /* Start time meas for enqueue function offload latency */
4609                 enq_start_time = rte_rdtsc_precise();
4610                 do {
4611                         enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4612                                         &ops_enq[enq], burst_sz - enq);
4613                 } while (unlikely(burst_sz != enq));
4614
4615                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4616                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4617                 TEST_ASSERT_SUCCESS(ret,
4618                                 "Failed to get stats for queue (%u) of device (%u)",
4619                                 queue_id, dev_id);
4620
4621                 enq_sw_last_time -= stats.acc_offload_cycles;
4622                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4623                                 enq_sw_last_time);
4624                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4625                                 enq_sw_last_time);
4626                 time_st->enq_sw_total_time += enq_sw_last_time;
4627
4628                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4629                                 stats.acc_offload_cycles);
4630                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4631                                 stats.acc_offload_cycles);
4632                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4633
4634                 /* give time for device to process ops */
4635                 rte_delay_us(200);
4636
4637                 /* Start time meas for dequeue function offload latency */
4638                 deq_start_time = rte_rdtsc_precise();
4639                 /* Dequeue one operation */
4640                 do {
4641                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4642                                         &ops_deq[deq], 1);
4643                 } while (unlikely(deq != 1));
4644
4645                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4646                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4647                                 deq_last_time);
4648                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4649                                 deq_last_time);
4650                 time_st->deq_total_time += deq_last_time;
4651
4652                 while (burst_sz != deq)
4653                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4654                                         &ops_deq[deq], burst_sz - deq);
4655
4656                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4657                 dequeued += deq;
4658         }
4659
4660         return i;
4661 }
4662 #endif
4663
4664 static int
4665 offload_cost_test(struct active_device *ad,
4666                 struct test_op_params *op_params)
4667 {
4668 #ifndef RTE_BBDEV_OFFLOAD_COST
4669         RTE_SET_USED(ad);
4670         RTE_SET_USED(op_params);
4671         printf("Offload latency test is disabled.\n");
4672         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4673         return TEST_SKIPPED;
4674 #else
4675         int iter;
4676         uint16_t burst_sz = op_params->burst_sz;
4677         const uint16_t num_to_process = op_params->num_to_process;
4678         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4679         const uint16_t queue_id = ad->queue_ids[0];
4680         struct test_buffers *bufs = NULL;
4681         struct rte_bbdev_info info;
4682         const char *op_type_str;
4683         struct test_time_stats time_st;
4684
4685         memset(&time_st, 0, sizeof(struct test_time_stats));
4686         time_st.enq_sw_min_time = UINT64_MAX;
4687         time_st.enq_acc_min_time = UINT64_MAX;
4688         time_st.deq_min_time = UINT64_MAX;
4689
4690         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4691                         "BURST_SIZE should be <= %u", MAX_BURST);
4692
4693         rte_bbdev_info_get(ad->dev_id, &info);
4694         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4695
4696         op_type_str = rte_bbdev_op_type_str(op_type);
4697         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4698
4699         printf("+ ------------------------------------------------------- +\n");
4700         printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4701                         info.dev_name, burst_sz, num_to_process, op_type_str);
4702
4703         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4704                 iter = offload_latency_test_dec(op_params->mp, bufs,
4705                                 op_params->ref_dec_op, ad->dev_id, queue_id,
4706                                 num_to_process, burst_sz, &time_st);
4707         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4708                 iter = offload_latency_test_enc(op_params->mp, bufs,
4709                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4710                                 num_to_process, burst_sz, &time_st);
4711         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4712                 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4713                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4714                                 num_to_process, burst_sz, &time_st);
4715         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4716                 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4717                         op_params->ref_dec_op, ad->dev_id, queue_id,
4718                         num_to_process, burst_sz, &time_st);
4719         else
4720                 iter = offload_latency_test_enc(op_params->mp, bufs,
4721                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4722                                 num_to_process, burst_sz, &time_st);
4723
4724         if (iter <= 0)
4725                 return TEST_FAILED;
4726
4727         printf("Enqueue driver offload cost latency:\n"
4728                         "\tavg: %lg cycles, %lg us\n"
4729                         "\tmin: %lg cycles, %lg us\n"
4730                         "\tmax: %lg cycles, %lg us\n"
4731                         "Enqueue accelerator offload cost latency:\n"
4732                         "\tavg: %lg cycles, %lg us\n"
4733                         "\tmin: %lg cycles, %lg us\n"
4734                         "\tmax: %lg cycles, %lg us\n",
4735                         (double)time_st.enq_sw_total_time / (double)iter,
4736                         (double)(time_st.enq_sw_total_time * 1000000) /
4737                         (double)iter / (double)rte_get_tsc_hz(),
4738                         (double)time_st.enq_sw_min_time,
4739                         (double)(time_st.enq_sw_min_time * 1000000) /
4740                         rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4741                         (double)(time_st.enq_sw_max_time * 1000000) /
4742                         rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4743                         (double)iter,
4744                         (double)(time_st.enq_acc_total_time * 1000000) /
4745                         (double)iter / (double)rte_get_tsc_hz(),
4746                         (double)time_st.enq_acc_min_time,
4747                         (double)(time_st.enq_acc_min_time * 1000000) /
4748                         rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4749                         (double)(time_st.enq_acc_max_time * 1000000) /
4750                         rte_get_tsc_hz());
4751
4752         printf("Dequeue offload cost latency - one op:\n"
4753                         "\tavg: %lg cycles, %lg us\n"
4754                         "\tmin: %lg cycles, %lg us\n"
4755                         "\tmax: %lg cycles, %lg us\n",
4756                         (double)time_st.deq_total_time / (double)iter,
4757                         (double)(time_st.deq_total_time * 1000000) /
4758                         (double)iter / (double)rte_get_tsc_hz(),
4759                         (double)time_st.deq_min_time,
4760                         (double)(time_st.deq_min_time * 1000000) /
4761                         rte_get_tsc_hz(), (double)time_st.deq_max_time,
4762                         (double)(time_st.deq_max_time * 1000000) /
4763                         rte_get_tsc_hz());
4764
4765         return TEST_SUCCESS;
4766 #endif
4767 }
4768
4769 #ifdef RTE_BBDEV_OFFLOAD_COST
4770 static int
4771 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4772                 const uint16_t num_to_process, uint16_t burst_sz,
4773                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4774                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4775 {
4776         int i, deq_total;
4777         struct rte_bbdev_dec_op *ops[MAX_BURST];
4778         uint64_t deq_start_time, deq_last_time;
4779
4780         /* Test deq offload latency from an empty queue */
4781
4782         for (i = 0, deq_total = 0; deq_total < num_to_process;
4783                         ++i, deq_total += burst_sz) {
4784                 deq_start_time = rte_rdtsc_precise();
4785
4786                 if (unlikely(num_to_process - deq_total < burst_sz))
4787                         burst_sz = num_to_process - deq_total;
4788                 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4789                         rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4790                                         burst_sz);
4791                 else
4792                         rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4793                                         burst_sz);
4794
4795                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4796                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4797                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4798                 *deq_total_time += deq_last_time;
4799         }
4800
4801         return i;
4802 }
4803
4804 static int
4805 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4806                 const uint16_t num_to_process, uint16_t burst_sz,
4807                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4808                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4809 {
4810         int i, deq_total;
4811         struct rte_bbdev_enc_op *ops[MAX_BURST];
4812         uint64_t deq_start_time, deq_last_time;
4813
4814         /* Test deq offload latency from an empty queue */
4815         for (i = 0, deq_total = 0; deq_total < num_to_process;
4816                         ++i, deq_total += burst_sz) {
4817                 deq_start_time = rte_rdtsc_precise();
4818
4819                 if (unlikely(num_to_process - deq_total < burst_sz))
4820                         burst_sz = num_to_process - deq_total;
4821                 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4822                         rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4823                                         burst_sz);
4824                 else
4825                         rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4826                                         burst_sz);
4827
4828                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4829                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4830                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4831                 *deq_total_time += deq_last_time;
4832         }
4833
4834         return i;
4835 }
4836
4837 #endif
4838
4839 static int
4840 offload_latency_empty_q_test(struct active_device *ad,
4841                 struct test_op_params *op_params)
4842 {
4843 #ifndef RTE_BBDEV_OFFLOAD_COST
4844         RTE_SET_USED(ad);
4845         RTE_SET_USED(op_params);
4846         printf("Offload latency empty dequeue test is disabled.\n");
4847         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4848         return TEST_SKIPPED;
4849 #else
4850         int iter;
4851         uint64_t deq_total_time, deq_min_time, deq_max_time;
4852         uint16_t burst_sz = op_params->burst_sz;
4853         const uint16_t num_to_process = op_params->num_to_process;
4854         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4855         const uint16_t queue_id = ad->queue_ids[0];
4856         struct rte_bbdev_info info;
4857         const char *op_type_str;
4858
4859         deq_total_time = deq_max_time = 0;
4860         deq_min_time = UINT64_MAX;
4861
4862         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4863                         "BURST_SIZE should be <= %u", MAX_BURST);
4864
4865         rte_bbdev_info_get(ad->dev_id, &info);
4866
4867         op_type_str = rte_bbdev_op_type_str(op_type);
4868         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4869
4870         printf("+ ------------------------------------------------------- +\n");
4871         printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4872                         info.dev_name, burst_sz, num_to_process, op_type_str);
4873
4874         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4875                         op_type == RTE_BBDEV_OP_LDPC_DEC)
4876                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4877                                 num_to_process, burst_sz, &deq_total_time,
4878                                 &deq_min_time, &deq_max_time, op_type);
4879         else
4880                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4881                                 num_to_process, burst_sz, &deq_total_time,
4882                                 &deq_min_time, &deq_max_time, op_type);
4883
4884         if (iter <= 0)
4885                 return TEST_FAILED;
4886
4887         printf("Empty dequeue offload:\n"
4888                         "\tavg: %lg cycles, %lg us\n"
4889                         "\tmin: %lg cycles, %lg us\n"
4890                         "\tmax: %lg cycles, %lg us\n",
4891                         (double)deq_total_time / (double)iter,
4892                         (double)(deq_total_time * 1000000) / (double)iter /
4893                         (double)rte_get_tsc_hz(), (double)deq_min_time,
4894                         (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4895                         (double)deq_max_time, (double)(deq_max_time * 1000000) /
4896                         rte_get_tsc_hz());
4897
4898         return TEST_SUCCESS;
4899 #endif
4900 }
4901
4902 static int
4903 bler_tc(void)
4904 {
4905         return run_test_case(bler_test);
4906 }
4907
4908 static int
4909 throughput_tc(void)
4910 {
4911         return run_test_case(throughput_test);
4912 }
4913
4914 static int
4915 offload_cost_tc(void)
4916 {
4917         return run_test_case(offload_cost_test);
4918 }
4919
4920 static int
4921 offload_latency_empty_q_tc(void)
4922 {
4923         return run_test_case(offload_latency_empty_q_test);
4924 }
4925
4926 static int
4927 latency_tc(void)
4928 {
4929         return run_test_case(latency_test);
4930 }
4931
4932 static int
4933 interrupt_tc(void)
4934 {
4935         return run_test_case(throughput_test);
4936 }
4937
4938 static struct unit_test_suite bbdev_bler_testsuite = {
4939         .suite_name = "BBdev BLER Tests",
4940         .setup = testsuite_setup,
4941         .teardown = testsuite_teardown,
4942         .unit_test_cases = {
4943                 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
4944                 TEST_CASES_END() /**< NULL terminate unit test array */
4945         }
4946 };
4947
4948 static struct unit_test_suite bbdev_throughput_testsuite = {
4949         .suite_name = "BBdev Throughput Tests",
4950         .setup = testsuite_setup,
4951         .teardown = testsuite_teardown,
4952         .unit_test_cases = {
4953                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
4954                 TEST_CASES_END() /**< NULL terminate unit test array */
4955         }
4956 };
4957
4958 static struct unit_test_suite bbdev_validation_testsuite = {
4959         .suite_name = "BBdev Validation Tests",
4960         .setup = testsuite_setup,
4961         .teardown = testsuite_teardown,
4962         .unit_test_cases = {
4963                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4964                 TEST_CASES_END() /**< NULL terminate unit test array */
4965         }
4966 };
4967
4968 static struct unit_test_suite bbdev_latency_testsuite = {
4969         .suite_name = "BBdev Latency Tests",
4970         .setup = testsuite_setup,
4971         .teardown = testsuite_teardown,
4972         .unit_test_cases = {
4973                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4974                 TEST_CASES_END() /**< NULL terminate unit test array */
4975         }
4976 };
4977
4978 static struct unit_test_suite bbdev_offload_cost_testsuite = {
4979         .suite_name = "BBdev Offload Cost Tests",
4980         .setup = testsuite_setup,
4981         .teardown = testsuite_teardown,
4982         .unit_test_cases = {
4983                 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
4984                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
4985                 TEST_CASES_END() /**< NULL terminate unit test array */
4986         }
4987 };
4988
4989 static struct unit_test_suite bbdev_interrupt_testsuite = {
4990         .suite_name = "BBdev Interrupt Tests",
4991         .setup = interrupt_testsuite_setup,
4992         .teardown = testsuite_teardown,
4993         .unit_test_cases = {
4994                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
4995                 TEST_CASES_END() /**< NULL terminate unit test array */
4996         }
4997 };
4998
4999 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
5000 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
5001 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
5002 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
5003 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
5004 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);