app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19 #include <rte_interrupts.h>
  20
  21 #include "main.h"
  22 #include "test_bbdev_vector.h"
  23
  24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  25
  26 #define MAX_QUEUES RTE_MAX_LCORE
  27 #define TEST_REPETITIONS 100
  28 #define WAIT_OFFLOAD_US 1000
  29
  30 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
  31 #include <fpga_lte_fec.h>
  32 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
  33 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
  34 #define VF_UL_4G_QUEUE_VALUE 4
  35 #define VF_DL_4G_QUEUE_VALUE 4
  36 #define UL_4G_BANDWIDTH 3
  37 #define DL_4G_BANDWIDTH 3
  38 #define UL_4G_LOAD_BALANCE 128
  39 #define DL_4G_LOAD_BALANCE 128
  40 #define FLR_4G_TIMEOUT 610
  41 #endif
  42
  43 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
  44 #include <rte_pmd_fpga_5gnr_fec.h>
  45 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
  46 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
  47 #define VF_UL_5G_QUEUE_VALUE 4
  48 #define VF_DL_5G_QUEUE_VALUE 4
  49 #define UL_5G_BANDWIDTH 3
  50 #define DL_5G_BANDWIDTH 3
  51 #define UL_5G_LOAD_BALANCE 128
  52 #define DL_5G_LOAD_BALANCE 128
  53 #define FLR_5G_TIMEOUT 610
  54 #endif
  55
  56 #ifdef RTE_BASEBAND_ACC100
  57 #include <rte_acc100_cfg.h>
  58 #define ACC100PF_DRIVER_NAME   ("intel_acc100_pf")
  59 #define ACC100VF_DRIVER_NAME   ("intel_acc100_vf")
  60 #define ACC100_QMGR_NUM_AQS 16
  61 #define ACC100_QMGR_NUM_QGS 2
  62 #define ACC100_QMGR_AQ_DEPTH 5
  63 #define ACC100_QMGR_INVALID_IDX -1
  64 #define ACC100_QMGR_RR 1
  65 #define ACC100_QOS_GBR 0
  66 #endif
  67
  68 #define OPS_CACHE_SIZE 256U
  69 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  70
  71 #define SYNC_WAIT 0
  72 #define SYNC_START 1
  73 #define INVALID_OPAQUE -1
  74
  75 #define INVALID_QUEUE_ID -1
  76 /* Increment for next code block in external HARQ memory */
  77 #define HARQ_INCR 32768
  78 /* Headroom for filler LLRs insertion in HARQ buffer */
  79 #define FILLER_HEADROOM 1024
  80 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
  81 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
  82 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
  83 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
  84 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
  85 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
  86 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
  87 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
  88 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
  89
  90 static struct test_bbdev_vector test_vector;
  91
  92 /* Switch between PMD and Interrupt for throughput TC */
  93 static bool intr_enabled;
  94
  95 /* LLR arithmetic representation for numerical conversion */
  96 static int ldpc_llr_decimals;
  97 static int ldpc_llr_size;
  98 /* Keep track of the LDPC decoder device capability flag */
  99 static uint32_t ldpc_cap_flags;
 100
 101 /* Represents tested active devices */
 102 static struct active_device {
 103         const char *driver_name;
 104         uint8_t dev_id;
 105         uint16_t supported_ops;
 106         uint16_t queue_ids[MAX_QUEUES];
 107         uint16_t nb_queues;
 108         struct rte_mempool *ops_mempool;
 109         struct rte_mempool *in_mbuf_pool;
 110         struct rte_mempool *hard_out_mbuf_pool;
 111         struct rte_mempool *soft_out_mbuf_pool;
 112         struct rte_mempool *harq_in_mbuf_pool;
 113         struct rte_mempool *harq_out_mbuf_pool;
 114 } active_devs[RTE_BBDEV_MAX_DEVS];
 115
 116 static uint8_t nb_active_devs;
 117
 118 /* Data buffers used by BBDEV ops */
 119 struct test_buffers {
 120         struct rte_bbdev_op_data *inputs;
 121         struct rte_bbdev_op_data *hard_outputs;
 122         struct rte_bbdev_op_data *soft_outputs;
 123         struct rte_bbdev_op_data *harq_inputs;
 124         struct rte_bbdev_op_data *harq_outputs;
 125 };
 126
 127 /* Operation parameters specific for given test case */
 128 struct test_op_params {
 129         struct rte_mempool *mp;
 130         struct rte_bbdev_dec_op *ref_dec_op;
 131         struct rte_bbdev_enc_op *ref_enc_op;
 132         uint16_t burst_sz;
 133         uint16_t num_to_process;
 134         uint16_t num_lcores;
 135         int vector_mask;
 136         rte_atomic16_t sync;
 137         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
 138 };
 139
 140 /* Contains per lcore params */
 141 struct thread_params {
 142         uint8_t dev_id;
 143         uint16_t queue_id;
 144         uint32_t lcore_id;
 145         uint64_t start_time;
 146         double ops_per_sec;
 147         double mbps;
 148         uint8_t iter_count;
 149         double iter_average;
 150         double bler;
 151         rte_atomic16_t nb_dequeued;
 152         rte_atomic16_t processing_status;
 153         rte_atomic16_t burst_sz;
 154         struct test_op_params *op_params;
 155         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
 156         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
 157 };
 158
 159 #ifdef RTE_BBDEV_OFFLOAD_COST
 160 /* Stores time statistics */
 161 struct test_time_stats {
 162         /* Stores software enqueue total working time */
 163         uint64_t enq_sw_total_time;
 164         /* Stores minimum value of software enqueue working time */
 165         uint64_t enq_sw_min_time;
 166         /* Stores maximum value of software enqueue working time */
 167         uint64_t enq_sw_max_time;
 168         /* Stores turbo enqueue total working time */
 169         uint64_t enq_acc_total_time;
 170         /* Stores minimum value of accelerator enqueue working time */
 171         uint64_t enq_acc_min_time;
 172         /* Stores maximum value of accelerator enqueue working time */
 173         uint64_t enq_acc_max_time;
 174         /* Stores dequeue total working time */
 175         uint64_t deq_total_time;
 176         /* Stores minimum value of dequeue working time */
 177         uint64_t deq_min_time;
 178         /* Stores maximum value of dequeue working time */
 179         uint64_t deq_max_time;
 180 };
 181 #endif
 182
 183 typedef int (test_case_function)(struct active_device *ad,
 184                 struct test_op_params *op_params);
 185
 186 static inline void
 187 mbuf_reset(struct rte_mbuf *m)
 188 {
 189         m->pkt_len = 0;
 190
 191         do {
 192                 m->data_len = 0;
 193                 m = m->next;
 194         } while (m != NULL);
 195 }
 196
 197 /* Read flag value 0/1 from bitmap */
 198 static inline bool
 199 check_bit(uint32_t bitmap, uint32_t bitmask)
 200 {
 201         return bitmap & bitmask;
 202 }
 203
 204 static inline void
 205 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 206 {
 207         ad->supported_ops |= (1 << op_type);
 208 }
 209
 210 static inline bool
 211 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 212 {
 213         return ad->supported_ops & (1 << op_type);
 214 }
 215
 216 static inline bool
 217 flags_match(uint32_t flags_req, uint32_t flags_present)
 218 {
 219         return (flags_req & flags_present) == flags_req;
 220 }
 221
 222 static void
 223 clear_soft_out_cap(uint32_t *op_flags)
 224 {
 225         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 226         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 227         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 228 }
 229
 230 /* This API is to convert all the test vector op data entries
 231  * to big endian format. It is used when the device supports
 232  * the input in the big endian format.
 233  */
 234 static inline void
 235 convert_op_data_to_be(void)
 236 {
 237         struct op_data_entries *op;
 238         enum op_data_type type;
 239         uint8_t nb_segs, *rem_data, temp;
 240         uint32_t *data, len;
 241         int complete, rem, i, j;
 242
 243         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
 244                 nb_segs = test_vector.entries[type].nb_segments;
 245                 op = &test_vector.entries[type];
 246
 247                 /* Invert byte endianness for all the segments */
 248                 for (i = 0; i < nb_segs; ++i) {
 249                         len = op->segments[i].length;
 250                         data = op->segments[i].addr;
 251
 252                         /* Swap complete u32 bytes */
 253                         complete = len / 4;
 254                         for (j = 0; j < complete; j++)
 255                                 data[j] = rte_bswap32(data[j]);
 256
 257                         /* Swap any remaining bytes */
 258                         rem = len % 4;
 259                         rem_data = (uint8_t *)&data[j];
 260                         for (j = 0; j < rem/2; j++) {
 261                                 temp = rem_data[j];
 262                                 rem_data[j] = rem_data[rem - j - 1];
 263                                 rem_data[rem - j - 1] = temp;
 264                         }
 265                 }
 266         }
 267 }
 268
 269 static int
 270 check_dev_cap(const struct rte_bbdev_info *dev_info)
 271 {
 272         unsigned int i;
 273         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
 274                 nb_harq_inputs, nb_harq_outputs;
 275         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 276         uint8_t dev_data_endianness = dev_info->drv.data_endianness;
 277
 278         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 279         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 280         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 281         nb_harq_inputs  = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
 282         nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
 283
 284         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 285                 if (op_cap->type != test_vector.op_type)
 286                         continue;
 287
 288                 if (dev_data_endianness == RTE_BIG_ENDIAN)
 289                         convert_op_data_to_be();
 290
 291                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 292                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 293                                         &op_cap->cap.turbo_dec;
 294                         /* Ignore lack of soft output capability, just skip
 295                          * checking if soft output is valid.
 296                          */
 297                         if ((test_vector.turbo_dec.op_flags &
 298                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 299                                         !(cap->capability_flags &
 300                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 301                                 printf(
 302                                         "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 303                                         dev_info->dev_name);
 304                                 clear_soft_out_cap(
 305                                         &test_vector.turbo_dec.op_flags);
 306                         }
 307
 308                         if (!flags_match(test_vector.turbo_dec.op_flags,
 309                                         cap->capability_flags))
 310                                 return TEST_FAILED;
 311                         if (nb_inputs > cap->num_buffers_src) {
 312                                 printf("Too many inputs defined: %u, max: %u\n",
 313                                         nb_inputs, cap->num_buffers_src);
 314                                 return TEST_FAILED;
 315                         }
 316                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 317                                         (test_vector.turbo_dec.op_flags &
 318                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 319                                 printf(
 320                                         "Too many soft outputs defined: %u, max: %u\n",
 321                                                 nb_soft_outputs,
 322                                                 cap->num_buffers_soft_out);
 323                                 return TEST_FAILED;
 324                         }
 325                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 326                                 printf(
 327                                         "Too many hard outputs defined: %u, max: %u\n",
 328                                                 nb_hard_outputs,
 329                                                 cap->num_buffers_hard_out);
 330                                 return TEST_FAILED;
 331                         }
 332                         if (intr_enabled && !(cap->capability_flags &
 333                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 334                                 printf(
 335                                         "Dequeue interrupts are not supported!\n");
 336                                 return TEST_FAILED;
 337                         }
 338
 339                         return TEST_SUCCESS;
 340                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 341                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 342                                         &op_cap->cap.turbo_enc;
 343
 344                         if (!flags_match(test_vector.turbo_enc.op_flags,
 345                                         cap->capability_flags))
 346                                 return TEST_FAILED;
 347                         if (nb_inputs > cap->num_buffers_src) {
 348                                 printf("Too many inputs defined: %u, max: %u\n",
 349                                         nb_inputs, cap->num_buffers_src);
 350                                 return TEST_FAILED;
 351                         }
 352                         if (nb_hard_outputs > cap->num_buffers_dst) {
 353                                 printf(
 354                                         "Too many hard outputs defined: %u, max: %u\n",
 355                                         nb_hard_outputs, cap->num_buffers_dst);
 356                                 return TEST_FAILED;
 357                         }
 358                         if (intr_enabled && !(cap->capability_flags &
 359                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 360                                 printf(
 361                                         "Dequeue interrupts are not supported!\n");
 362                                 return TEST_FAILED;
 363                         }
 364
 365                         return TEST_SUCCESS;
 366                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
 367                         const struct rte_bbdev_op_cap_ldpc_enc *cap =
 368                                         &op_cap->cap.ldpc_enc;
 369
 370                         if (!flags_match(test_vector.ldpc_enc.op_flags,
 371                                         cap->capability_flags)){
 372                                 printf("Flag Mismatch\n");
 373                                 return TEST_FAILED;
 374                         }
 375                         if (nb_inputs > cap->num_buffers_src) {
 376                                 printf("Too many inputs defined: %u, max: %u\n",
 377                                         nb_inputs, cap->num_buffers_src);
 378                                 return TEST_FAILED;
 379                         }
 380                         if (nb_hard_outputs > cap->num_buffers_dst) {
 381                                 printf(
 382                                         "Too many hard outputs defined: %u, max: %u\n",
 383                                         nb_hard_outputs, cap->num_buffers_dst);
 384                                 return TEST_FAILED;
 385                         }
 386                         if (intr_enabled && !(cap->capability_flags &
 387                                         RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
 388                                 printf(
 389                                         "Dequeue interrupts are not supported!\n");
 390                                 return TEST_FAILED;
 391                         }
 392
 393                         return TEST_SUCCESS;
 394                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
 395                         const struct rte_bbdev_op_cap_ldpc_dec *cap =
 396                                         &op_cap->cap.ldpc_dec;
 397
 398                         if (!flags_match(test_vector.ldpc_dec.op_flags,
 399                                         cap->capability_flags)){
 400                                 printf("Flag Mismatch\n");
 401                                 return TEST_FAILED;
 402                         }
 403                         if (nb_inputs > cap->num_buffers_src) {
 404                                 printf("Too many inputs defined: %u, max: %u\n",
 405                                         nb_inputs, cap->num_buffers_src);
 406                                 return TEST_FAILED;
 407                         }
 408                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 409                                 printf(
 410                                         "Too many hard outputs defined: %u, max: %u\n",
 411                                         nb_hard_outputs,
 412                                         cap->num_buffers_hard_out);
 413                                 return TEST_FAILED;
 414                         }
 415                         if (nb_harq_inputs > cap->num_buffers_hard_out) {
 416                                 printf(
 417                                         "Too many HARQ inputs defined: %u, max: %u\n",
 418                                         nb_harq_inputs,
 419                                         cap->num_buffers_hard_out);
 420                                 return TEST_FAILED;
 421                         }
 422                         if (nb_harq_outputs > cap->num_buffers_hard_out) {
 423                                 printf(
 424                                         "Too many HARQ outputs defined: %u, max: %u\n",
 425                                         nb_harq_outputs,
 426                                         cap->num_buffers_hard_out);
 427                                 return TEST_FAILED;
 428                         }
 429                         if (intr_enabled && !(cap->capability_flags &
 430                                         RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
 431                                 printf(
 432                                         "Dequeue interrupts are not supported!\n");
 433                                 return TEST_FAILED;
 434                         }
 435                         if (intr_enabled && (test_vector.ldpc_dec.op_flags &
 436                                 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
 437                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
 438                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
 439                                         ))) {
 440                                 printf("Skip loop-back with interrupt\n");
 441                                 return TEST_FAILED;
 442                         }
 443                         return TEST_SUCCESS;
 444                 }
 445         }
 446
 447         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 448                 return TEST_SUCCESS; /* Special case for NULL device */
 449
 450         return TEST_FAILED;
 451 }
 452
 453 /* calculates optimal mempool size not smaller than the val */
 454 static unsigned int
 455 optimal_mempool_size(unsigned int val)
 456 {
 457         return rte_align32pow2(val + 1) - 1;
 458 }
 459
 460 /* allocates mbuf mempool for inputs and outputs */
 461 static struct rte_mempool *
 462 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 463                 int socket_id, unsigned int mbuf_pool_size,
 464                 const char *op_type_str)
 465 {
 466         unsigned int i;
 467         uint32_t max_seg_sz = 0;
 468         char pool_name[RTE_MEMPOOL_NAMESIZE];
 469
 470         /* find max input segment size */
 471         for (i = 0; i < entries->nb_segments; ++i)
 472                 if (entries->segments[i].length > max_seg_sz)
 473                         max_seg_sz = entries->segments[i].length;
 474
 475         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 476                         dev_id);
 477         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 478                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
 479                                         + FILLER_HEADROOM,
 480                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 481 }
 482
 483 static int
 484 create_mempools(struct active_device *ad, int socket_id,
 485                 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
 486 {
 487         struct rte_mempool *mp;
 488         unsigned int ops_pool_size, mbuf_pool_size = 0;
 489         char pool_name[RTE_MEMPOOL_NAMESIZE];
 490         const char *op_type_str;
 491         enum rte_bbdev_op_type op_type = org_op_type;
 492
 493         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 494         struct op_data_entries *hard_out =
 495                         &test_vector.entries[DATA_HARD_OUTPUT];
 496         struct op_data_entries *soft_out =
 497                         &test_vector.entries[DATA_SOFT_OUTPUT];
 498         struct op_data_entries *harq_in =
 499                         &test_vector.entries[DATA_HARQ_INPUT];
 500         struct op_data_entries *harq_out =
 501                         &test_vector.entries[DATA_HARQ_OUTPUT];
 502
 503         /* allocate ops mempool */
 504         ops_pool_size = optimal_mempool_size(RTE_MAX(
 505                         /* Ops used plus 1 reference op */
 506                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 507                         /* Minimal cache size plus 1 reference op */
 508                         (unsigned int)(1.5 * rte_lcore_count() *
 509                                         OPS_CACHE_SIZE + 1)),
 510                         OPS_POOL_SIZE_MIN));
 511
 512         if (org_op_type == RTE_BBDEV_OP_NONE)
 513                 op_type = RTE_BBDEV_OP_TURBO_ENC;
 514
 515         op_type_str = rte_bbdev_op_type_str(op_type);
 516         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 517
 518         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 519                         ad->dev_id);
 520         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 521                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 522         TEST_ASSERT_NOT_NULL(mp,
 523                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 524                         ops_pool_size,
 525                         ad->dev_id,
 526                         socket_id);
 527         ad->ops_mempool = mp;
 528
 529         /* Do not create inputs and outputs mbufs for BaseBand Null Device */
 530         if (org_op_type == RTE_BBDEV_OP_NONE)
 531                 return TEST_SUCCESS;
 532
 533         /* Inputs */
 534         if (in->nb_segments > 0) {
 535                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 536                                 in->nb_segments);
 537                 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
 538                                 mbuf_pool_size, "in");
 539                 TEST_ASSERT_NOT_NULL(mp,
 540                                 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 541                                 mbuf_pool_size,
 542                                 ad->dev_id,
 543                                 socket_id);
 544                 ad->in_mbuf_pool = mp;
 545         }
 546
 547         /* Hard outputs */
 548         if (hard_out->nb_segments > 0) {
 549                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 550                                 hard_out->nb_segments);
 551                 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
 552                                 mbuf_pool_size,
 553                                 "hard_out");
 554                 TEST_ASSERT_NOT_NULL(mp,
 555                                 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 556                                 mbuf_pool_size,
 557                                 ad->dev_id,
 558                                 socket_id);
 559                 ad->hard_out_mbuf_pool = mp;
 560         }
 561
 562         /* Soft outputs */
 563         if (soft_out->nb_segments > 0) {
 564                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 565                                 soft_out->nb_segments);
 566                 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
 567                                 mbuf_pool_size,
 568                                 "soft_out");
 569                 TEST_ASSERT_NOT_NULL(mp,
 570                                 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 571                                 mbuf_pool_size,
 572                                 ad->dev_id,
 573                                 socket_id);
 574                 ad->soft_out_mbuf_pool = mp;
 575         }
 576
 577         /* HARQ inputs */
 578         if (harq_in->nb_segments > 0) {
 579                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 580                                 harq_in->nb_segments);
 581                 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
 582                                 mbuf_pool_size,
 583                                 "harq_in");
 584                 TEST_ASSERT_NOT_NULL(mp,
 585                                 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
 586                                 mbuf_pool_size,
 587                                 ad->dev_id,
 588                                 socket_id);
 589                 ad->harq_in_mbuf_pool = mp;
 590         }
 591
 592         /* HARQ outputs */
 593         if (harq_out->nb_segments > 0) {
 594                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 595                                 harq_out->nb_segments);
 596                 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
 597                                 mbuf_pool_size,
 598                                 "harq_out");
 599                 TEST_ASSERT_NOT_NULL(mp,
 600                                 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
 601                                 mbuf_pool_size,
 602                                 ad->dev_id,
 603                                 socket_id);
 604                 ad->harq_out_mbuf_pool = mp;
 605         }
 606
 607         return TEST_SUCCESS;
 608 }
 609
 610 static int
 611 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 612                 struct test_bbdev_vector *vector)
 613 {
 614         int ret;
 615         unsigned int queue_id;
 616         struct rte_bbdev_queue_conf qconf;
 617         struct active_device *ad = &active_devs[nb_active_devs];
 618         unsigned int nb_queues;
 619         enum rte_bbdev_op_type op_type = vector->op_type;
 620
 621 /* Configure fpga lte fec with PF & VF values
 622  * if '-i' flag is set and using fpga device
 623  */
 624 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
 625         if ((get_init_device() == true) &&
 626                 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
 627                 struct rte_fpga_lte_fec_conf conf;
 628                 unsigned int i;
 629
 630                 printf("Configure FPGA LTE FEC Driver %s with default values\n",
 631                                 info->drv.driver_name);
 632
 633                 /* clear default configuration before initialization */
 634                 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf));
 635
 636                 /* Set PF mode :
 637                  * true if PF is used for data plane
 638                  * false for VFs
 639                  */
 640                 conf.pf_mode_en = true;
 641
 642                 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
 643                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 644                         conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
 645                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 646                         conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
 647                 }
 648
 649                 /* UL bandwidth. Needed for schedule algorithm */
 650                 conf.ul_bandwidth = UL_4G_BANDWIDTH;
 651                 /* DL bandwidth */
 652                 conf.dl_bandwidth = DL_4G_BANDWIDTH;
 653
 654                 /* UL & DL load Balance Factor to 64 */
 655                 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
 656                 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
 657
 658                 /**< FLR timeout value */
 659                 conf.flr_time_out = FLR_4G_TIMEOUT;
 660
 661                 /* setup FPGA PF with configuration information */
 662                 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf);
 663                 TEST_ASSERT_SUCCESS(ret,
 664                                 "Failed to configure 4G FPGA PF for bbdev %s",
 665                                 info->dev_name);
 666         }
 667 #endif
 668 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
 669         if ((get_init_device() == true) &&
 670                 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
 671                 struct rte_fpga_5gnr_fec_conf conf;
 672                 unsigned int i;
 673
 674                 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
 675                                 info->drv.driver_name);
 676
 677                 /* clear default configuration before initialization */
 678                 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
 679
 680                 /* Set PF mode :
 681                  * true if PF is used for data plane
 682                  * false for VFs
 683                  */
 684                 conf.pf_mode_en = true;
 685
 686                 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
 687                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 688                         conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
 689                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 690                         conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
 691                 }
 692
 693                 /* UL bandwidth. Needed for schedule algorithm */
 694                 conf.ul_bandwidth = UL_5G_BANDWIDTH;
 695                 /* DL bandwidth */
 696                 conf.dl_bandwidth = DL_5G_BANDWIDTH;
 697
 698                 /* UL & DL load Balance Factor to 64 */
 699                 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
 700                 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
 701
 702                 /**< FLR timeout value */
 703                 conf.flr_time_out = FLR_5G_TIMEOUT;
 704
 705                 /* setup FPGA PF with configuration information */
 706                 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf);
 707                 TEST_ASSERT_SUCCESS(ret,
 708                                 "Failed to configure 5G FPGA PF for bbdev %s",
 709                                 info->dev_name);
 710         }
 711 #endif
 712 #ifdef RTE_BASEBAND_ACC100
 713         if ((get_init_device() == true) &&
 714                 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) {
 715                 struct rte_acc100_conf conf;
 716                 unsigned int i;
 717
 718                 printf("Configure ACC100 FEC Driver %s with default values\n",
 719                                 info->drv.driver_name);
 720
 721                 /* clear default configuration before initialization */
 722                 memset(&conf, 0, sizeof(struct rte_acc100_conf));
 723
 724                 /* Always set in PF mode for built-in configuration */
 725                 conf.pf_mode_en = true;
 726                 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) {
 727                         conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 728                         conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 729                         conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR;
 730                         conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 731                         conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 732                         conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR;
 733                         conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 734                         conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 735                         conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR;
 736                         conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 737                         conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 738                         conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR;
 739                 }
 740
 741                 conf.input_pos_llr_1_bit = true;
 742                 conf.output_pos_llr_1_bit = true;
 743                 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */
 744
 745                 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
 746                 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 747                 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 748                 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 749                 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
 750                 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 751                 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 752                 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 753                 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
 754                 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 755                 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 756                 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 757                 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
 758                 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 759                 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 760                 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 761
 762                 /* setup PF with configuration information */
 763                 ret = rte_acc100_configure(info->dev_name, &conf);
 764                 TEST_ASSERT_SUCCESS(ret,
 765                                 "Failed to configure ACC100 PF for bbdev %s",
 766                                 info->dev_name);
 767         }
 768 #endif
 769         /* Let's refresh this now this is configured */
 770         rte_bbdev_info_get(dev_id, info);
 771         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 772         nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
 773
 774         /* setup device */
 775         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 776         if (ret < 0) {
 777                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 778                                 dev_id, nb_queues, info->socket_id, ret);
 779                 return TEST_FAILED;
 780         }
 781
 782         /* configure interrupts if needed */
 783         if (intr_enabled) {
 784                 ret = rte_bbdev_intr_enable(dev_id);
 785                 if (ret < 0) {
 786                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 787                                         ret);
 788                         return TEST_FAILED;
 789                 }
 790         }
 791
 792         /* setup device queues */
 793         qconf.socket = info->socket_id;
 794         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 795         qconf.priority = 0;
 796         qconf.deferred_start = 0;
 797         qconf.op_type = op_type;
 798
 799         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 800                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 801                 if (ret != 0) {
 802                         printf(
 803                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 804                                         queue_id, qconf.priority, dev_id);
 805                         qconf.priority++;
 806                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 807                                         &qconf);
 808                 }
 809                 if (ret != 0) {
 810                         printf("All queues on dev %u allocated: %u\n",
 811                                         dev_id, queue_id);
 812                         break;
 813                 }
 814                 ad->queue_ids[queue_id] = queue_id;
 815         }
 816         TEST_ASSERT(queue_id != 0,
 817                         "ERROR Failed to configure any queues on dev %u",
 818                         dev_id);
 819         ad->nb_queues = queue_id;
 820
 821         set_avail_op(ad, op_type);
 822
 823         return TEST_SUCCESS;
 824 }
 825
 826 static int
 827 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 828                 struct test_bbdev_vector *vector)
 829 {
 830         int ret;
 831
 832         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 833         active_devs[nb_active_devs].dev_id = dev_id;
 834
 835         ret = add_bbdev_dev(dev_id, info, vector);
 836         if (ret == TEST_SUCCESS)
 837                 ++nb_active_devs;
 838         return ret;
 839 }
 840
 841 static uint8_t
 842 populate_active_devices(void)
 843 {
 844         int ret;
 845         uint8_t dev_id;
 846         uint8_t nb_devs_added = 0;
 847         struct rte_bbdev_info info;
 848
 849         RTE_BBDEV_FOREACH(dev_id) {
 850                 rte_bbdev_info_get(dev_id, &info);
 851
 852                 if (check_dev_cap(&info)) {
 853                         printf(
 854                                 "Device %d (%s) does not support specified capabilities\n",
 855                                         dev_id, info.dev_name);
 856                         continue;
 857                 }
 858
 859                 ret = add_active_device(dev_id, &info, &test_vector);
 860                 if (ret != 0) {
 861                         printf("Adding active bbdev %s skipped\n",
 862                                         info.dev_name);
 863                         continue;
 864                 }
 865                 nb_devs_added++;
 866         }
 867
 868         return nb_devs_added;
 869 }
 870
 871 static int
 872 read_test_vector(void)
 873 {
 874         int ret;
 875
 876         memset(&test_vector, 0, sizeof(test_vector));
 877         printf("Test vector file = %s\n", get_vector_filename());
 878         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 879         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 880                         get_vector_filename());
 881
 882         return TEST_SUCCESS;
 883 }
 884
 885 static int
 886 testsuite_setup(void)
 887 {
 888         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 889
 890         if (populate_active_devices() == 0) {
 891                 printf("No suitable devices found!\n");
 892                 return TEST_SKIPPED;
 893         }
 894
 895         return TEST_SUCCESS;
 896 }
 897
 898 static int
 899 interrupt_testsuite_setup(void)
 900 {
 901         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 902
 903         /* Enable interrupts */
 904         intr_enabled = true;
 905
 906         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 907         if (populate_active_devices() == 0 ||
 908                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 909                 intr_enabled = false;
 910                 printf("No suitable devices found!\n");
 911                 return TEST_SKIPPED;
 912         }
 913
 914         return TEST_SUCCESS;
 915 }
 916
 917 static void
 918 testsuite_teardown(void)
 919 {
 920         uint8_t dev_id;
 921
 922         /* Unconfigure devices */
 923         RTE_BBDEV_FOREACH(dev_id)
 924                 rte_bbdev_close(dev_id);
 925
 926         /* Clear active devices structs. */
 927         memset(active_devs, 0, sizeof(active_devs));
 928         nb_active_devs = 0;
 929
 930         /* Disable interrupts */
 931         intr_enabled = false;
 932 }
 933
 934 static int
 935 ut_setup(void)
 936 {
 937         uint8_t i, dev_id;
 938
 939         for (i = 0; i < nb_active_devs; i++) {
 940                 dev_id = active_devs[i].dev_id;
 941                 /* reset bbdev stats */
 942                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 943                                 "Failed to reset stats of bbdev %u", dev_id);
 944                 /* start the device */
 945                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 946                                 "Failed to start bbdev %u", dev_id);
 947         }
 948
 949         return TEST_SUCCESS;
 950 }
 951
 952 static void
 953 ut_teardown(void)
 954 {
 955         uint8_t i, dev_id;
 956         struct rte_bbdev_stats stats;
 957
 958         for (i = 0; i < nb_active_devs; i++) {
 959                 dev_id = active_devs[i].dev_id;
 960                 /* read stats and print */
 961                 rte_bbdev_stats_get(dev_id, &stats);
 962                 /* Stop the device */
 963                 rte_bbdev_stop(dev_id);
 964         }
 965 }
 966
 967 static int
 968 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 969                 struct op_data_entries *ref_entries,
 970                 struct rte_mempool *mbuf_pool, const uint16_t n,
 971                 enum op_data_type op_type, uint16_t min_alignment)
 972 {
 973         int ret;
 974         unsigned int i, j;
 975         bool large_input = false;
 976
 977         for (i = 0; i < n; ++i) {
 978                 char *data;
 979                 struct op_data_buf *seg = &ref_entries->segments[0];
 980                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 981                 TEST_ASSERT_NOT_NULL(m_head,
 982                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 983                                 op_type, n * ref_entries->nb_segments,
 984                                 mbuf_pool->size);
 985
 986                 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
 987                         /*
 988                          * Special case when DPDK mbuf cannot handle
 989                          * the required input size
 990                          */
 991                         printf("Warning: Larger input size than DPDK mbuf %d\n",
 992                                         seg->length);
 993                         large_input = true;
 994                 }
 995                 bufs[i].data = m_head;
 996                 bufs[i].offset = 0;
 997                 bufs[i].length = 0;
 998
 999                 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
1000                         if ((op_type == DATA_INPUT) && large_input) {
1001                                 /* Allocate a fake overused mbuf */
1002                                 data = rte_malloc(NULL, seg->length, 0);
1003                                 TEST_ASSERT_NOT_NULL(data,
1004                                         "rte malloc failed with %u bytes",
1005                                         seg->length);
1006                                 memcpy(data, seg->addr, seg->length);
1007                                 m_head->buf_addr = data;
1008                                 m_head->buf_iova = rte_malloc_virt2iova(data);
1009                                 m_head->data_off = 0;
1010                                 m_head->data_len = seg->length;
1011                         } else {
1012                                 data = rte_pktmbuf_append(m_head, seg->length);
1013                                 TEST_ASSERT_NOT_NULL(data,
1014                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
1015                                         seg->length, op_type);
1016
1017                                 TEST_ASSERT(data == RTE_PTR_ALIGN(
1018                                                 data, min_alignment),
1019                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
1020                                         data, min_alignment);
1021                                 rte_memcpy(data, seg->addr, seg->length);
1022                         }
1023
1024                         bufs[i].length += seg->length;
1025
1026                         for (j = 1; j < ref_entries->nb_segments; ++j) {
1027                                 struct rte_mbuf *m_tail =
1028                                                 rte_pktmbuf_alloc(mbuf_pool);
1029                                 TEST_ASSERT_NOT_NULL(m_tail,
1030                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1031                                                 op_type,
1032                                                 n * ref_entries->nb_segments,
1033                                                 mbuf_pool->size);
1034                                 seg += 1;
1035
1036                                 data = rte_pktmbuf_append(m_tail, seg->length);
1037                                 TEST_ASSERT_NOT_NULL(data,
1038                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
1039                                                 seg->length, op_type);
1040
1041                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
1042                                                 min_alignment),
1043                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
1044                                                 data, min_alignment);
1045                                 rte_memcpy(data, seg->addr, seg->length);
1046                                 bufs[i].length += seg->length;
1047
1048                                 ret = rte_pktmbuf_chain(m_head, m_tail);
1049                                 TEST_ASSERT_SUCCESS(ret,
1050                                                 "Couldn't chain mbufs from %d data type mbuf pool",
1051                                                 op_type);
1052                         }
1053                 } else {
1054
1055                         /* allocate chained-mbuf for output buffer */
1056                         for (j = 1; j < ref_entries->nb_segments; ++j) {
1057                                 struct rte_mbuf *m_tail =
1058                                                 rte_pktmbuf_alloc(mbuf_pool);
1059                                 TEST_ASSERT_NOT_NULL(m_tail,
1060                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1061                                                 op_type,
1062                                                 n * ref_entries->nb_segments,
1063                                                 mbuf_pool->size);
1064
1065                                 ret = rte_pktmbuf_chain(m_head, m_tail);
1066                                 TEST_ASSERT_SUCCESS(ret,
1067                                                 "Couldn't chain mbufs from %d data type mbuf pool",
1068                                                 op_type);
1069                         }
1070                 }
1071         }
1072
1073         return 0;
1074 }
1075
1076 static int
1077 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
1078                 const int socket)
1079 {
1080         int i;
1081
1082         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
1083         if (*buffers == NULL) {
1084                 printf("WARNING: Failed to allocate op_data on socket %d\n",
1085                                 socket);
1086                 /* try to allocate memory on other detected sockets */
1087                 for (i = 0; i < socket; i++) {
1088                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
1089                         if (*buffers != NULL)
1090                                 break;
1091                 }
1092         }
1093
1094         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
1095 }
1096
1097 static void
1098 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
1099                 const uint16_t n, const int8_t max_llr_modulus)
1100 {
1101         uint16_t i, byte_idx;
1102
1103         for (i = 0; i < n; ++i) {
1104                 struct rte_mbuf *m = input_ops[i].data;
1105                 while (m != NULL) {
1106                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1107                                         input_ops[i].offset);
1108                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1109                                         ++byte_idx)
1110                                 llr[byte_idx] = round((double)max_llr_modulus *
1111                                                 llr[byte_idx] / INT8_MAX);
1112
1113                         m = m->next;
1114                 }
1115         }
1116 }
1117
1118 /*
1119  * We may have to insert filler bits
1120  * when they are required by the HARQ assumption
1121  */
1122 static void
1123 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1124                 const uint16_t n, struct test_op_params *op_params)
1125 {
1126         struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1127
1128         if (input_ops == NULL)
1129                 return;
1130         /* No need to add filler if not required by device */
1131         if (!(ldpc_cap_flags &
1132                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1133                 return;
1134         /* No need to add filler for loopback operation */
1135         if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1136                 return;
1137
1138         uint16_t i, j, parity_offset;
1139         for (i = 0; i < n; ++i) {
1140                 struct rte_mbuf *m = input_ops[i].data;
1141                 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1142                                 input_ops[i].offset);
1143                 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1144                                 * dec.z_c - dec.n_filler;
1145                 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1146                 m->data_len = new_hin_size;
1147                 input_ops[i].length = new_hin_size;
1148                 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1149                                 j--)
1150                         llr[j] = llr[j - dec.n_filler];
1151                 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1152                 for (j = 0; j < dec.n_filler; j++)
1153                         llr[parity_offset + j] = llr_max_pre_scaling;
1154         }
1155 }
1156
1157 static void
1158 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1159                 const uint16_t n, const int8_t llr_size,
1160                 const int8_t llr_decimals)
1161 {
1162         if (input_ops == NULL)
1163                 return;
1164
1165         uint16_t i, byte_idx;
1166
1167         int16_t llr_max, llr_min, llr_tmp;
1168         llr_max = (1 << (llr_size - 1)) - 1;
1169         llr_min = -llr_max;
1170         for (i = 0; i < n; ++i) {
1171                 struct rte_mbuf *m = input_ops[i].data;
1172                 while (m != NULL) {
1173                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1174                                         input_ops[i].offset);
1175                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1176                                         ++byte_idx) {
1177
1178                                 llr_tmp = llr[byte_idx];
1179                                 if (llr_decimals == 4)
1180                                         llr_tmp *= 8;
1181                                 else if (llr_decimals == 2)
1182                                         llr_tmp *= 2;
1183                                 else if (llr_decimals == 0)
1184                                         llr_tmp /= 2;
1185                                 llr_tmp = RTE_MIN(llr_max,
1186                                                 RTE_MAX(llr_min, llr_tmp));
1187                                 llr[byte_idx] = (int8_t) llr_tmp;
1188                         }
1189
1190                         m = m->next;
1191                 }
1192         }
1193 }
1194
1195
1196
1197 static int
1198 fill_queue_buffers(struct test_op_params *op_params,
1199                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1200                 struct rte_mempool *soft_out_mp,
1201                 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1202                 uint16_t queue_id,
1203                 const struct rte_bbdev_op_cap *capabilities,
1204                 uint16_t min_alignment, const int socket_id)
1205 {
1206         int ret;
1207         enum op_data_type type;
1208         const uint16_t n = op_params->num_to_process;
1209
1210         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1211                 in_mp,
1212                 soft_out_mp,
1213                 hard_out_mp,
1214                 harq_in_mp,
1215                 harq_out_mp,
1216         };
1217
1218         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1219                 &op_params->q_bufs[socket_id][queue_id].inputs,
1220                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1221                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1222                 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1223                 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1224         };
1225
1226         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1227                 struct op_data_entries *ref_entries =
1228                                 &test_vector.entries[type];
1229                 if (ref_entries->nb_segments == 0)
1230                         continue;
1231
1232                 ret = allocate_buffers_on_socket(queue_ops[type],
1233                                 n * sizeof(struct rte_bbdev_op_data),
1234                                 socket_id);
1235                 TEST_ASSERT_SUCCESS(ret,
1236                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
1237
1238                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1239                                 mbuf_pools[type], n, type, min_alignment);
1240                 TEST_ASSERT_SUCCESS(ret,
1241                                 "Couldn't init rte_bbdev_op_data structs");
1242         }
1243
1244         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1245                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1246                         capabilities->cap.turbo_dec.max_llr_modulus);
1247
1248         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1249                 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1250                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1251                 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1252                                 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1253                 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1254                                 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1255                 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1256                 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1257                 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1258                 if (!loopback && !llr_comp)
1259                         ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1260                                         ldpc_llr_size, ldpc_llr_decimals);
1261                 if (!loopback && !harq_comp)
1262                         ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1263                                         ldpc_llr_size, ldpc_llr_decimals);
1264                 if (!loopback)
1265                         ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1266                                         op_params);
1267         }
1268
1269         return 0;
1270 }
1271
1272 static void
1273 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1274 {
1275         unsigned int i, j;
1276
1277         rte_mempool_free(ad->ops_mempool);
1278         rte_mempool_free(ad->in_mbuf_pool);
1279         rte_mempool_free(ad->hard_out_mbuf_pool);
1280         rte_mempool_free(ad->soft_out_mbuf_pool);
1281         rte_mempool_free(ad->harq_in_mbuf_pool);
1282         rte_mempool_free(ad->harq_out_mbuf_pool);
1283
1284         for (i = 0; i < rte_lcore_count(); ++i) {
1285                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1286                         rte_free(op_params->q_bufs[j][i].inputs);
1287                         rte_free(op_params->q_bufs[j][i].hard_outputs);
1288                         rte_free(op_params->q_bufs[j][i].soft_outputs);
1289                         rte_free(op_params->q_bufs[j][i].harq_inputs);
1290                         rte_free(op_params->q_bufs[j][i].harq_outputs);
1291                 }
1292         }
1293 }
1294
1295 static void
1296 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1297                 unsigned int start_idx,
1298                 struct rte_bbdev_op_data *inputs,
1299                 struct rte_bbdev_op_data *hard_outputs,
1300                 struct rte_bbdev_op_data *soft_outputs,
1301                 struct rte_bbdev_dec_op *ref_op)
1302 {
1303         unsigned int i;
1304         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1305
1306         for (i = 0; i < n; ++i) {
1307                 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1308                         ops[i]->turbo_dec.tb_params.ea =
1309                                         turbo_dec->tb_params.ea;
1310                         ops[i]->turbo_dec.tb_params.eb =
1311                                         turbo_dec->tb_params.eb;
1312                         ops[i]->turbo_dec.tb_params.k_pos =
1313                                         turbo_dec->tb_params.k_pos;
1314                         ops[i]->turbo_dec.tb_params.k_neg =
1315                                         turbo_dec->tb_params.k_neg;
1316                         ops[i]->turbo_dec.tb_params.c =
1317                                         turbo_dec->tb_params.c;
1318                         ops[i]->turbo_dec.tb_params.c_neg =
1319                                         turbo_dec->tb_params.c_neg;
1320                         ops[i]->turbo_dec.tb_params.cab =
1321                                         turbo_dec->tb_params.cab;
1322                         ops[i]->turbo_dec.tb_params.r =
1323                                         turbo_dec->tb_params.r;
1324                 } else {
1325                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1326                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1327                 }
1328
1329                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1330                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1331                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1332                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1333                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1334                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1335                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1336
1337                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1338                 ops[i]->turbo_dec.input = inputs[start_idx + i];
1339                 if (soft_outputs != NULL)
1340                         ops[i]->turbo_dec.soft_output =
1341                                 soft_outputs[start_idx + i];
1342         }
1343 }
1344
1345 static void
1346 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1347                 unsigned int start_idx,
1348                 struct rte_bbdev_op_data *inputs,
1349                 struct rte_bbdev_op_data *outputs,
1350                 struct rte_bbdev_enc_op *ref_op)
1351 {
1352         unsigned int i;
1353         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1354         for (i = 0; i < n; ++i) {
1355                 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1356                         ops[i]->turbo_enc.tb_params.ea =
1357                                         turbo_enc->tb_params.ea;
1358                         ops[i]->turbo_enc.tb_params.eb =
1359                                         turbo_enc->tb_params.eb;
1360                         ops[i]->turbo_enc.tb_params.k_pos =
1361                                         turbo_enc->tb_params.k_pos;
1362                         ops[i]->turbo_enc.tb_params.k_neg =
1363                                         turbo_enc->tb_params.k_neg;
1364                         ops[i]->turbo_enc.tb_params.c =
1365                                         turbo_enc->tb_params.c;
1366                         ops[i]->turbo_enc.tb_params.c_neg =
1367                                         turbo_enc->tb_params.c_neg;
1368                         ops[i]->turbo_enc.tb_params.cab =
1369                                         turbo_enc->tb_params.cab;
1370                         ops[i]->turbo_enc.tb_params.ncb_pos =
1371                                         turbo_enc->tb_params.ncb_pos;
1372                         ops[i]->turbo_enc.tb_params.ncb_neg =
1373                                         turbo_enc->tb_params.ncb_neg;
1374                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1375                 } else {
1376                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1377                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1378                         ops[i]->turbo_enc.cb_params.ncb =
1379                                         turbo_enc->cb_params.ncb;
1380                 }
1381                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1382                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1383                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1384
1385                 ops[i]->turbo_enc.output = outputs[start_idx + i];
1386                 ops[i]->turbo_enc.input = inputs[start_idx + i];
1387         }
1388 }
1389
1390
1391 /* Returns a random number drawn from a normal distribution
1392  * with mean of 0 and variance of 1
1393  * Marsaglia algorithm
1394  */
1395 static double
1396 randn(int n)
1397 {
1398         double S, Z, U1, U2, u, v, fac;
1399
1400         do {
1401                 U1 = (double)rand() / RAND_MAX;
1402                 U2 = (double)rand() / RAND_MAX;
1403                 u = 2. * U1 - 1.;
1404                 v = 2. * U2 - 1.;
1405                 S = u * u + v * v;
1406         } while (S >= 1 || S == 0);
1407         fac = sqrt(-2. * log(S) / S);
1408         Z = (n % 2) ? u * fac : v * fac;
1409         return Z;
1410 }
1411
1412 static inline double
1413 maxstar(double A, double B)
1414 {
1415         if (fabs(A - B) > 5)
1416                 return RTE_MAX(A, B);
1417         else
1418                 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1419 }
1420
1421 /*
1422  * Generate Qm LLRS for Qm==8
1423  * Modulation, AWGN and LLR estimation from max log development
1424  */
1425 static void
1426 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1427 {
1428         int qm = 8;
1429         int qam = 256;
1430         int m, k;
1431         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1432         /* 5.1.4 of TS38.211 */
1433         const double symbols_I[256] = {
1434                         5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1435                         5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1436                         11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1437                         15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1438                         15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1439                         1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1440                         1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1441                         15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1442                         13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1443                         -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1444                         -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1445                         -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1446                         -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1447                         -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1448                         -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1449                         -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1450                         -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1451                         -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1452                         -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1453                         -13, -13, -15, -15, -13, -13, -15, -15};
1454         const double symbols_Q[256] = {
1455                         5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1456                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1457                         15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1458                         11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1459                         15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1460                         -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1461                         -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1462                         -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1463                         -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1464                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1465                         7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1466                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1467                         13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1468                         3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1469                         13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1470                         -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1471                         -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1472                         -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1473                         -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1474                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1475         /* Average constellation point energy */
1476         N0 *= 170.0;
1477         for (k = 0; k < qm; k++)
1478                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1479         /* 5.1.4 of TS38.211 */
1480         I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1481                         (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1482         Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1483                         (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1484         /* AWGN channel */
1485         I += sqrt(N0 / 2) * randn(0);
1486         Q += sqrt(N0 / 2) * randn(1);
1487         /*
1488          * Calculate the log of the probability that each of
1489          * the constellation points was transmitted
1490          */
1491         for (m = 0; m < qam; m++)
1492                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1493                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1494         /* Calculate an LLR for each of the k_64QAM bits in the set */
1495         for (k = 0; k < qm; k++) {
1496                 p0 = -999999;
1497                 p1 = -999999;
1498                 /* For each constellation point */
1499                 for (m = 0; m < qam; m++) {
1500                         if ((m >> (qm - k - 1)) & 1)
1501                                 p1 = maxstar(p1, log_syml_prob[m]);
1502                         else
1503                                 p0 = maxstar(p0, log_syml_prob[m]);
1504                 }
1505                 /* Calculate the LLR */
1506                 llr_ = p0 - p1;
1507                 llr_ *= (1 << ldpc_llr_decimals);
1508                 llr_ = round(llr_);
1509                 if (llr_ > llr_max)
1510                         llr_ = llr_max;
1511                 if (llr_ < -llr_max)
1512                         llr_ = -llr_max;
1513                 llrs[qm * i + k] = (int8_t) llr_;
1514         }
1515 }
1516
1517
1518 /*
1519  * Generate Qm LLRS for Qm==6
1520  * Modulation, AWGN and LLR estimation from max log development
1521  */
1522 static void
1523 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1524 {
1525         int qm = 6;
1526         int qam = 64;
1527         int m, k;
1528         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1529         /* 5.1.4 of TS38.211 */
1530         const double symbols_I[64] = {
1531                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1532                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1533                         -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1534                         -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1535                         -5, -5, -7, -7, -5, -5, -7, -7};
1536         const double symbols_Q[64] = {
1537                         3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1538                         -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1539                         -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1540                         5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1541                         -3, -1, -3, -1, -5, -7, -5, -7};
1542         /* Average constellation point energy */
1543         N0 *= 42.0;
1544         for (k = 0; k < qm; k++)
1545                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1546         /* 5.1.4 of TS38.211 */
1547         I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1548         Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1549         /* AWGN channel */
1550         I += sqrt(N0 / 2) * randn(0);
1551         Q += sqrt(N0 / 2) * randn(1);
1552         /*
1553          * Calculate the log of the probability that each of
1554          * the constellation points was transmitted
1555          */
1556         for (m = 0; m < qam; m++)
1557                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1558                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1559         /* Calculate an LLR for each of the k_64QAM bits in the set */
1560         for (k = 0; k < qm; k++) {
1561                 p0 = -999999;
1562                 p1 = -999999;
1563                 /* For each constellation point */
1564                 for (m = 0; m < qam; m++) {
1565                         if ((m >> (qm - k - 1)) & 1)
1566                                 p1 = maxstar(p1, log_syml_prob[m]);
1567                         else
1568                                 p0 = maxstar(p0, log_syml_prob[m]);
1569                 }
1570                 /* Calculate the LLR */
1571                 llr_ = p0 - p1;
1572                 llr_ *= (1 << ldpc_llr_decimals);
1573                 llr_ = round(llr_);
1574                 if (llr_ > llr_max)
1575                         llr_ = llr_max;
1576                 if (llr_ < -llr_max)
1577                         llr_ = -llr_max;
1578                 llrs[qm * i + k] = (int8_t) llr_;
1579         }
1580 }
1581
1582 /*
1583  * Generate Qm LLRS for Qm==4
1584  * Modulation, AWGN and LLR estimation from max log development
1585  */
1586 static void
1587 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1588 {
1589         int qm = 4;
1590         int qam = 16;
1591         int m, k;
1592         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1593         /* 5.1.4 of TS38.211 */
1594         const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1595                         -1, -1, -3, -3, -1, -1, -3, -3};
1596         const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1597                         1, 3, 1, 3, -1, -3, -1, -3};
1598         /* Average constellation point energy */
1599         N0 *= 10.0;
1600         for (k = 0; k < qm; k++)
1601                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1602         /* 5.1.4 of TS38.211 */
1603         I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1604         Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1605         /* AWGN channel */
1606         I += sqrt(N0 / 2) * randn(0);
1607         Q += sqrt(N0 / 2) * randn(1);
1608         /*
1609          * Calculate the log of the probability that each of
1610          * the constellation points was transmitted
1611          */
1612         for (m = 0; m < qam; m++)
1613                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1614                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1615         /* Calculate an LLR for each of the k_64QAM bits in the set */
1616         for (k = 0; k < qm; k++) {
1617                 p0 = -999999;
1618                 p1 = -999999;
1619                 /* For each constellation point */
1620                 for (m = 0; m < qam; m++) {
1621                         if ((m >> (qm - k - 1)) & 1)
1622                                 p1 = maxstar(p1, log_syml_prob[m]);
1623                         else
1624                                 p0 = maxstar(p0, log_syml_prob[m]);
1625                 }
1626                 /* Calculate the LLR */
1627                 llr_ = p0 - p1;
1628                 llr_ *= (1 << ldpc_llr_decimals);
1629                 llr_ = round(llr_);
1630                 if (llr_ > llr_max)
1631                         llr_ = llr_max;
1632                 if (llr_ < -llr_max)
1633                         llr_ = -llr_max;
1634                 llrs[qm * i + k] = (int8_t) llr_;
1635         }
1636 }
1637
1638 static void
1639 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1640 {
1641         double b, b1, n;
1642         double coeff = 2.0 * sqrt(N0);
1643
1644         /* Ignore in vectors rare quasi null LLRs not to be saturated */
1645         if (llrs[j] < 8 && llrs[j] > -8)
1646                 return;
1647
1648         /* Note don't change sign here */
1649         n = randn(j % 2);
1650         b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1651                         + coeff * n) / N0;
1652         b = b1 * (1 << ldpc_llr_decimals);
1653         b = round(b);
1654         if (b > llr_max)
1655                 b = llr_max;
1656         if (b < -llr_max)
1657                 b = -llr_max;
1658         llrs[j] = (int8_t) b;
1659 }
1660
1661 /* Generate LLR for a given SNR */
1662 static void
1663 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1664                 struct rte_bbdev_dec_op *ref_op)
1665 {
1666         struct rte_mbuf *m;
1667         uint16_t qm;
1668         uint32_t i, j, e, range;
1669         double N0, llr_max;
1670
1671         e = ref_op->ldpc_dec.cb_params.e;
1672         qm = ref_op->ldpc_dec.q_m;
1673         llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1674         range = e / qm;
1675         N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1676
1677         for (i = 0; i < n; ++i) {
1678                 m = inputs[i].data;
1679                 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1680                 if (qm == 8) {
1681                         for (j = 0; j < range; ++j)
1682                                 gen_qm8_llr(llrs, j, N0, llr_max);
1683                 } else if (qm == 6) {
1684                         for (j = 0; j < range; ++j)
1685                                 gen_qm6_llr(llrs, j, N0, llr_max);
1686                 } else if (qm == 4) {
1687                         for (j = 0; j < range; ++j)
1688                                 gen_qm4_llr(llrs, j, N0, llr_max);
1689                 } else {
1690                         for (j = 0; j < e; ++j)
1691                                 gen_qm2_llr(llrs, j, N0, llr_max);
1692                 }
1693         }
1694 }
1695
1696 static void
1697 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1698                 unsigned int start_idx,
1699                 struct rte_bbdev_op_data *inputs,
1700                 struct rte_bbdev_op_data *hard_outputs,
1701                 struct rte_bbdev_op_data *soft_outputs,
1702                 struct rte_bbdev_op_data *harq_inputs,
1703                 struct rte_bbdev_op_data *harq_outputs,
1704                 struct rte_bbdev_dec_op *ref_op)
1705 {
1706         unsigned int i;
1707         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1708
1709         for (i = 0; i < n; ++i) {
1710                 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1711                         ops[i]->ldpc_dec.tb_params.ea =
1712                                         ldpc_dec->tb_params.ea;
1713                         ops[i]->ldpc_dec.tb_params.eb =
1714                                         ldpc_dec->tb_params.eb;
1715                         ops[i]->ldpc_dec.tb_params.c =
1716                                         ldpc_dec->tb_params.c;
1717                         ops[i]->ldpc_dec.tb_params.cab =
1718                                         ldpc_dec->tb_params.cab;
1719                         ops[i]->ldpc_dec.tb_params.r =
1720                                         ldpc_dec->tb_params.r;
1721                 } else {
1722                         ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1723                 }
1724
1725                 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1726                 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1727                 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1728                 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1729                 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1730                 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1731                 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1732                 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1733                 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1734
1735                 if (hard_outputs != NULL)
1736                         ops[i]->ldpc_dec.hard_output =
1737                                         hard_outputs[start_idx + i];
1738                 if (inputs != NULL)
1739                         ops[i]->ldpc_dec.input =
1740                                         inputs[start_idx + i];
1741                 if (soft_outputs != NULL)
1742                         ops[i]->ldpc_dec.soft_output =
1743                                         soft_outputs[start_idx + i];
1744                 if (harq_inputs != NULL)
1745                         ops[i]->ldpc_dec.harq_combined_input =
1746                                         harq_inputs[start_idx + i];
1747                 if (harq_outputs != NULL)
1748                         ops[i]->ldpc_dec.harq_combined_output =
1749                                         harq_outputs[start_idx + i];
1750         }
1751 }
1752
1753
1754 static void
1755 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1756                 unsigned int start_idx,
1757                 struct rte_bbdev_op_data *inputs,
1758                 struct rte_bbdev_op_data *outputs,
1759                 struct rte_bbdev_enc_op *ref_op)
1760 {
1761         unsigned int i;
1762         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1763         for (i = 0; i < n; ++i) {
1764                 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1765                         ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1766                         ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1767                         ops[i]->ldpc_enc.tb_params.cab =
1768                                         ldpc_enc->tb_params.cab;
1769                         ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1770                         ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1771                 } else {
1772                         ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1773                 }
1774                 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1775                 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1776                 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1777                 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1778                 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1779                 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1780                 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1781                 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1782                 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1783                 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1784         }
1785 }
1786
1787 static int
1788 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1789                 unsigned int order_idx, const int expected_status)
1790 {
1791         int status = op->status;
1792         /* ignore parity mismatch false alarms for long iterations */
1793         if (get_iter_max() >= 10) {
1794                 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1795                                 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1796                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1797                         status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1798                 }
1799                 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1800                                 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1801                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1802                         status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1803                 }
1804         }
1805
1806         TEST_ASSERT(status == expected_status,
1807                         "op_status (%d) != expected_status (%d)",
1808                         op->status, expected_status);
1809
1810         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1811                         "Ordering error, expected %p, got %p",
1812                         (void *)(uintptr_t)order_idx, op->opaque_data);
1813
1814         return TEST_SUCCESS;
1815 }
1816
1817 static int
1818 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1819                 unsigned int order_idx, const int expected_status)
1820 {
1821         TEST_ASSERT(op->status == expected_status,
1822                         "op_status (%d) != expected_status (%d)",
1823                         op->status, expected_status);
1824
1825         if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1826                 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1827                                 "Ordering error, expected %p, got %p",
1828                                 (void *)(uintptr_t)order_idx, op->opaque_data);
1829
1830         return TEST_SUCCESS;
1831 }
1832
1833 static inline int
1834 validate_op_chain(struct rte_bbdev_op_data *op,
1835                 struct op_data_entries *orig_op)
1836 {
1837         uint8_t i;
1838         struct rte_mbuf *m = op->data;
1839         uint8_t nb_dst_segments = orig_op->nb_segments;
1840         uint32_t total_data_size = 0;
1841
1842         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1843                         "Number of segments differ in original (%u) and filled (%u) op",
1844                         nb_dst_segments, m->nb_segs);
1845
1846         /* Validate each mbuf segment length */
1847         for (i = 0; i < nb_dst_segments; ++i) {
1848                 /* Apply offset to the first mbuf segment */
1849                 uint16_t offset = (i == 0) ? op->offset : 0;
1850                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1851                 total_data_size += orig_op->segments[i].length;
1852
1853                 TEST_ASSERT(orig_op->segments[i].length == data_len,
1854                                 "Length of segment differ in original (%u) and filled (%u) op",
1855                                 orig_op->segments[i].length, data_len);
1856                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1857                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1858                                 data_len,
1859                                 "Output buffers (CB=%u) are not equal", i);
1860                 m = m->next;
1861         }
1862
1863         /* Validate total mbuf pkt length */
1864         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1865         TEST_ASSERT(total_data_size == pkt_len,
1866                         "Length of data differ in original (%u) and filled (%u) op",
1867                         total_data_size, pkt_len);
1868
1869         return TEST_SUCCESS;
1870 }
1871
1872 /*
1873  * Compute K0 for a given configuration for HARQ output length computation
1874  * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1875  */
1876 static inline uint16_t
1877 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1878 {
1879         if (rv_index == 0)
1880                 return 0;
1881         uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1882         if (n_cb == n) {
1883                 if (rv_index == 1)
1884                         return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1885                 else if (rv_index == 2)
1886                         return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1887                 else
1888                         return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1889         }
1890         /* LBRM case - includes a division by N */
1891         if (rv_index == 1)
1892                 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1893                                 / n) * z_c;
1894         else if (rv_index == 2)
1895                 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1896                                 / n) * z_c;
1897         else
1898                 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1899                                 / n) * z_c;
1900 }
1901
1902 /* HARQ output length including the Filler bits */
1903 static inline uint16_t
1904 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1905 {
1906         uint16_t k0 = 0;
1907         uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1908         k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1909         /* Compute RM out size and number of rows */
1910         uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1911                         * ops_ld->z_c - ops_ld->n_filler;
1912         uint16_t deRmOutSize = RTE_MIN(
1913                         k0 + ops_ld->cb_params.e +
1914                         ((k0 > parity_offset) ?
1915                                         0 : ops_ld->n_filler),
1916                                         ops_ld->n_cb);
1917         uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1918                         / ops_ld->z_c);
1919         uint16_t harq_output_len = numRows * ops_ld->z_c;
1920         return harq_output_len;
1921 }
1922
1923 static inline int
1924 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1925                 struct op_data_entries *orig_op,
1926                 struct rte_bbdev_op_ldpc_dec *ops_ld)
1927 {
1928         uint8_t i;
1929         uint32_t j, jj, k;
1930         struct rte_mbuf *m = op->data;
1931         uint8_t nb_dst_segments = orig_op->nb_segments;
1932         uint32_t total_data_size = 0;
1933         int8_t *harq_orig, *harq_out, abs_harq_origin;
1934         uint32_t byte_error = 0, cum_error = 0, error;
1935         int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1936         int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1937         uint16_t parity_offset;
1938
1939         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1940                         "Number of segments differ in original (%u) and filled (%u) op",
1941                         nb_dst_segments, m->nb_segs);
1942
1943         /* Validate each mbuf segment length */
1944         for (i = 0; i < nb_dst_segments; ++i) {
1945                 /* Apply offset to the first mbuf segment */
1946                 uint16_t offset = (i == 0) ? op->offset : 0;
1947                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1948                 total_data_size += orig_op->segments[i].length;
1949
1950                 TEST_ASSERT(orig_op->segments[i].length <
1951                                 (uint32_t)(data_len + 64),
1952                                 "Length of segment differ in original (%u) and filled (%u) op",
1953                                 orig_op->segments[i].length, data_len);
1954                 harq_orig = (int8_t *) orig_op->segments[i].addr;
1955                 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1956
1957                 if (!(ldpc_cap_flags &
1958                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1959                                 ) || (ops_ld->op_flags &
1960                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1961                         data_len -= ops_ld->z_c;
1962                         parity_offset = data_len;
1963                 } else {
1964                         /* Compute RM out size and number of rows */
1965                         parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1966                                         * ops_ld->z_c - ops_ld->n_filler;
1967                         uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1968                                         ops_ld->n_filler;
1969                         if (data_len > deRmOutSize)
1970                                 data_len = deRmOutSize;
1971                         if (data_len > orig_op->segments[i].length)
1972                                 data_len = orig_op->segments[i].length;
1973                 }
1974                 /*
1975                  * HARQ output can have minor differences
1976                  * due to integer representation and related scaling
1977                  */
1978                 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1979                         if (j == parity_offset) {
1980                                 /* Special Handling of the filler bits */
1981                                 for (k = 0; k < ops_ld->n_filler; k++) {
1982                                         if (harq_out[jj] !=
1983                                                         llr_max_pre_scaling) {
1984                                                 printf("HARQ Filler issue %d: %d %d\n",
1985                                                         jj, harq_out[jj],
1986                                                         llr_max);
1987                                                 byte_error++;
1988                                         }
1989                                         jj++;
1990                                 }
1991                         }
1992                         if (!(ops_ld->op_flags &
1993                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1994                                 if (ldpc_llr_decimals > 1)
1995                                         harq_out[jj] = (harq_out[jj] + 1)
1996                                                 >> (ldpc_llr_decimals - 1);
1997                                 /* Saturated to S7 */
1998                                 if (harq_orig[j] > llr_max)
1999                                         harq_orig[j] = llr_max;
2000                                 if (harq_orig[j] < -llr_max)
2001                                         harq_orig[j] = -llr_max;
2002                         }
2003                         if (harq_orig[j] != harq_out[jj]) {
2004                                 error = (harq_orig[j] > harq_out[jj]) ?
2005                                                 harq_orig[j] - harq_out[jj] :
2006                                                 harq_out[jj] - harq_orig[j];
2007                                 abs_harq_origin = harq_orig[j] > 0 ?
2008                                                         harq_orig[j] :
2009                                                         -harq_orig[j];
2010                                 /* Residual quantization error */
2011                                 if ((error > 8 && (abs_harq_origin <
2012                                                 (llr_max - 16))) ||
2013                                                 (error > 16)) {
2014                                         printf("HARQ mismatch %d: exp %d act %d => %d\n",
2015                                                         j, harq_orig[j],
2016                                                         harq_out[jj], error);
2017                                         byte_error++;
2018                                         cum_error += error;
2019                                 }
2020                         }
2021                 }
2022                 m = m->next;
2023         }
2024
2025         if (byte_error)
2026                 TEST_ASSERT(byte_error <= 1,
2027                                 "HARQ output mismatch (%d) %d",
2028                                 byte_error, cum_error);
2029
2030         /* Validate total mbuf pkt length */
2031         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
2032         TEST_ASSERT(total_data_size < pkt_len + 64,
2033                         "Length of data differ in original (%u) and filled (%u) op",
2034                         total_data_size, pkt_len);
2035
2036         return TEST_SUCCESS;
2037 }
2038
2039 static int
2040 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2041                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2042 {
2043         unsigned int i;
2044         int ret;
2045         struct op_data_entries *hard_data_orig =
2046                         &test_vector.entries[DATA_HARD_OUTPUT];
2047         struct op_data_entries *soft_data_orig =
2048                         &test_vector.entries[DATA_SOFT_OUTPUT];
2049         struct rte_bbdev_op_turbo_dec *ops_td;
2050         struct rte_bbdev_op_data *hard_output;
2051         struct rte_bbdev_op_data *soft_output;
2052         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
2053
2054         for (i = 0; i < n; ++i) {
2055                 ops_td = &ops[i]->turbo_dec;
2056                 hard_output = &ops_td->hard_output;
2057                 soft_output = &ops_td->soft_output;
2058
2059                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2060                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2061                                         "Returned iter_count (%d) > expected iter_count (%d)",
2062                                         ops_td->iter_count, ref_td->iter_count);
2063                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2064                 TEST_ASSERT_SUCCESS(ret,
2065                                 "Checking status and ordering for decoder failed");
2066
2067                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2068                                 hard_data_orig),
2069                                 "Hard output buffers (CB=%u) are not equal",
2070                                 i);
2071
2072                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
2073                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2074                                         soft_data_orig),
2075                                         "Soft output buffers (CB=%u) are not equal",
2076                                         i);
2077         }
2078
2079         return TEST_SUCCESS;
2080 }
2081
2082 /* Check Number of code blocks errors */
2083 static int
2084 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
2085 {
2086         unsigned int i;
2087         struct op_data_entries *hard_data_orig =
2088                         &test_vector.entries[DATA_HARD_OUTPUT];
2089         struct rte_bbdev_op_ldpc_dec *ops_td;
2090         struct rte_bbdev_op_data *hard_output;
2091         int errors = 0;
2092         struct rte_mbuf *m;
2093
2094         for (i = 0; i < n; ++i) {
2095                 ops_td = &ops[i]->ldpc_dec;
2096                 hard_output = &ops_td->hard_output;
2097                 m = hard_output->data;
2098                 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
2099                                 hard_data_orig->segments[0].addr,
2100                                 hard_data_orig->segments[0].length))
2101                         errors++;
2102         }
2103         return errors;
2104 }
2105
2106 static int
2107 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2108                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2109 {
2110         unsigned int i;
2111         int ret;
2112         struct op_data_entries *hard_data_orig =
2113                         &test_vector.entries[DATA_HARD_OUTPUT];
2114         struct op_data_entries *soft_data_orig =
2115                         &test_vector.entries[DATA_SOFT_OUTPUT];
2116         struct op_data_entries *harq_data_orig =
2117                                 &test_vector.entries[DATA_HARQ_OUTPUT];
2118         struct rte_bbdev_op_ldpc_dec *ops_td;
2119         struct rte_bbdev_op_data *hard_output;
2120         struct rte_bbdev_op_data *harq_output;
2121         struct rte_bbdev_op_data *soft_output;
2122         struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2123
2124         for (i = 0; i < n; ++i) {
2125                 ops_td = &ops[i]->ldpc_dec;
2126                 hard_output = &ops_td->hard_output;
2127                 harq_output = &ops_td->harq_combined_output;
2128                 soft_output = &ops_td->soft_output;
2129
2130                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2131                 TEST_ASSERT_SUCCESS(ret,
2132                                 "Checking status and ordering for decoder failed");
2133                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2134                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2135                                         "Returned iter_count (%d) > expected iter_count (%d)",
2136                                         ops_td->iter_count, ref_td->iter_count);
2137                 /*
2138                  * We can ignore output data when the decoding failed to
2139                  * converge or for loop-back cases
2140                  */
2141                 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2142                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2143                                 ) && (
2144                                 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2145                                                 )) == 0)
2146                         TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2147                                         hard_data_orig),
2148                                         "Hard output buffers (CB=%u) are not equal",
2149                                         i);
2150
2151                 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2152                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2153                                         soft_data_orig),
2154                                         "Soft output buffers (CB=%u) are not equal",
2155                                         i);
2156                 if (ref_op->ldpc_dec.op_flags &
2157                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2158                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2159                                         harq_data_orig, ops_td),
2160                                         "HARQ output buffers (CB=%u) are not equal",
2161                                         i);
2162                 }
2163                 if (ref_op->ldpc_dec.op_flags &
2164                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2165                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2166                                         harq_data_orig, ops_td),
2167                                         "HARQ output buffers (CB=%u) are not equal",
2168                                         i);
2169
2170         }
2171
2172         return TEST_SUCCESS;
2173 }
2174
2175
2176 static int
2177 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2178                 struct rte_bbdev_enc_op *ref_op)
2179 {
2180         unsigned int i;
2181         int ret;
2182         struct op_data_entries *hard_data_orig =
2183                         &test_vector.entries[DATA_HARD_OUTPUT];
2184
2185         for (i = 0; i < n; ++i) {
2186                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2187                 TEST_ASSERT_SUCCESS(ret,
2188                                 "Checking status and ordering for encoder failed");
2189                 TEST_ASSERT_SUCCESS(validate_op_chain(
2190                                 &ops[i]->turbo_enc.output,
2191                                 hard_data_orig),
2192                                 "Output buffers (CB=%u) are not equal",
2193                                 i);
2194         }
2195
2196         return TEST_SUCCESS;
2197 }
2198
2199 static int
2200 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2201                 struct rte_bbdev_enc_op *ref_op)
2202 {
2203         unsigned int i;
2204         int ret;
2205         struct op_data_entries *hard_data_orig =
2206                         &test_vector.entries[DATA_HARD_OUTPUT];
2207
2208         for (i = 0; i < n; ++i) {
2209                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2210                 TEST_ASSERT_SUCCESS(ret,
2211                                 "Checking status and ordering for encoder failed");
2212                 TEST_ASSERT_SUCCESS(validate_op_chain(
2213                                 &ops[i]->ldpc_enc.output,
2214                                 hard_data_orig),
2215                                 "Output buffers (CB=%u) are not equal",
2216                                 i);
2217         }
2218
2219         return TEST_SUCCESS;
2220 }
2221
2222 static void
2223 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2224 {
2225         unsigned int i;
2226         struct op_data_entries *entry;
2227
2228         op->turbo_dec = test_vector.turbo_dec;
2229         entry = &test_vector.entries[DATA_INPUT];
2230         for (i = 0; i < entry->nb_segments; ++i)
2231                 op->turbo_dec.input.length +=
2232                                 entry->segments[i].length;
2233 }
2234
2235 static void
2236 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2237 {
2238         unsigned int i;
2239         struct op_data_entries *entry;
2240
2241         op->ldpc_dec = test_vector.ldpc_dec;
2242         entry = &test_vector.entries[DATA_INPUT];
2243         for (i = 0; i < entry->nb_segments; ++i)
2244                 op->ldpc_dec.input.length +=
2245                                 entry->segments[i].length;
2246         if (test_vector.ldpc_dec.op_flags &
2247                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2248                 entry = &test_vector.entries[DATA_HARQ_INPUT];
2249                 for (i = 0; i < entry->nb_segments; ++i)
2250                         op->ldpc_dec.harq_combined_input.length +=
2251                                 entry->segments[i].length;
2252         }
2253 }
2254
2255
2256 static void
2257 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2258 {
2259         unsigned int i;
2260         struct op_data_entries *entry;
2261
2262         op->turbo_enc = test_vector.turbo_enc;
2263         entry = &test_vector.entries[DATA_INPUT];
2264         for (i = 0; i < entry->nb_segments; ++i)
2265                 op->turbo_enc.input.length +=
2266                                 entry->segments[i].length;
2267 }
2268
2269 static void
2270 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2271 {
2272         unsigned int i;
2273         struct op_data_entries *entry;
2274
2275         op->ldpc_enc = test_vector.ldpc_enc;
2276         entry = &test_vector.entries[DATA_INPUT];
2277         for (i = 0; i < entry->nb_segments; ++i)
2278                 op->ldpc_enc.input.length +=
2279                                 entry->segments[i].length;
2280 }
2281
2282 static uint32_t
2283 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2284 {
2285         uint8_t i;
2286         uint32_t c, r, tb_size = 0;
2287
2288         if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2289                 tb_size = op->turbo_dec.tb_params.k_neg;
2290         } else {
2291                 c = op->turbo_dec.tb_params.c;
2292                 r = op->turbo_dec.tb_params.r;
2293                 for (i = 0; i < c-r; i++)
2294                         tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2295                                 op->turbo_dec.tb_params.k_neg :
2296                                 op->turbo_dec.tb_params.k_pos;
2297         }
2298         return tb_size;
2299 }
2300
2301 static uint32_t
2302 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2303 {
2304         uint8_t i;
2305         uint32_t c, r, tb_size = 0;
2306         uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2307
2308         if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2309                 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2310         } else {
2311                 c = op->ldpc_dec.tb_params.c;
2312                 r = op->ldpc_dec.tb_params.r;
2313                 for (i = 0; i < c-r; i++)
2314                         tb_size += sys_cols * op->ldpc_dec.z_c
2315                                         - op->ldpc_dec.n_filler;
2316         }
2317         return tb_size;
2318 }
2319
2320 static uint32_t
2321 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2322 {
2323         uint8_t i;
2324         uint32_t c, r, tb_size = 0;
2325
2326         if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2327                 tb_size = op->turbo_enc.tb_params.k_neg;
2328         } else {
2329                 c = op->turbo_enc.tb_params.c;
2330                 r = op->turbo_enc.tb_params.r;
2331                 for (i = 0; i < c-r; i++)
2332                         tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2333                                 op->turbo_enc.tb_params.k_neg :
2334                                 op->turbo_enc.tb_params.k_pos;
2335         }
2336         return tb_size;
2337 }
2338
2339 static uint32_t
2340 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2341 {
2342         uint8_t i;
2343         uint32_t c, r, tb_size = 0;
2344         uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2345
2346         if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2347                 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2348         } else {
2349                 c = op->turbo_enc.tb_params.c;
2350                 r = op->turbo_enc.tb_params.r;
2351                 for (i = 0; i < c-r; i++)
2352                         tb_size += sys_cols * op->ldpc_enc.z_c
2353                                         - op->ldpc_enc.n_filler;
2354         }
2355         return tb_size;
2356 }
2357
2358
2359 static int
2360 init_test_op_params(struct test_op_params *op_params,
2361                 enum rte_bbdev_op_type op_type, const int expected_status,
2362                 const int vector_mask, struct rte_mempool *ops_mp,
2363                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2364 {
2365         int ret = 0;
2366         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2367                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2368                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2369                                 &op_params->ref_dec_op, 1);
2370         else
2371                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2372                                 &op_params->ref_enc_op, 1);
2373
2374         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2375
2376         op_params->mp = ops_mp;
2377         op_params->burst_sz = burst_sz;
2378         op_params->num_to_process = num_to_process;
2379         op_params->num_lcores = num_lcores;
2380         op_params->vector_mask = vector_mask;
2381         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2382                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2383                 op_params->ref_dec_op->status = expected_status;
2384         else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2385                         || op_type == RTE_BBDEV_OP_LDPC_ENC)
2386                 op_params->ref_enc_op->status = expected_status;
2387         return 0;
2388 }
2389
2390 static int
2391 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2392                 struct test_op_params *op_params)
2393 {
2394         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2395         unsigned int i;
2396         struct active_device *ad;
2397         unsigned int burst_sz = get_burst_sz();
2398         enum rte_bbdev_op_type op_type = test_vector.op_type;
2399         const struct rte_bbdev_op_cap *capabilities = NULL;
2400
2401         ad = &active_devs[dev_id];
2402
2403         /* Check if device supports op_type */
2404         if (!is_avail_op(ad, test_vector.op_type))
2405                 return TEST_SUCCESS;
2406
2407         struct rte_bbdev_info info;
2408         rte_bbdev_info_get(ad->dev_id, &info);
2409         socket_id = GET_SOCKET(info.socket_id);
2410
2411         f_ret = create_mempools(ad, socket_id, op_type,
2412                         get_num_ops());
2413         if (f_ret != TEST_SUCCESS) {
2414                 printf("Couldn't create mempools");
2415                 goto fail;
2416         }
2417         if (op_type == RTE_BBDEV_OP_NONE)
2418                 op_type = RTE_BBDEV_OP_TURBO_ENC;
2419
2420         f_ret = init_test_op_params(op_params, test_vector.op_type,
2421                         test_vector.expected_status,
2422                         test_vector.mask,
2423                         ad->ops_mempool,
2424                         burst_sz,
2425                         get_num_ops(),
2426                         get_num_lcores());
2427         if (f_ret != TEST_SUCCESS) {
2428                 printf("Couldn't init test op params");
2429                 goto fail;
2430         }
2431
2432
2433         /* Find capabilities */
2434         const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2435         for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2436                 if (cap->type == test_vector.op_type) {
2437                         capabilities = cap;
2438                         break;
2439                 }
2440                 cap++;
2441         }
2442         TEST_ASSERT_NOT_NULL(capabilities,
2443                         "Couldn't find capabilities");
2444
2445         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2446                 create_reference_dec_op(op_params->ref_dec_op);
2447         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2448                 create_reference_enc_op(op_params->ref_enc_op);
2449         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2450                 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2451         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2452                 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2453
2454         for (i = 0; i < ad->nb_queues; ++i) {
2455                 f_ret = fill_queue_buffers(op_params,
2456                                 ad->in_mbuf_pool,
2457                                 ad->hard_out_mbuf_pool,
2458                                 ad->soft_out_mbuf_pool,
2459                                 ad->harq_in_mbuf_pool,
2460                                 ad->harq_out_mbuf_pool,
2461                                 ad->queue_ids[i],
2462                                 capabilities,
2463                                 info.drv.min_alignment,
2464                                 socket_id);
2465                 if (f_ret != TEST_SUCCESS) {
2466                         printf("Couldn't init queue buffers");
2467                         goto fail;
2468                 }
2469         }
2470
2471         /* Run test case function */
2472         t_ret = test_case_func(ad, op_params);
2473
2474         /* Free active device resources and return */
2475         free_buffers(ad, op_params);
2476         return t_ret;
2477
2478 fail:
2479         free_buffers(ad, op_params);
2480         return TEST_FAILED;
2481 }
2482
2483 /* Run given test function per active device per supported op type
2484  * per burst size.
2485  */
2486 static int
2487 run_test_case(test_case_function *test_case_func)
2488 {
2489         int ret = 0;
2490         uint8_t dev;
2491
2492         /* Alloc op_params */
2493         struct test_op_params *op_params = rte_zmalloc(NULL,
2494                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2495         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2496                         RTE_ALIGN(sizeof(struct test_op_params),
2497                                 RTE_CACHE_LINE_SIZE));
2498
2499         /* For each device run test case function */
2500         for (dev = 0; dev < nb_active_devs; ++dev)
2501                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2502
2503         rte_free(op_params);
2504
2505         return ret;
2506 }
2507
2508
2509 /* Push back the HARQ output from DDR to host */
2510 static void
2511 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2512                 struct rte_bbdev_dec_op **ops,
2513                 const uint16_t n)
2514 {
2515         uint16_t j;
2516         int save_status, ret;
2517         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2518         struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2519         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2520         bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2521         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2522         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2523         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2524         for (j = 0; j < n; ++j) {
2525                 if ((loopback && mem_out) || hc_out) {
2526                         save_status = ops[j]->status;
2527                         ops[j]->ldpc_dec.op_flags =
2528                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2529                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2530                         if (h_comp)
2531                                 ops[j]->ldpc_dec.op_flags +=
2532                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2533                         ops[j]->ldpc_dec.harq_combined_input.offset =
2534                                         harq_offset;
2535                         ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2536                         harq_offset += HARQ_INCR;
2537                         if (!loopback)
2538                                 ops[j]->ldpc_dec.harq_combined_input.length =
2539                                 ops[j]->ldpc_dec.harq_combined_output.length;
2540                         rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2541                                         &ops[j], 1);
2542                         ret = 0;
2543                         while (ret == 0)
2544                                 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2545                                                 dev_id, queue_id,
2546                                                 &ops_deq[j], 1);
2547                         ops[j]->ldpc_dec.op_flags = flags;
2548                         ops[j]->status = save_status;
2549                 }
2550         }
2551 }
2552
2553 /*
2554  * Push back the HARQ output from HW DDR to Host
2555  * Preload HARQ memory input and adjust HARQ offset
2556  */
2557 static void
2558 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2559                 struct rte_bbdev_dec_op **ops, const uint16_t n,
2560                 bool preload)
2561 {
2562         uint16_t j;
2563         int deq;
2564         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2565         struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS];
2566         struct rte_bbdev_dec_op *ops_deq[MAX_OPS];
2567         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2568         bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2569         bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2570         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2571         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2572         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2573         if ((mem_in || hc_in) && preload) {
2574                 for (j = 0; j < n; ++j) {
2575                         save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input;
2576                         save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output;
2577                         ops[j]->ldpc_dec.op_flags =
2578                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2579                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2580                         if (h_comp)
2581                                 ops[j]->ldpc_dec.op_flags +=
2582                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2583                         ops[j]->ldpc_dec.harq_combined_output.offset =
2584                                         harq_offset;
2585                         ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2586                         harq_offset += HARQ_INCR;
2587                 }
2588                 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n);
2589                 deq = 0;
2590                 while (deq != n)
2591                         deq += rte_bbdev_dequeue_ldpc_dec_ops(
2592                                         dev_id, queue_id, &ops_deq[deq],
2593                                         n - deq);
2594                 /* Restore the operations */
2595                 for (j = 0; j < n; ++j) {
2596                         ops[j]->ldpc_dec.op_flags = flags;
2597                         ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j];
2598                         ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j];
2599                 }
2600         }
2601         harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2602         for (j = 0; j < n; ++j) {
2603                 /* Adjust HARQ offset when we reach external DDR */
2604                 if (mem_in || hc_in)
2605                         ops[j]->ldpc_dec.harq_combined_input.offset
2606                                 = harq_offset;
2607                 if (mem_out || hc_out)
2608                         ops[j]->ldpc_dec.harq_combined_output.offset
2609                                 = harq_offset;
2610                 harq_offset += HARQ_INCR;
2611         }
2612 }
2613
2614 static void
2615 dequeue_event_callback(uint16_t dev_id,
2616                 enum rte_bbdev_event_type event, void *cb_arg,
2617                 void *ret_param)
2618 {
2619         int ret;
2620         uint16_t i;
2621         uint64_t total_time;
2622         uint16_t deq, burst_sz, num_ops;
2623         uint16_t queue_id = *(uint16_t *) ret_param;
2624         struct rte_bbdev_info info;
2625         double tb_len_bits;
2626         struct thread_params *tp = cb_arg;
2627
2628         /* Find matching thread params using queue_id */
2629         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2630                 if (tp->queue_id == queue_id)
2631                         break;
2632
2633         if (i == MAX_QUEUES) {
2634                 printf("%s: Queue_id from interrupt details was not found!\n",
2635                                 __func__);
2636                 return;
2637         }
2638
2639         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2640                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2641                 printf(
2642                         "Dequeue interrupt handler called for incorrect event!\n");
2643                 return;
2644         }
2645
2646         burst_sz = rte_atomic16_read(&tp->burst_sz);
2647         num_ops = tp->op_params->num_to_process;
2648
2649         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2650                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2651                                 &tp->dec_ops[
2652                                         rte_atomic16_read(&tp->nb_dequeued)],
2653                                 burst_sz);
2654         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2655                 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2656                                 &tp->dec_ops[
2657                                         rte_atomic16_read(&tp->nb_dequeued)],
2658                                 burst_sz);
2659         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2660                 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2661                                 &tp->enc_ops[
2662                                         rte_atomic16_read(&tp->nb_dequeued)],
2663                                 burst_sz);
2664         else /*RTE_BBDEV_OP_TURBO_ENC*/
2665                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2666                                 &tp->enc_ops[
2667                                         rte_atomic16_read(&tp->nb_dequeued)],
2668                                 burst_sz);
2669
2670         if (deq < burst_sz) {
2671                 printf(
2672                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2673                         burst_sz, deq);
2674                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2675                 return;
2676         }
2677
2678         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2679                 rte_atomic16_add(&tp->nb_dequeued, deq);
2680                 return;
2681         }
2682
2683         total_time = rte_rdtsc_precise() - tp->start_time;
2684
2685         rte_bbdev_info_get(dev_id, &info);
2686
2687         ret = TEST_SUCCESS;
2688
2689         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2690                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2691                 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2692                                 tp->op_params->vector_mask);
2693                 /* get the max of iter_count for all dequeued ops */
2694                 for (i = 0; i < num_ops; ++i)
2695                         tp->iter_count = RTE_MAX(
2696                                         tp->dec_ops[i]->turbo_dec.iter_count,
2697                                         tp->iter_count);
2698                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2699         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2700                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2701                 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2702                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2703         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2704                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2705                 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2706                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2707         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2708                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2709                 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2710                                 tp->op_params->vector_mask);
2711                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2712         }
2713
2714         if (ret) {
2715                 printf("Buffers validation failed\n");
2716                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2717         }
2718
2719         switch (test_vector.op_type) {
2720         case RTE_BBDEV_OP_TURBO_DEC:
2721                 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2722                 break;
2723         case RTE_BBDEV_OP_TURBO_ENC:
2724                 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2725                 break;
2726         case RTE_BBDEV_OP_LDPC_DEC:
2727                 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2728                 break;
2729         case RTE_BBDEV_OP_LDPC_ENC:
2730                 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2731                 break;
2732         case RTE_BBDEV_OP_NONE:
2733                 tb_len_bits = 0.0;
2734                 break;
2735         default:
2736                 printf("Unknown op type: %d\n", test_vector.op_type);
2737                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2738                 return;
2739         }
2740
2741         tp->ops_per_sec += ((double)num_ops) /
2742                         ((double)total_time / (double)rte_get_tsc_hz());
2743         tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2744                         ((double)total_time / (double)rte_get_tsc_hz());
2745
2746         rte_atomic16_add(&tp->nb_dequeued, deq);
2747 }
2748
2749 static int
2750 throughput_intr_lcore_ldpc_dec(void *arg)
2751 {
2752         struct thread_params *tp = arg;
2753         unsigned int enqueued;
2754         const uint16_t queue_id = tp->queue_id;
2755         const uint16_t burst_sz = tp->op_params->burst_sz;
2756         const uint16_t num_to_process = tp->op_params->num_to_process;
2757         struct rte_bbdev_dec_op *ops[num_to_process];
2758         struct test_buffers *bufs = NULL;
2759         struct rte_bbdev_info info;
2760         int ret, i, j;
2761         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2762         uint16_t num_to_enq, enq;
2763
2764         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2765                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2766         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2767                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2768
2769         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2770                         "BURST_SIZE should be <= %u", MAX_BURST);
2771
2772         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2773                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2774                         tp->dev_id, queue_id);
2775
2776         rte_bbdev_info_get(tp->dev_id, &info);
2777
2778         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2779                         "NUM_OPS cannot exceed %u for this device",
2780                         info.drv.queue_size_lim);
2781
2782         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2783
2784         rte_atomic16_clear(&tp->processing_status);
2785         rte_atomic16_clear(&tp->nb_dequeued);
2786
2787         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2788                 rte_pause();
2789
2790         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2791                                 num_to_process);
2792         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2793                         num_to_process);
2794         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2795                 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2796                                 bufs->hard_outputs, bufs->soft_outputs,
2797                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2798
2799         /* Set counter to validate the ordering */
2800         for (j = 0; j < num_to_process; ++j)
2801                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2802
2803         for (j = 0; j < TEST_REPETITIONS; ++j) {
2804                 for (i = 0; i < num_to_process; ++i) {
2805                         if (!loopback)
2806                                 rte_pktmbuf_reset(
2807                                         ops[i]->ldpc_dec.hard_output.data);
2808                         if (hc_out || loopback)
2809                                 mbuf_reset(
2810                                 ops[i]->ldpc_dec.harq_combined_output.data);
2811                 }
2812
2813                 tp->start_time = rte_rdtsc_precise();
2814                 for (enqueued = 0; enqueued < num_to_process;) {
2815                         num_to_enq = burst_sz;
2816
2817                         if (unlikely(num_to_process - enqueued < num_to_enq))
2818                                 num_to_enq = num_to_process - enqueued;
2819
2820                         enq = 0;
2821                         do {
2822                                 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2823                                                 tp->dev_id,
2824                                                 queue_id, &ops[enqueued],
2825                                                 num_to_enq);
2826                         } while (unlikely(num_to_enq != enq));
2827                         enqueued += enq;
2828
2829                         /* Write to thread burst_sz current number of enqueued
2830                          * descriptors. It ensures that proper number of
2831                          * descriptors will be dequeued in callback
2832                          * function - needed for last batch in case where
2833                          * the number of operations is not a multiple of
2834                          * burst size.
2835                          */
2836                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2837
2838                         /* Wait until processing of previous batch is
2839                          * completed
2840                          */
2841                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2842                                         (int16_t) enqueued)
2843                                 rte_pause();
2844                 }
2845                 if (j != TEST_REPETITIONS - 1)
2846                         rte_atomic16_clear(&tp->nb_dequeued);
2847         }
2848
2849         return TEST_SUCCESS;
2850 }
2851
2852 static int
2853 throughput_intr_lcore_dec(void *arg)
2854 {
2855         struct thread_params *tp = arg;
2856         unsigned int enqueued;
2857         const uint16_t queue_id = tp->queue_id;
2858         const uint16_t burst_sz = tp->op_params->burst_sz;
2859         const uint16_t num_to_process = tp->op_params->num_to_process;
2860         struct rte_bbdev_dec_op *ops[num_to_process];
2861         struct test_buffers *bufs = NULL;
2862         struct rte_bbdev_info info;
2863         int ret, i, j;
2864         uint16_t num_to_enq, enq;
2865
2866         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2867                         "BURST_SIZE should be <= %u", MAX_BURST);
2868
2869         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2870                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2871                         tp->dev_id, queue_id);
2872
2873         rte_bbdev_info_get(tp->dev_id, &info);
2874
2875         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2876                         "NUM_OPS cannot exceed %u for this device",
2877                         info.drv.queue_size_lim);
2878
2879         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2880
2881         rte_atomic16_clear(&tp->processing_status);
2882         rte_atomic16_clear(&tp->nb_dequeued);
2883
2884         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2885                 rte_pause();
2886
2887         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2888                                 num_to_process);
2889         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2890                         num_to_process);
2891         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2892                 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2893                                 bufs->hard_outputs, bufs->soft_outputs,
2894                                 tp->op_params->ref_dec_op);
2895
2896         /* Set counter to validate the ordering */
2897         for (j = 0; j < num_to_process; ++j)
2898                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2899
2900         for (j = 0; j < TEST_REPETITIONS; ++j) {
2901                 for (i = 0; i < num_to_process; ++i)
2902                         rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2903
2904                 tp->start_time = rte_rdtsc_precise();
2905                 for (enqueued = 0; enqueued < num_to_process;) {
2906                         num_to_enq = burst_sz;
2907
2908                         if (unlikely(num_to_process - enqueued < num_to_enq))
2909                                 num_to_enq = num_to_process - enqueued;
2910
2911                         enq = 0;
2912                         do {
2913                                 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2914                                                 queue_id, &ops[enqueued],
2915                                                 num_to_enq);
2916                         } while (unlikely(num_to_enq != enq));
2917                         enqueued += enq;
2918
2919                         /* Write to thread burst_sz current number of enqueued
2920                          * descriptors. It ensures that proper number of
2921                          * descriptors will be dequeued in callback
2922                          * function - needed for last batch in case where
2923                          * the number of operations is not a multiple of
2924                          * burst size.
2925                          */
2926                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2927
2928                         /* Wait until processing of previous batch is
2929                          * completed
2930                          */
2931                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2932                                         (int16_t) enqueued)
2933                                 rte_pause();
2934                 }
2935                 if (j != TEST_REPETITIONS - 1)
2936                         rte_atomic16_clear(&tp->nb_dequeued);
2937         }
2938
2939         return TEST_SUCCESS;
2940 }
2941
2942 static int
2943 throughput_intr_lcore_enc(void *arg)
2944 {
2945         struct thread_params *tp = arg;
2946         unsigned int enqueued;
2947         const uint16_t queue_id = tp->queue_id;
2948         const uint16_t burst_sz = tp->op_params->burst_sz;
2949         const uint16_t num_to_process = tp->op_params->num_to_process;
2950         struct rte_bbdev_enc_op *ops[num_to_process];
2951         struct test_buffers *bufs = NULL;
2952         struct rte_bbdev_info info;
2953         int ret, i, j;
2954         uint16_t num_to_enq, enq;
2955
2956         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2957                         "BURST_SIZE should be <= %u", MAX_BURST);
2958
2959         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2960                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2961                         tp->dev_id, queue_id);
2962
2963         rte_bbdev_info_get(tp->dev_id, &info);
2964
2965         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2966                         "NUM_OPS cannot exceed %u for this device",
2967                         info.drv.queue_size_lim);
2968
2969         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2970
2971         rte_atomic16_clear(&tp->processing_status);
2972         rte_atomic16_clear(&tp->nb_dequeued);
2973
2974         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2975                 rte_pause();
2976
2977         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2978                         num_to_process);
2979         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2980                         num_to_process);
2981         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2982                 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2983                                 bufs->hard_outputs, tp->op_params->ref_enc_op);
2984
2985         /* Set counter to validate the ordering */
2986         for (j = 0; j < num_to_process; ++j)
2987                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2988
2989         for (j = 0; j < TEST_REPETITIONS; ++j) {
2990                 for (i = 0; i < num_to_process; ++i)
2991                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2992
2993                 tp->start_time = rte_rdtsc_precise();
2994                 for (enqueued = 0; enqueued < num_to_process;) {
2995                         num_to_enq = burst_sz;
2996
2997                         if (unlikely(num_to_process - enqueued < num_to_enq))
2998                                 num_to_enq = num_to_process - enqueued;
2999
3000                         enq = 0;
3001                         do {
3002                                 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3003                                                 queue_id, &ops[enqueued],
3004                                                 num_to_enq);
3005                         } while (unlikely(enq != num_to_enq));
3006                         enqueued += enq;
3007
3008                         /* Write to thread burst_sz current number of enqueued
3009                          * descriptors. It ensures that proper number of
3010                          * descriptors will be dequeued in callback
3011                          * function - needed for last batch in case where
3012                          * the number of operations is not a multiple of
3013                          * burst size.
3014                          */
3015                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
3016
3017                         /* Wait until processing of previous batch is
3018                          * completed
3019                          */
3020                         while (rte_atomic16_read(&tp->nb_dequeued) !=
3021                                         (int16_t) enqueued)
3022                                 rte_pause();
3023                 }
3024                 if (j != TEST_REPETITIONS - 1)
3025                         rte_atomic16_clear(&tp->nb_dequeued);
3026         }
3027
3028         return TEST_SUCCESS;
3029 }
3030
3031
3032 static int
3033 throughput_intr_lcore_ldpc_enc(void *arg)
3034 {
3035         struct thread_params *tp = arg;
3036         unsigned int enqueued;
3037         const uint16_t queue_id = tp->queue_id;
3038         const uint16_t burst_sz = tp->op_params->burst_sz;
3039         const uint16_t num_to_process = tp->op_params->num_to_process;
3040         struct rte_bbdev_enc_op *ops[num_to_process];
3041         struct test_buffers *bufs = NULL;
3042         struct rte_bbdev_info info;
3043         int ret, i, j;
3044         uint16_t num_to_enq, enq;
3045
3046         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3047                         "BURST_SIZE should be <= %u", MAX_BURST);
3048
3049         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
3050                         "Failed to enable interrupts for dev: %u, queue_id: %u",
3051                         tp->dev_id, queue_id);
3052
3053         rte_bbdev_info_get(tp->dev_id, &info);
3054
3055         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
3056                         "NUM_OPS cannot exceed %u for this device",
3057                         info.drv.queue_size_lim);
3058
3059         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3060
3061         rte_atomic16_clear(&tp->processing_status);
3062         rte_atomic16_clear(&tp->nb_dequeued);
3063
3064         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3065                 rte_pause();
3066
3067         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
3068                         num_to_process);
3069         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3070                         num_to_process);
3071         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3072                 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
3073                                 bufs->inputs, bufs->hard_outputs,
3074                                 tp->op_params->ref_enc_op);
3075
3076         /* Set counter to validate the ordering */
3077         for (j = 0; j < num_to_process; ++j)
3078                 ops[j]->opaque_data = (void *)(uintptr_t)j;
3079
3080         for (j = 0; j < TEST_REPETITIONS; ++j) {
3081                 for (i = 0; i < num_to_process; ++i)
3082                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
3083
3084                 tp->start_time = rte_rdtsc_precise();
3085                 for (enqueued = 0; enqueued < num_to_process;) {
3086                         num_to_enq = burst_sz;
3087
3088                         if (unlikely(num_to_process - enqueued < num_to_enq))
3089                                 num_to_enq = num_to_process - enqueued;
3090
3091                         enq = 0;
3092                         do {
3093                                 enq += rte_bbdev_enqueue_ldpc_enc_ops(
3094                                                 tp->dev_id,
3095                                                 queue_id, &ops[enqueued],
3096                                                 num_to_enq);
3097                         } while (unlikely(enq != num_to_enq));
3098                         enqueued += enq;
3099
3100                         /* Write to thread burst_sz current number of enqueued
3101                          * descriptors. It ensures that proper number of
3102                          * descriptors will be dequeued in callback
3103                          * function - needed for last batch in case where
3104                          * the number of operations is not a multiple of
3105                          * burst size.
3106                          */
3107                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
3108
3109                         /* Wait until processing of previous batch is
3110                          * completed
3111                          */
3112                         while (rte_atomic16_read(&tp->nb_dequeued) !=
3113                                         (int16_t) enqueued)
3114                                 rte_pause();
3115                 }
3116                 if (j != TEST_REPETITIONS - 1)
3117                         rte_atomic16_clear(&tp->nb_dequeued);
3118         }
3119
3120         return TEST_SUCCESS;
3121 }
3122
3123 static int
3124 throughput_pmd_lcore_dec(void *arg)
3125 {
3126         struct thread_params *tp = arg;
3127         uint16_t enq, deq;
3128         uint64_t total_time = 0, start_time;
3129         const uint16_t queue_id = tp->queue_id;
3130         const uint16_t burst_sz = tp->op_params->burst_sz;
3131         const uint16_t num_ops = tp->op_params->num_to_process;
3132         struct rte_bbdev_dec_op *ops_enq[num_ops];
3133         struct rte_bbdev_dec_op *ops_deq[num_ops];
3134         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3135         struct test_buffers *bufs = NULL;
3136         int i, j, ret;
3137         struct rte_bbdev_info info;
3138         uint16_t num_to_enq;
3139
3140         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3141                         "BURST_SIZE should be <= %u", MAX_BURST);
3142
3143         rte_bbdev_info_get(tp->dev_id, &info);
3144
3145         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3146                         "NUM_OPS cannot exceed %u for this device",
3147                         info.drv.queue_size_lim);
3148
3149         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3150
3151         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3152                 rte_pause();
3153
3154         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3155         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3156
3157         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3158                 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3159                                 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3160
3161         /* Set counter to validate the ordering */
3162         for (j = 0; j < num_ops; ++j)
3163                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3164
3165         for (i = 0; i < TEST_REPETITIONS; ++i) {
3166
3167                 for (j = 0; j < num_ops; ++j)
3168                         mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3169
3170                 start_time = rte_rdtsc_precise();
3171
3172                 for (enq = 0, deq = 0; enq < num_ops;) {
3173                         num_to_enq = burst_sz;
3174
3175                         if (unlikely(num_ops - enq < num_to_enq))
3176                                 num_to_enq = num_ops - enq;
3177
3178                         enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3179                                         queue_id, &ops_enq[enq], num_to_enq);
3180
3181                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3182                                         queue_id, &ops_deq[deq], enq - deq);
3183                 }
3184
3185                 /* dequeue the remaining */
3186                 while (deq < enq) {
3187                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3188                                         queue_id, &ops_deq[deq], enq - deq);
3189                 }
3190
3191                 total_time += rte_rdtsc_precise() - start_time;
3192         }
3193
3194         tp->iter_count = 0;
3195         /* get the max of iter_count for all dequeued ops */
3196         for (i = 0; i < num_ops; ++i) {
3197                 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3198                                 tp->iter_count);
3199         }
3200
3201         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3202                 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3203                                 tp->op_params->vector_mask);
3204                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3205         }
3206
3207         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3208
3209         double tb_len_bits = calc_dec_TB_size(ref_op);
3210
3211         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3212                         ((double)total_time / (double)rte_get_tsc_hz());
3213         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3214                         1000000.0) / ((double)total_time /
3215                         (double)rte_get_tsc_hz());
3216
3217         return TEST_SUCCESS;
3218 }
3219
3220 static int
3221 bler_pmd_lcore_ldpc_dec(void *arg)
3222 {
3223         struct thread_params *tp = arg;
3224         uint16_t enq, deq;
3225         uint64_t total_time = 0, start_time;
3226         const uint16_t queue_id = tp->queue_id;
3227         const uint16_t burst_sz = tp->op_params->burst_sz;
3228         const uint16_t num_ops = tp->op_params->num_to_process;
3229         struct rte_bbdev_dec_op *ops_enq[num_ops];
3230         struct rte_bbdev_dec_op *ops_deq[num_ops];
3231         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3232         struct test_buffers *bufs = NULL;
3233         int i, j, ret;
3234         float parity_bler = 0;
3235         struct rte_bbdev_info info;
3236         uint16_t num_to_enq;
3237         bool extDdr = check_bit(ldpc_cap_flags,
3238                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3239         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3240                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3241         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3242                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3243
3244         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3245                         "BURST_SIZE should be <= %u", MAX_BURST);
3246
3247         rte_bbdev_info_get(tp->dev_id, &info);
3248
3249         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3250                         "NUM_OPS cannot exceed %u for this device",
3251                         info.drv.queue_size_lim);
3252
3253         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3254
3255         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3256                 rte_pause();
3257
3258         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3259         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3260
3261         /* For BLER tests we need to enable early termination */
3262         if (!check_bit(ref_op->ldpc_dec.op_flags,
3263                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3264                 ref_op->ldpc_dec.op_flags +=
3265                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3266         ref_op->ldpc_dec.iter_max = get_iter_max();
3267         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3268
3269         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3270                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3271                                 bufs->hard_outputs, bufs->soft_outputs,
3272                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3273         generate_llr_input(num_ops, bufs->inputs, ref_op);
3274
3275         /* Set counter to validate the ordering */
3276         for (j = 0; j < num_ops; ++j)
3277                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3278
3279         for (i = 0; i < 1; ++i) { /* Could add more iterations */
3280                 for (j = 0; j < num_ops; ++j) {
3281                         if (!loopback)
3282                                 mbuf_reset(
3283                                 ops_enq[j]->ldpc_dec.hard_output.data);
3284                         if (hc_out || loopback)
3285                                 mbuf_reset(
3286                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3287                 }
3288                 if (extDdr)
3289                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3290                                         num_ops, true);
3291                 start_time = rte_rdtsc_precise();
3292
3293                 for (enq = 0, deq = 0; enq < num_ops;) {
3294                         num_to_enq = burst_sz;
3295
3296                         if (unlikely(num_ops - enq < num_to_enq))
3297                                 num_to_enq = num_ops - enq;
3298
3299                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3300                                         queue_id, &ops_enq[enq], num_to_enq);
3301
3302                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3303                                         queue_id, &ops_deq[deq], enq - deq);
3304                 }
3305
3306                 /* dequeue the remaining */
3307                 while (deq < enq) {
3308                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3309                                         queue_id, &ops_deq[deq], enq - deq);
3310                 }
3311
3312                 total_time += rte_rdtsc_precise() - start_time;
3313         }
3314
3315         tp->iter_count = 0;
3316         tp->iter_average = 0;
3317         /* get the max of iter_count for all dequeued ops */
3318         for (i = 0; i < num_ops; ++i) {
3319                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3320                                 tp->iter_count);
3321                 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3322                 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3323                         parity_bler += 1.0;
3324         }
3325
3326         parity_bler /= num_ops; /* This one is based on SYND */
3327         tp->iter_average /= num_ops;
3328         tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3329
3330         if (test_vector.op_type != RTE_BBDEV_OP_NONE
3331                         && tp->bler == 0
3332                         && parity_bler == 0
3333                         && !hc_out) {
3334                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3335                                 tp->op_params->vector_mask);
3336                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3337         }
3338
3339         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3340
3341         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3342         tp->ops_per_sec = ((double)num_ops * 1) /
3343                         ((double)total_time / (double)rte_get_tsc_hz());
3344         tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3345                         1000000.0) / ((double)total_time /
3346                         (double)rte_get_tsc_hz());
3347
3348         return TEST_SUCCESS;
3349 }
3350
3351 static int
3352 throughput_pmd_lcore_ldpc_dec(void *arg)
3353 {
3354         struct thread_params *tp = arg;
3355         uint16_t enq, deq;
3356         uint64_t total_time = 0, start_time;
3357         const uint16_t queue_id = tp->queue_id;
3358         const uint16_t burst_sz = tp->op_params->burst_sz;
3359         const uint16_t num_ops = tp->op_params->num_to_process;
3360         struct rte_bbdev_dec_op *ops_enq[num_ops];
3361         struct rte_bbdev_dec_op *ops_deq[num_ops];
3362         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3363         struct test_buffers *bufs = NULL;
3364         int i, j, ret;
3365         struct rte_bbdev_info info;
3366         uint16_t num_to_enq;
3367         bool extDdr = check_bit(ldpc_cap_flags,
3368                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3369         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3370                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3371         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3372                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3373
3374         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3375                         "BURST_SIZE should be <= %u", MAX_BURST);
3376
3377         rte_bbdev_info_get(tp->dev_id, &info);
3378
3379         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3380                         "NUM_OPS cannot exceed %u for this device",
3381                         info.drv.queue_size_lim);
3382
3383         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3384
3385         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3386                 rte_pause();
3387
3388         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3389         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3390
3391         /* For throughput tests we need to disable early termination */
3392         if (check_bit(ref_op->ldpc_dec.op_flags,
3393                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3394                 ref_op->ldpc_dec.op_flags -=
3395                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3396         ref_op->ldpc_dec.iter_max = get_iter_max();
3397         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3398
3399         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3400                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3401                                 bufs->hard_outputs, bufs->soft_outputs,
3402                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3403
3404         /* Set counter to validate the ordering */
3405         for (j = 0; j < num_ops; ++j)
3406                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3407
3408         for (i = 0; i < TEST_REPETITIONS; ++i) {
3409                 for (j = 0; j < num_ops; ++j) {
3410                         if (!loopback)
3411                                 mbuf_reset(
3412                                 ops_enq[j]->ldpc_dec.hard_output.data);
3413                         if (hc_out || loopback)
3414                                 mbuf_reset(
3415                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3416                 }
3417                 if (extDdr)
3418                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3419                                         num_ops, true);
3420                 start_time = rte_rdtsc_precise();
3421
3422                 for (enq = 0, deq = 0; enq < num_ops;) {
3423                         num_to_enq = burst_sz;
3424
3425                         if (unlikely(num_ops - enq < num_to_enq))
3426                                 num_to_enq = num_ops - enq;
3427
3428                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3429                                         queue_id, &ops_enq[enq], num_to_enq);
3430
3431                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3432                                         queue_id, &ops_deq[deq], enq - deq);
3433                 }
3434
3435                 /* dequeue the remaining */
3436                 while (deq < enq) {
3437                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3438                                         queue_id, &ops_deq[deq], enq - deq);
3439                 }
3440
3441                 total_time += rte_rdtsc_precise() - start_time;
3442         }
3443
3444         tp->iter_count = 0;
3445         /* get the max of iter_count for all dequeued ops */
3446         for (i = 0; i < num_ops; ++i) {
3447                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3448                                 tp->iter_count);
3449         }
3450         if (extDdr) {
3451                 /* Read loopback is not thread safe */
3452                 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3453         }
3454
3455         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3456                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3457                                 tp->op_params->vector_mask);
3458                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3459         }
3460
3461         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3462
3463         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3464
3465         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3466                         ((double)total_time / (double)rte_get_tsc_hz());
3467         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3468                         1000000.0) / ((double)total_time /
3469                         (double)rte_get_tsc_hz());
3470
3471         return TEST_SUCCESS;
3472 }
3473
3474 static int
3475 throughput_pmd_lcore_enc(void *arg)
3476 {
3477         struct thread_params *tp = arg;
3478         uint16_t enq, deq;
3479         uint64_t total_time = 0, start_time;
3480         const uint16_t queue_id = tp->queue_id;
3481         const uint16_t burst_sz = tp->op_params->burst_sz;
3482         const uint16_t num_ops = tp->op_params->num_to_process;
3483         struct rte_bbdev_enc_op *ops_enq[num_ops];
3484         struct rte_bbdev_enc_op *ops_deq[num_ops];
3485         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3486         struct test_buffers *bufs = NULL;
3487         int i, j, ret;
3488         struct rte_bbdev_info info;
3489         uint16_t num_to_enq;
3490
3491         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3492                         "BURST_SIZE should be <= %u", MAX_BURST);
3493
3494         rte_bbdev_info_get(tp->dev_id, &info);
3495
3496         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3497                         "NUM_OPS cannot exceed %u for this device",
3498                         info.drv.queue_size_lim);
3499
3500         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3501
3502         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3503                 rte_pause();
3504
3505         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3506                         num_ops);
3507         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3508                         num_ops);
3509         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3510                 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3511                                 bufs->hard_outputs, ref_op);
3512
3513         /* Set counter to validate the ordering */
3514         for (j = 0; j < num_ops; ++j)
3515                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3516
3517         for (i = 0; i < TEST_REPETITIONS; ++i) {
3518
3519                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3520                         for (j = 0; j < num_ops; ++j)
3521                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3522
3523                 start_time = rte_rdtsc_precise();
3524
3525                 for (enq = 0, deq = 0; enq < num_ops;) {
3526                         num_to_enq = burst_sz;
3527
3528                         if (unlikely(num_ops - enq < num_to_enq))
3529                                 num_to_enq = num_ops - enq;
3530
3531                         enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3532                                         queue_id, &ops_enq[enq], num_to_enq);
3533
3534                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3535                                         queue_id, &ops_deq[deq], enq - deq);
3536                 }
3537
3538                 /* dequeue the remaining */
3539                 while (deq < enq) {
3540                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3541                                         queue_id, &ops_deq[deq], enq - deq);
3542                 }
3543
3544                 total_time += rte_rdtsc_precise() - start_time;
3545         }
3546
3547         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3548                 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3549                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3550         }
3551
3552         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3553
3554         double tb_len_bits = calc_enc_TB_size(ref_op);
3555
3556         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3557                         ((double)total_time / (double)rte_get_tsc_hz());
3558         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3559                         / 1000000.0) / ((double)total_time /
3560                         (double)rte_get_tsc_hz());
3561
3562         return TEST_SUCCESS;
3563 }
3564
3565 static int
3566 throughput_pmd_lcore_ldpc_enc(void *arg)
3567 {
3568         struct thread_params *tp = arg;
3569         uint16_t enq, deq;
3570         uint64_t total_time = 0, start_time;
3571         const uint16_t queue_id = tp->queue_id;
3572         const uint16_t burst_sz = tp->op_params->burst_sz;
3573         const uint16_t num_ops = tp->op_params->num_to_process;
3574         struct rte_bbdev_enc_op *ops_enq[num_ops];
3575         struct rte_bbdev_enc_op *ops_deq[num_ops];
3576         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3577         struct test_buffers *bufs = NULL;
3578         int i, j, ret;
3579         struct rte_bbdev_info info;
3580         uint16_t num_to_enq;
3581
3582         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3583                         "BURST_SIZE should be <= %u", MAX_BURST);
3584
3585         rte_bbdev_info_get(tp->dev_id, &info);
3586
3587         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3588                         "NUM_OPS cannot exceed %u for this device",
3589                         info.drv.queue_size_lim);
3590
3591         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3592
3593         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3594                 rte_pause();
3595
3596         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3597                         num_ops);
3598         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3599                         num_ops);
3600         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3601                 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3602                                 bufs->hard_outputs, ref_op);
3603
3604         /* Set counter to validate the ordering */
3605         for (j = 0; j < num_ops; ++j)
3606                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3607
3608         for (i = 0; i < TEST_REPETITIONS; ++i) {
3609
3610                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3611                         for (j = 0; j < num_ops; ++j)
3612                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3613
3614                 start_time = rte_rdtsc_precise();
3615
3616                 for (enq = 0, deq = 0; enq < num_ops;) {
3617                         num_to_enq = burst_sz;
3618
3619                         if (unlikely(num_ops - enq < num_to_enq))
3620                                 num_to_enq = num_ops - enq;
3621
3622                         enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3623                                         queue_id, &ops_enq[enq], num_to_enq);
3624
3625                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3626                                         queue_id, &ops_deq[deq], enq - deq);
3627                 }
3628
3629                 /* dequeue the remaining */
3630                 while (deq < enq) {
3631                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3632                                         queue_id, &ops_deq[deq], enq - deq);
3633                 }
3634
3635                 total_time += rte_rdtsc_precise() - start_time;
3636         }
3637
3638         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3639                 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3640                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3641         }
3642
3643         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3644
3645         double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3646
3647         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3648                         ((double)total_time / (double)rte_get_tsc_hz());
3649         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3650                         / 1000000.0) / ((double)total_time /
3651                         (double)rte_get_tsc_hz());
3652
3653         return TEST_SUCCESS;
3654 }
3655
3656 static void
3657 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3658 {
3659         unsigned int iter = 0;
3660         double total_mops = 0, total_mbps = 0;
3661
3662         for (iter = 0; iter < used_cores; iter++) {
3663                 printf(
3664                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3665                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3666                         t_params[iter].mbps);
3667                 total_mops += t_params[iter].ops_per_sec;
3668                 total_mbps += t_params[iter].mbps;
3669         }
3670         printf(
3671                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3672                 used_cores, total_mops, total_mbps);
3673 }
3674
3675 /* Aggregate the performance results over the number of cores used */
3676 static void
3677 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3678 {
3679         unsigned int core_idx = 0;
3680         double total_mops = 0, total_mbps = 0;
3681         uint8_t iter_count = 0;
3682
3683         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3684                 printf(
3685                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3686                         t_params[core_idx].lcore_id,
3687                         t_params[core_idx].ops_per_sec,
3688                         t_params[core_idx].mbps,
3689                         t_params[core_idx].iter_count);
3690                 total_mops += t_params[core_idx].ops_per_sec;
3691                 total_mbps += t_params[core_idx].mbps;
3692                 iter_count = RTE_MAX(iter_count,
3693                                 t_params[core_idx].iter_count);
3694         }
3695         printf(
3696                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3697                 used_cores, total_mops, total_mbps, iter_count);
3698 }
3699
3700 /* Aggregate the performance results over the number of cores used */
3701 static void
3702 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3703 {
3704         unsigned int core_idx = 0;
3705         double total_mbps = 0, total_bler = 0, total_iter = 0;
3706         double snr = get_snr();
3707
3708         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3709                 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3710                                 t_params[core_idx].lcore_id,
3711                                 t_params[core_idx].bler * 100,
3712                                 t_params[core_idx].iter_average,
3713                                 t_params[core_idx].mbps,
3714                                 get_vector_filename());
3715                 total_mbps += t_params[core_idx].mbps;
3716                 total_bler += t_params[core_idx].bler;
3717                 total_iter += t_params[core_idx].iter_average;
3718         }
3719         total_bler /= used_cores;
3720         total_iter /= used_cores;
3721
3722         printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3723                         snr, total_bler * 100, total_iter, get_iter_max(),
3724                         total_mbps, get_vector_filename());
3725 }
3726
3727 /*
3728  * Test function that determines BLER wireless performance
3729  */
3730 static int
3731 bler_test(struct active_device *ad,
3732                 struct test_op_params *op_params)
3733 {
3734         int ret;
3735         unsigned int lcore_id, used_cores = 0;
3736         struct thread_params *t_params;
3737         struct rte_bbdev_info info;
3738         lcore_function_t *bler_function;
3739         uint16_t num_lcores;
3740         const char *op_type_str;
3741
3742         rte_bbdev_info_get(ad->dev_id, &info);
3743
3744         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3745         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3746                         test_vector.op_type);
3747
3748         printf("+ ------------------------------------------------------- +\n");
3749         printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3750                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3751                         op_params->num_to_process, op_params->num_lcores,
3752                         op_type_str,
3753                         intr_enabled ? "Interrupt mode" : "PMD mode",
3754                         (double)rte_get_tsc_hz() / 1000000000.0);
3755
3756         /* Set number of lcores */
3757         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3758                         ? ad->nb_queues
3759                         : op_params->num_lcores;
3760
3761         /* Allocate memory for thread parameters structure */
3762         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3763                         RTE_CACHE_LINE_SIZE);
3764         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3765                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3766                                 RTE_CACHE_LINE_SIZE));
3767
3768         if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) &&
3769                         !check_bit(test_vector.ldpc_dec.op_flags,
3770                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
3771                         && !check_bit(test_vector.ldpc_dec.op_flags,
3772                         RTE_BBDEV_LDPC_LLR_COMPRESSION))
3773                 bler_function = bler_pmd_lcore_ldpc_dec;
3774         else
3775                 return TEST_SKIPPED;
3776
3777         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3778
3779         /* Main core is set at first entry */
3780         t_params[0].dev_id = ad->dev_id;
3781         t_params[0].lcore_id = rte_lcore_id();
3782         t_params[0].op_params = op_params;
3783         t_params[0].queue_id = ad->queue_ids[used_cores++];
3784         t_params[0].iter_count = 0;
3785
3786         RTE_LCORE_FOREACH_WORKER(lcore_id) {
3787                 if (used_cores >= num_lcores)
3788                         break;
3789
3790                 t_params[used_cores].dev_id = ad->dev_id;
3791                 t_params[used_cores].lcore_id = lcore_id;
3792                 t_params[used_cores].op_params = op_params;
3793                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3794                 t_params[used_cores].iter_count = 0;
3795
3796                 rte_eal_remote_launch(bler_function,
3797                                 &t_params[used_cores++], lcore_id);
3798         }
3799
3800         rte_atomic16_set(&op_params->sync, SYNC_START);
3801         ret = bler_function(&t_params[0]);
3802
3803         /* Main core is always used */
3804         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3805                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3806
3807         print_dec_bler(t_params, num_lcores);
3808
3809         /* Return if test failed */
3810         if (ret) {
3811                 rte_free(t_params);
3812                 return ret;
3813         }
3814
3815         /* Function to print something  here*/
3816         rte_free(t_params);
3817         return ret;
3818 }
3819
3820 /*
3821  * Test function that determines how long an enqueue + dequeue of a burst
3822  * takes on available lcores.
3823  */
3824 static int
3825 throughput_test(struct active_device *ad,
3826                 struct test_op_params *op_params)
3827 {
3828         int ret;
3829         unsigned int lcore_id, used_cores = 0;
3830         struct thread_params *t_params, *tp;
3831         struct rte_bbdev_info info;
3832         lcore_function_t *throughput_function;
3833         uint16_t num_lcores;
3834         const char *op_type_str;
3835
3836         rte_bbdev_info_get(ad->dev_id, &info);
3837
3838         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3839         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3840                         test_vector.op_type);
3841
3842         printf("+ ------------------------------------------------------- +\n");
3843         printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3844                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3845                         op_params->num_to_process, op_params->num_lcores,
3846                         op_type_str,
3847                         intr_enabled ? "Interrupt mode" : "PMD mode",
3848                         (double)rte_get_tsc_hz() / 1000000000.0);
3849
3850         /* Set number of lcores */
3851         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3852                         ? ad->nb_queues
3853                         : op_params->num_lcores;
3854
3855         /* Allocate memory for thread parameters structure */
3856         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3857                         RTE_CACHE_LINE_SIZE);
3858         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3859                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3860                                 RTE_CACHE_LINE_SIZE));
3861
3862         if (intr_enabled) {
3863                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3864                         throughput_function = throughput_intr_lcore_dec;
3865                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3866                         throughput_function = throughput_intr_lcore_ldpc_dec;
3867                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3868                         throughput_function = throughput_intr_lcore_enc;
3869                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3870                         throughput_function = throughput_intr_lcore_ldpc_enc;
3871                 else
3872                         throughput_function = throughput_intr_lcore_enc;
3873
3874                 /* Dequeue interrupt callback registration */
3875                 ret = rte_bbdev_callback_register(ad->dev_id,
3876                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3877                                 t_params);
3878                 if (ret < 0) {
3879                         rte_free(t_params);
3880                         return ret;
3881                 }
3882         } else {
3883                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3884                         throughput_function = throughput_pmd_lcore_dec;
3885                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3886                         throughput_function = throughput_pmd_lcore_ldpc_dec;
3887                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3888                         throughput_function = throughput_pmd_lcore_enc;
3889                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3890                         throughput_function = throughput_pmd_lcore_ldpc_enc;
3891                 else
3892                         throughput_function = throughput_pmd_lcore_enc;
3893         }
3894
3895         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3896
3897         /* Main core is set at first entry */
3898         t_params[0].dev_id = ad->dev_id;
3899         t_params[0].lcore_id = rte_lcore_id();
3900         t_params[0].op_params = op_params;
3901         t_params[0].queue_id = ad->queue_ids[used_cores++];
3902         t_params[0].iter_count = 0;
3903
3904         RTE_LCORE_FOREACH_WORKER(lcore_id) {
3905                 if (used_cores >= num_lcores)
3906                         break;
3907
3908                 t_params[used_cores].dev_id = ad->dev_id;
3909                 t_params[used_cores].lcore_id = lcore_id;
3910                 t_params[used_cores].op_params = op_params;
3911                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3912                 t_params[used_cores].iter_count = 0;
3913
3914                 rte_eal_remote_launch(throughput_function,
3915                                 &t_params[used_cores++], lcore_id);
3916         }
3917
3918         rte_atomic16_set(&op_params->sync, SYNC_START);
3919         ret = throughput_function(&t_params[0]);
3920
3921         /* Main core is always used */
3922         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3923                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3924
3925         /* Return if test failed */
3926         if (ret) {
3927                 rte_free(t_params);
3928                 return ret;
3929         }
3930
3931         /* Print throughput if interrupts are disabled and test passed */
3932         if (!intr_enabled) {
3933                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3934                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3935                         print_dec_throughput(t_params, num_lcores);
3936                 else
3937                         print_enc_throughput(t_params, num_lcores);
3938                 rte_free(t_params);
3939                 return ret;
3940         }
3941
3942         /* In interrupt TC we need to wait for the interrupt callback to deqeue
3943          * all pending operations. Skip waiting for queues which reported an
3944          * error using processing_status variable.
3945          * Wait for main lcore operations.
3946          */
3947         tp = &t_params[0];
3948         while ((rte_atomic16_read(&tp->nb_dequeued) <
3949                         op_params->num_to_process) &&
3950                         (rte_atomic16_read(&tp->processing_status) !=
3951                         TEST_FAILED))
3952                 rte_pause();
3953
3954         tp->ops_per_sec /= TEST_REPETITIONS;
3955         tp->mbps /= TEST_REPETITIONS;
3956         ret |= (int)rte_atomic16_read(&tp->processing_status);
3957
3958         /* Wait for worker lcores operations */
3959         for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3960                 tp = &t_params[used_cores];
3961
3962                 while ((rte_atomic16_read(&tp->nb_dequeued) <
3963                                 op_params->num_to_process) &&
3964                                 (rte_atomic16_read(&tp->processing_status) !=
3965                                 TEST_FAILED))
3966                         rte_pause();
3967
3968                 tp->ops_per_sec /= TEST_REPETITIONS;
3969                 tp->mbps /= TEST_REPETITIONS;
3970                 ret |= (int)rte_atomic16_read(&tp->processing_status);
3971         }
3972
3973         /* Print throughput if test passed */
3974         if (!ret) {
3975                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3976                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3977                         print_dec_throughput(t_params, num_lcores);
3978                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3979                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3980                         print_enc_throughput(t_params, num_lcores);
3981         }
3982
3983         rte_free(t_params);
3984         return ret;
3985 }
3986
3987 static int
3988 latency_test_dec(struct rte_mempool *mempool,
3989                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3990                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3991                 const uint16_t num_to_process, uint16_t burst_sz,
3992                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3993 {
3994         int ret = TEST_SUCCESS;
3995         uint16_t i, j, dequeued;
3996         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3997         uint64_t start_time = 0, last_time = 0;
3998
3999         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4000                 uint16_t enq = 0, deq = 0;
4001                 bool first_time = true;
4002                 last_time = 0;
4003
4004                 if (unlikely(num_to_process - dequeued < burst_sz))
4005                         burst_sz = num_to_process - dequeued;
4006
4007                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4008                 TEST_ASSERT_SUCCESS(ret,
4009                                 "rte_bbdev_dec_op_alloc_bulk() failed");
4010                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4011                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4012                                         bufs->inputs,
4013                                         bufs->hard_outputs,
4014                                         bufs->soft_outputs,
4015                                         ref_op);
4016
4017                 /* Set counter to validate the ordering */
4018                 for (j = 0; j < burst_sz; ++j)
4019                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4020
4021                 start_time = rte_rdtsc_precise();
4022
4023                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
4024                                 burst_sz);
4025                 TEST_ASSERT(enq == burst_sz,
4026                                 "Error enqueueing burst, expected %u, got %u",
4027                                 burst_sz, enq);
4028
4029                 /* Dequeue */
4030                 do {
4031                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4032                                         &ops_deq[deq], burst_sz - deq);
4033                         if (likely(first_time && (deq > 0))) {
4034                                 last_time = rte_rdtsc_precise() - start_time;
4035                                 first_time = false;
4036                         }
4037                 } while (unlikely(burst_sz != deq));
4038
4039                 *max_time = RTE_MAX(*max_time, last_time);
4040                 *min_time = RTE_MIN(*min_time, last_time);
4041                 *total_time += last_time;
4042
4043                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4044                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
4045                                         vector_mask);
4046                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4047                 }
4048
4049                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4050                 dequeued += deq;
4051         }
4052
4053         return i;
4054 }
4055
4056 /* Test case for latency/validation for LDPC Decoder */
4057 static int
4058 latency_test_ldpc_dec(struct rte_mempool *mempool,
4059                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
4060                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
4061                 const uint16_t num_to_process, uint16_t burst_sz,
4062                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time,
4063                 bool disable_et)
4064 {
4065         int ret = TEST_SUCCESS;
4066         uint16_t i, j, dequeued;
4067         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4068         uint64_t start_time = 0, last_time = 0;
4069         bool extDdr = ldpc_cap_flags &
4070                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4071
4072         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4073                 uint16_t enq = 0, deq = 0;
4074                 bool first_time = true;
4075                 last_time = 0;
4076
4077                 if (unlikely(num_to_process - dequeued < burst_sz))
4078                         burst_sz = num_to_process - dequeued;
4079
4080                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4081                 TEST_ASSERT_SUCCESS(ret,
4082                                 "rte_bbdev_dec_op_alloc_bulk() failed");
4083
4084                 /* For latency tests we need to disable early termination */
4085                 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags,
4086                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
4087                         ref_op->ldpc_dec.op_flags -=
4088                                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
4089                 ref_op->ldpc_dec.iter_max = get_iter_max();
4090                 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
4091
4092                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4093                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4094                                         bufs->inputs,
4095                                         bufs->hard_outputs,
4096                                         bufs->soft_outputs,
4097                                         bufs->harq_inputs,
4098                                         bufs->harq_outputs,
4099                                         ref_op);
4100
4101                 if (extDdr)
4102                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4103                                         burst_sz, true);
4104
4105                 /* Set counter to validate the ordering */
4106                 for (j = 0; j < burst_sz; ++j)
4107                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4108
4109                 start_time = rte_rdtsc_precise();
4110
4111                 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4112                                 &ops_enq[enq], burst_sz);
4113                 TEST_ASSERT(enq == burst_sz,
4114                                 "Error enqueueing burst, expected %u, got %u",
4115                                 burst_sz, enq);
4116
4117                 /* Dequeue */
4118                 do {
4119                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4120                                         &ops_deq[deq], burst_sz - deq);
4121                         if (likely(first_time && (deq > 0))) {
4122                                 last_time = rte_rdtsc_precise() - start_time;
4123                                 first_time = false;
4124                         }
4125                 } while (unlikely(burst_sz != deq));
4126
4127                 *max_time = RTE_MAX(*max_time, last_time);
4128                 *min_time = RTE_MIN(*min_time, last_time);
4129                 *total_time += last_time;
4130
4131                 if (extDdr)
4132                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4133
4134                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4135                         ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4136                                         vector_mask);
4137                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4138                 }
4139
4140                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4141                 dequeued += deq;
4142         }
4143         return i;
4144 }
4145
4146 static int
4147 latency_test_enc(struct rte_mempool *mempool,
4148                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4149                 uint16_t dev_id, uint16_t queue_id,
4150                 const uint16_t num_to_process, uint16_t burst_sz,
4151                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4152 {
4153         int ret = TEST_SUCCESS;
4154         uint16_t i, j, dequeued;
4155         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4156         uint64_t start_time = 0, last_time = 0;
4157
4158         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4159                 uint16_t enq = 0, deq = 0;
4160                 bool first_time = true;
4161                 last_time = 0;
4162
4163                 if (unlikely(num_to_process - dequeued < burst_sz))
4164                         burst_sz = num_to_process - dequeued;
4165
4166                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4167                 TEST_ASSERT_SUCCESS(ret,
4168                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4169                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4170                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4171                                         bufs->inputs,
4172                                         bufs->hard_outputs,
4173                                         ref_op);
4174
4175                 /* Set counter to validate the ordering */
4176                 for (j = 0; j < burst_sz; ++j)
4177                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4178
4179                 start_time = rte_rdtsc_precise();
4180
4181                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4182                                 burst_sz);
4183                 TEST_ASSERT(enq == burst_sz,
4184                                 "Error enqueueing burst, expected %u, got %u",
4185                                 burst_sz, enq);
4186
4187                 /* Dequeue */
4188                 do {
4189                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4190                                         &ops_deq[deq], burst_sz - deq);
4191                         if (likely(first_time && (deq > 0))) {
4192                                 last_time += rte_rdtsc_precise() - start_time;
4193                                 first_time = false;
4194                         }
4195                 } while (unlikely(burst_sz != deq));
4196
4197                 *max_time = RTE_MAX(*max_time, last_time);
4198                 *min_time = RTE_MIN(*min_time, last_time);
4199                 *total_time += last_time;
4200
4201                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4202                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4203                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4204                 }
4205
4206                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4207                 dequeued += deq;
4208         }
4209
4210         return i;
4211 }
4212
4213 static int
4214 latency_test_ldpc_enc(struct rte_mempool *mempool,
4215                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4216                 uint16_t dev_id, uint16_t queue_id,
4217                 const uint16_t num_to_process, uint16_t burst_sz,
4218                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4219 {
4220         int ret = TEST_SUCCESS;
4221         uint16_t i, j, dequeued;
4222         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4223         uint64_t start_time = 0, last_time = 0;
4224
4225         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4226                 uint16_t enq = 0, deq = 0;
4227                 bool first_time = true;
4228                 last_time = 0;
4229
4230                 if (unlikely(num_to_process - dequeued < burst_sz))
4231                         burst_sz = num_to_process - dequeued;
4232
4233                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4234                 TEST_ASSERT_SUCCESS(ret,
4235                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4236                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4237                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4238                                         bufs->inputs,
4239                                         bufs->hard_outputs,
4240                                         ref_op);
4241
4242                 /* Set counter to validate the ordering */
4243                 for (j = 0; j < burst_sz; ++j)
4244                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4245
4246                 start_time = rte_rdtsc_precise();
4247
4248                 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4249                                 &ops_enq[enq], burst_sz);
4250                 TEST_ASSERT(enq == burst_sz,
4251                                 "Error enqueueing burst, expected %u, got %u",
4252                                 burst_sz, enq);
4253
4254                 /* Dequeue */
4255                 do {
4256                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4257                                         &ops_deq[deq], burst_sz - deq);
4258                         if (likely(first_time && (deq > 0))) {
4259                                 last_time += rte_rdtsc_precise() - start_time;
4260                                 first_time = false;
4261                         }
4262                 } while (unlikely(burst_sz != deq));
4263
4264                 *max_time = RTE_MAX(*max_time, last_time);
4265                 *min_time = RTE_MIN(*min_time, last_time);
4266                 *total_time += last_time;
4267
4268                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4269                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4270                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4271                 }
4272
4273                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4274                 dequeued += deq;
4275         }
4276
4277         return i;
4278 }
4279
4280 /* Common function for running validation and latency test cases */
4281 static int
4282 validation_latency_test(struct active_device *ad,
4283                 struct test_op_params *op_params, bool latency_flag)
4284 {
4285         int iter;
4286         uint16_t burst_sz = op_params->burst_sz;
4287         const uint16_t num_to_process = op_params->num_to_process;
4288         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4289         const uint16_t queue_id = ad->queue_ids[0];
4290         struct test_buffers *bufs = NULL;
4291         struct rte_bbdev_info info;
4292         uint64_t total_time, min_time, max_time;
4293         const char *op_type_str;
4294
4295         total_time = max_time = 0;
4296         min_time = UINT64_MAX;
4297
4298         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4299                         "BURST_SIZE should be <= %u", MAX_BURST);
4300
4301         rte_bbdev_info_get(ad->dev_id, &info);
4302         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4303
4304         op_type_str = rte_bbdev_op_type_str(op_type);
4305         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4306
4307         printf("+ ------------------------------------------------------- +\n");
4308         if (latency_flag)
4309                 printf("== test: latency\ndev:");
4310         else
4311                 printf("== test: validation\ndev:");
4312         printf("%s, burst size: %u, num ops: %u, op type: %s\n",
4313                         info.dev_name, burst_sz, num_to_process, op_type_str);
4314
4315         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4316                 iter = latency_test_dec(op_params->mp, bufs,
4317                                 op_params->ref_dec_op, op_params->vector_mask,
4318                                 ad->dev_id, queue_id, num_to_process,
4319                                 burst_sz, &total_time, &min_time, &max_time);
4320         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4321                 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4322                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4323                                 num_to_process, burst_sz, &total_time,
4324                                 &min_time, &max_time);
4325         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4326                 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4327                                 op_params->ref_dec_op, op_params->vector_mask,
4328                                 ad->dev_id, queue_id, num_to_process,
4329                                 burst_sz, &total_time, &min_time, &max_time,
4330                                 latency_flag);
4331         else /* RTE_BBDEV_OP_TURBO_ENC */
4332                 iter = latency_test_enc(op_params->mp, bufs,
4333                                 op_params->ref_enc_op,
4334                                 ad->dev_id, queue_id,
4335                                 num_to_process, burst_sz, &total_time,
4336                                 &min_time, &max_time);
4337
4338         if (iter <= 0)
4339                 return TEST_FAILED;
4340
4341         printf("Operation latency:\n"
4342                         "\tavg: %lg cycles, %lg us\n"
4343                         "\tmin: %lg cycles, %lg us\n"
4344                         "\tmax: %lg cycles, %lg us\n",
4345                         (double)total_time / (double)iter,
4346                         (double)(total_time * 1000000) / (double)iter /
4347                         (double)rte_get_tsc_hz(), (double)min_time,
4348                         (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4349                         (double)max_time, (double)(max_time * 1000000) /
4350                         (double)rte_get_tsc_hz());
4351
4352         return TEST_SUCCESS;
4353 }
4354
4355 static int
4356 latency_test(struct active_device *ad, struct test_op_params *op_params)
4357 {
4358         return validation_latency_test(ad, op_params, true);
4359 }
4360
4361 static int
4362 validation_test(struct active_device *ad, struct test_op_params *op_params)
4363 {
4364         return validation_latency_test(ad, op_params, false);
4365 }
4366
4367 #ifdef RTE_BBDEV_OFFLOAD_COST
4368 static int
4369 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4370                 struct rte_bbdev_stats *stats)
4371 {
4372         struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4373         struct rte_bbdev_stats *q_stats;
4374
4375         if (queue_id >= dev->data->num_queues)
4376                 return -1;
4377
4378         q_stats = &dev->data->queues[queue_id].queue_stats;
4379
4380         stats->enqueued_count = q_stats->enqueued_count;
4381         stats->dequeued_count = q_stats->dequeued_count;
4382         stats->enqueue_err_count = q_stats->enqueue_err_count;
4383         stats->dequeue_err_count = q_stats->dequeue_err_count;
4384         stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4385
4386         return 0;
4387 }
4388
4389 static int
4390 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4391                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4392                 uint16_t queue_id, const uint16_t num_to_process,
4393                 uint16_t burst_sz, struct test_time_stats *time_st)
4394 {
4395         int i, dequeued, ret;
4396         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4397         uint64_t enq_start_time, deq_start_time;
4398         uint64_t enq_sw_last_time, deq_last_time;
4399         struct rte_bbdev_stats stats;
4400
4401         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4402                 uint16_t enq = 0, deq = 0;
4403
4404                 if (unlikely(num_to_process - dequeued < burst_sz))
4405                         burst_sz = num_to_process - dequeued;
4406
4407                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4408                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4409                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4410                                         bufs->inputs,
4411                                         bufs->hard_outputs,
4412                                         bufs->soft_outputs,
4413                                         ref_op);
4414
4415                 /* Start time meas for enqueue function offload latency */
4416                 enq_start_time = rte_rdtsc_precise();
4417                 do {
4418                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4419                                         &ops_enq[enq], burst_sz - enq);
4420                 } while (unlikely(burst_sz != enq));
4421
4422                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4423                 TEST_ASSERT_SUCCESS(ret,
4424                                 "Failed to get stats for queue (%u) of device (%u)",
4425                                 queue_id, dev_id);
4426
4427                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4428                                 stats.acc_offload_cycles;
4429                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4430                                 enq_sw_last_time);
4431                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4432                                 enq_sw_last_time);
4433                 time_st->enq_sw_total_time += enq_sw_last_time;
4434
4435                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4436                                 stats.acc_offload_cycles);
4437                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4438                                 stats.acc_offload_cycles);
4439                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4440
4441                 /* give time for device to process ops */
4442                 rte_delay_us(WAIT_OFFLOAD_US);
4443
4444                 /* Start time meas for dequeue function offload latency */
4445                 deq_start_time = rte_rdtsc_precise();
4446                 /* Dequeue one operation */
4447                 do {
4448                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4449                                         &ops_deq[deq], enq);
4450                 } while (unlikely(deq == 0));
4451
4452                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4453                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4454                                 deq_last_time);
4455                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4456                                 deq_last_time);
4457                 time_st->deq_total_time += deq_last_time;
4458
4459                 /* Dequeue remaining operations if needed*/
4460                 while (burst_sz != deq)
4461                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4462                                         &ops_deq[deq], burst_sz - deq);
4463
4464                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4465                 dequeued += deq;
4466         }
4467
4468         return i;
4469 }
4470
4471 static int
4472 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4473                 struct test_buffers *bufs,
4474                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4475                 uint16_t queue_id, const uint16_t num_to_process,
4476                 uint16_t burst_sz, struct test_time_stats *time_st)
4477 {
4478         int i, dequeued, ret;
4479         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4480         uint64_t enq_start_time, deq_start_time;
4481         uint64_t enq_sw_last_time, deq_last_time;
4482         struct rte_bbdev_stats stats;
4483         bool extDdr = ldpc_cap_flags &
4484                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4485
4486         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4487                 uint16_t enq = 0, deq = 0;
4488
4489                 if (unlikely(num_to_process - dequeued < burst_sz))
4490                         burst_sz = num_to_process - dequeued;
4491
4492                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4493                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4494                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4495                                         bufs->inputs,
4496                                         bufs->hard_outputs,
4497                                         bufs->soft_outputs,
4498                                         bufs->harq_inputs,
4499                                         bufs->harq_outputs,
4500                                         ref_op);
4501
4502                 if (extDdr)
4503                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4504                                         burst_sz, true);
4505
4506                 /* Start time meas for enqueue function offload latency */
4507                 enq_start_time = rte_rdtsc_precise();
4508                 do {
4509                         enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4510                                         &ops_enq[enq], burst_sz - enq);
4511                 } while (unlikely(burst_sz != enq));
4512
4513                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4514                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4515                 TEST_ASSERT_SUCCESS(ret,
4516                                 "Failed to get stats for queue (%u) of device (%u)",
4517                                 queue_id, dev_id);
4518
4519                 enq_sw_last_time -= stats.acc_offload_cycles;
4520                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4521                                 enq_sw_last_time);
4522                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4523                                 enq_sw_last_time);
4524                 time_st->enq_sw_total_time += enq_sw_last_time;
4525
4526                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4527                                 stats.acc_offload_cycles);
4528                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4529                                 stats.acc_offload_cycles);
4530                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4531
4532                 /* give time for device to process ops */
4533                 rte_delay_us(WAIT_OFFLOAD_US);
4534
4535                 /* Start time meas for dequeue function offload latency */
4536                 deq_start_time = rte_rdtsc_precise();
4537                 /* Dequeue one operation */
4538                 do {
4539                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4540                                         &ops_deq[deq], enq);
4541                 } while (unlikely(deq == 0));
4542
4543                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4544                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4545                                 deq_last_time);
4546                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4547                                 deq_last_time);
4548                 time_st->deq_total_time += deq_last_time;
4549
4550                 /* Dequeue remaining operations if needed*/
4551                 while (burst_sz != deq)
4552                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4553                                         &ops_deq[deq], burst_sz - deq);
4554
4555                 if (extDdr) {
4556                         /* Read loopback is not thread safe */
4557                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4558                 }
4559
4560                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4561                 dequeued += deq;
4562         }
4563
4564         return i;
4565 }
4566
4567 static int
4568 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4569                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4570                 uint16_t queue_id, const uint16_t num_to_process,
4571                 uint16_t burst_sz, struct test_time_stats *time_st)
4572 {
4573         int i, dequeued, ret;
4574         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4575         uint64_t enq_start_time, deq_start_time;
4576         uint64_t enq_sw_last_time, deq_last_time;
4577         struct rte_bbdev_stats stats;
4578
4579         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4580                 uint16_t enq = 0, deq = 0;
4581
4582                 if (unlikely(num_to_process - dequeued < burst_sz))
4583                         burst_sz = num_to_process - dequeued;
4584
4585                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4586                 TEST_ASSERT_SUCCESS(ret,
4587                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4588                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4589                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4590                                         bufs->inputs,
4591                                         bufs->hard_outputs,
4592                                         ref_op);
4593
4594                 /* Start time meas for enqueue function offload latency */
4595                 enq_start_time = rte_rdtsc_precise();
4596                 do {
4597                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4598                                         &ops_enq[enq], burst_sz - enq);
4599                 } while (unlikely(burst_sz != enq));
4600
4601                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4602
4603                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4604                 TEST_ASSERT_SUCCESS(ret,
4605                                 "Failed to get stats for queue (%u) of device (%u)",
4606                                 queue_id, dev_id);
4607                 enq_sw_last_time -= stats.acc_offload_cycles;
4608                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4609                                 enq_sw_last_time);
4610                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4611                                 enq_sw_last_time);
4612                 time_st->enq_sw_total_time += enq_sw_last_time;
4613
4614                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4615                                 stats.acc_offload_cycles);
4616                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4617                                 stats.acc_offload_cycles);
4618                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4619
4620                 /* give time for device to process ops */
4621                 rte_delay_us(WAIT_OFFLOAD_US);
4622
4623                 /* Start time meas for dequeue function offload latency */
4624                 deq_start_time = rte_rdtsc_precise();
4625                 /* Dequeue one operation */
4626                 do {
4627                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4628                                         &ops_deq[deq], enq);
4629                 } while (unlikely(deq == 0));
4630
4631                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4632                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4633                                 deq_last_time);
4634                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4635                                 deq_last_time);
4636                 time_st->deq_total_time += deq_last_time;
4637
4638                 while (burst_sz != deq)
4639                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4640                                         &ops_deq[deq], burst_sz - deq);
4641
4642                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4643                 dequeued += deq;
4644         }
4645
4646         return i;
4647 }
4648
4649 static int
4650 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4651                 struct test_buffers *bufs,
4652                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4653                 uint16_t queue_id, const uint16_t num_to_process,
4654                 uint16_t burst_sz, struct test_time_stats *time_st)
4655 {
4656         int i, dequeued, ret;
4657         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4658         uint64_t enq_start_time, deq_start_time;
4659         uint64_t enq_sw_last_time, deq_last_time;
4660         struct rte_bbdev_stats stats;
4661
4662         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4663                 uint16_t enq = 0, deq = 0;
4664
4665                 if (unlikely(num_to_process - dequeued < burst_sz))
4666                         burst_sz = num_to_process - dequeued;
4667
4668                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4669                 TEST_ASSERT_SUCCESS(ret,
4670                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4671                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4672                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4673                                         bufs->inputs,
4674                                         bufs->hard_outputs,
4675                                         ref_op);
4676
4677                 /* Start time meas for enqueue function offload latency */
4678                 enq_start_time = rte_rdtsc_precise();
4679                 do {
4680                         enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4681                                         &ops_enq[enq], burst_sz - enq);
4682                 } while (unlikely(burst_sz != enq));
4683
4684                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4685                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4686                 TEST_ASSERT_SUCCESS(ret,
4687                                 "Failed to get stats for queue (%u) of device (%u)",
4688                                 queue_id, dev_id);
4689
4690                 enq_sw_last_time -= stats.acc_offload_cycles;
4691                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4692                                 enq_sw_last_time);
4693                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4694                                 enq_sw_last_time);
4695                 time_st->enq_sw_total_time += enq_sw_last_time;
4696
4697                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4698                                 stats.acc_offload_cycles);
4699                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4700                                 stats.acc_offload_cycles);
4701                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4702
4703                 /* give time for device to process ops */
4704                 rte_delay_us(WAIT_OFFLOAD_US);
4705
4706                 /* Start time meas for dequeue function offload latency */
4707                 deq_start_time = rte_rdtsc_precise();
4708                 /* Dequeue one operation */
4709                 do {
4710                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4711                                         &ops_deq[deq], enq);
4712                 } while (unlikely(deq == 0));
4713
4714                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4715                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4716                                 deq_last_time);
4717                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4718                                 deq_last_time);
4719                 time_st->deq_total_time += deq_last_time;
4720
4721                 while (burst_sz != deq)
4722                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4723                                         &ops_deq[deq], burst_sz - deq);
4724
4725                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4726                 dequeued += deq;
4727         }
4728
4729         return i;
4730 }
4731 #endif
4732
4733 static int
4734 offload_cost_test(struct active_device *ad,
4735                 struct test_op_params *op_params)
4736 {
4737 #ifndef RTE_BBDEV_OFFLOAD_COST
4738         RTE_SET_USED(ad);
4739         RTE_SET_USED(op_params);
4740         printf("Offload latency test is disabled.\n");
4741         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4742         return TEST_SKIPPED;
4743 #else
4744         int iter;
4745         uint16_t burst_sz = op_params->burst_sz;
4746         const uint16_t num_to_process = op_params->num_to_process;
4747         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4748         const uint16_t queue_id = ad->queue_ids[0];
4749         struct test_buffers *bufs = NULL;
4750         struct rte_bbdev_info info;
4751         const char *op_type_str;
4752         struct test_time_stats time_st;
4753
4754         memset(&time_st, 0, sizeof(struct test_time_stats));
4755         time_st.enq_sw_min_time = UINT64_MAX;
4756         time_st.enq_acc_min_time = UINT64_MAX;
4757         time_st.deq_min_time = UINT64_MAX;
4758
4759         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4760                         "BURST_SIZE should be <= %u", MAX_BURST);
4761
4762         rte_bbdev_info_get(ad->dev_id, &info);
4763         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4764
4765         op_type_str = rte_bbdev_op_type_str(op_type);
4766         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4767
4768         printf("+ ------------------------------------------------------- +\n");
4769         printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4770                         info.dev_name, burst_sz, num_to_process, op_type_str);
4771
4772         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4773                 iter = offload_latency_test_dec(op_params->mp, bufs,
4774                                 op_params->ref_dec_op, ad->dev_id, queue_id,
4775                                 num_to_process, burst_sz, &time_st);
4776         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4777                 iter = offload_latency_test_enc(op_params->mp, bufs,
4778                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4779                                 num_to_process, burst_sz, &time_st);
4780         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4781                 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4782                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4783                                 num_to_process, burst_sz, &time_st);
4784         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4785                 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4786                         op_params->ref_dec_op, ad->dev_id, queue_id,
4787                         num_to_process, burst_sz, &time_st);
4788         else
4789                 iter = offload_latency_test_enc(op_params->mp, bufs,
4790                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4791                                 num_to_process, burst_sz, &time_st);
4792
4793         if (iter <= 0)
4794                 return TEST_FAILED;
4795
4796         printf("Enqueue driver offload cost latency:\n"
4797                         "\tavg: %lg cycles, %lg us\n"
4798                         "\tmin: %lg cycles, %lg us\n"
4799                         "\tmax: %lg cycles, %lg us\n"
4800                         "Enqueue accelerator offload cost latency:\n"
4801                         "\tavg: %lg cycles, %lg us\n"
4802                         "\tmin: %lg cycles, %lg us\n"
4803                         "\tmax: %lg cycles, %lg us\n",
4804                         (double)time_st.enq_sw_total_time / (double)iter,
4805                         (double)(time_st.enq_sw_total_time * 1000000) /
4806                         (double)iter / (double)rte_get_tsc_hz(),
4807                         (double)time_st.enq_sw_min_time,
4808                         (double)(time_st.enq_sw_min_time * 1000000) /
4809                         rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4810                         (double)(time_st.enq_sw_max_time * 1000000) /
4811                         rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4812                         (double)iter,
4813                         (double)(time_st.enq_acc_total_time * 1000000) /
4814                         (double)iter / (double)rte_get_tsc_hz(),
4815                         (double)time_st.enq_acc_min_time,
4816                         (double)(time_st.enq_acc_min_time * 1000000) /
4817                         rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4818                         (double)(time_st.enq_acc_max_time * 1000000) /
4819                         rte_get_tsc_hz());
4820
4821         printf("Dequeue offload cost latency - one op:\n"
4822                         "\tavg: %lg cycles, %lg us\n"
4823                         "\tmin: %lg cycles, %lg us\n"
4824                         "\tmax: %lg cycles, %lg us\n",
4825                         (double)time_st.deq_total_time / (double)iter,
4826                         (double)(time_st.deq_total_time * 1000000) /
4827                         (double)iter / (double)rte_get_tsc_hz(),
4828                         (double)time_st.deq_min_time,
4829                         (double)(time_st.deq_min_time * 1000000) /
4830                         rte_get_tsc_hz(), (double)time_st.deq_max_time,
4831                         (double)(time_st.deq_max_time * 1000000) /
4832                         rte_get_tsc_hz());
4833
4834         struct rte_bbdev_stats stats = {0};
4835         get_bbdev_queue_stats(ad->dev_id, queue_id, &stats);
4836         if (op_type != RTE_BBDEV_OP_LDPC_DEC) {
4837                 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process,
4838                                 "Mismatch in enqueue count %10"PRIu64" %d",
4839                                 stats.enqueued_count, num_to_process);
4840                 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process,
4841                                 "Mismatch in dequeue count %10"PRIu64" %d",
4842                                 stats.dequeued_count, num_to_process);
4843         }
4844         TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0,
4845                         "Enqueue count Error %10"PRIu64"",
4846                         stats.enqueue_err_count);
4847         TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0,
4848                         "Dequeue count Error (%10"PRIu64"",
4849                         stats.dequeue_err_count);
4850
4851         return TEST_SUCCESS;
4852 #endif
4853 }
4854
4855 #ifdef RTE_BBDEV_OFFLOAD_COST
4856 static int
4857 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4858                 const uint16_t num_to_process, uint16_t burst_sz,
4859                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4860                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4861 {
4862         int i, deq_total;
4863         struct rte_bbdev_dec_op *ops[MAX_BURST];
4864         uint64_t deq_start_time, deq_last_time;
4865
4866         /* Test deq offload latency from an empty queue */
4867
4868         for (i = 0, deq_total = 0; deq_total < num_to_process;
4869                         ++i, deq_total += burst_sz) {
4870                 deq_start_time = rte_rdtsc_precise();
4871
4872                 if (unlikely(num_to_process - deq_total < burst_sz))
4873                         burst_sz = num_to_process - deq_total;
4874                 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4875                         rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4876                                         burst_sz);
4877                 else
4878                         rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4879                                         burst_sz);
4880
4881                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4882                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4883                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4884                 *deq_total_time += deq_last_time;
4885         }
4886
4887         return i;
4888 }
4889
4890 static int
4891 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4892                 const uint16_t num_to_process, uint16_t burst_sz,
4893                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4894                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4895 {
4896         int i, deq_total;
4897         struct rte_bbdev_enc_op *ops[MAX_BURST];
4898         uint64_t deq_start_time, deq_last_time;
4899
4900         /* Test deq offload latency from an empty queue */
4901         for (i = 0, deq_total = 0; deq_total < num_to_process;
4902                         ++i, deq_total += burst_sz) {
4903                 deq_start_time = rte_rdtsc_precise();
4904
4905                 if (unlikely(num_to_process - deq_total < burst_sz))
4906                         burst_sz = num_to_process - deq_total;
4907                 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4908                         rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4909                                         burst_sz);
4910                 else
4911                         rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4912                                         burst_sz);
4913
4914                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4915                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4916                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4917                 *deq_total_time += deq_last_time;
4918         }
4919
4920         return i;
4921 }
4922
4923 #endif
4924
4925 static int
4926 offload_latency_empty_q_test(struct active_device *ad,
4927                 struct test_op_params *op_params)
4928 {
4929 #ifndef RTE_BBDEV_OFFLOAD_COST
4930         RTE_SET_USED(ad);
4931         RTE_SET_USED(op_params);
4932         printf("Offload latency empty dequeue test is disabled.\n");
4933         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4934         return TEST_SKIPPED;
4935 #else
4936         int iter;
4937         uint64_t deq_total_time, deq_min_time, deq_max_time;
4938         uint16_t burst_sz = op_params->burst_sz;
4939         const uint16_t num_to_process = op_params->num_to_process;
4940         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4941         const uint16_t queue_id = ad->queue_ids[0];
4942         struct rte_bbdev_info info;
4943         const char *op_type_str;
4944
4945         deq_total_time = deq_max_time = 0;
4946         deq_min_time = UINT64_MAX;
4947
4948         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4949                         "BURST_SIZE should be <= %u", MAX_BURST);
4950
4951         rte_bbdev_info_get(ad->dev_id, &info);
4952
4953         op_type_str = rte_bbdev_op_type_str(op_type);
4954         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4955
4956         printf("+ ------------------------------------------------------- +\n");
4957         printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4958                         info.dev_name, burst_sz, num_to_process, op_type_str);
4959
4960         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4961                         op_type == RTE_BBDEV_OP_LDPC_DEC)
4962                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4963                                 num_to_process, burst_sz, &deq_total_time,
4964                                 &deq_min_time, &deq_max_time, op_type);
4965         else
4966                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4967                                 num_to_process, burst_sz, &deq_total_time,
4968                                 &deq_min_time, &deq_max_time, op_type);
4969
4970         if (iter <= 0)
4971                 return TEST_FAILED;
4972
4973         printf("Empty dequeue offload:\n"
4974                         "\tavg: %lg cycles, %lg us\n"
4975                         "\tmin: %lg cycles, %lg us\n"
4976                         "\tmax: %lg cycles, %lg us\n",
4977                         (double)deq_total_time / (double)iter,
4978                         (double)(deq_total_time * 1000000) / (double)iter /
4979                         (double)rte_get_tsc_hz(), (double)deq_min_time,
4980                         (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4981                         (double)deq_max_time, (double)(deq_max_time * 1000000) /
4982                         rte_get_tsc_hz());
4983
4984         return TEST_SUCCESS;
4985 #endif
4986 }
4987
4988 static int
4989 bler_tc(void)
4990 {
4991         return run_test_case(bler_test);
4992 }
4993
4994 static int
4995 throughput_tc(void)
4996 {
4997         return run_test_case(throughput_test);
4998 }
4999
5000 static int
5001 offload_cost_tc(void)
5002 {
5003         return run_test_case(offload_cost_test);
5004 }
5005
5006 static int
5007 offload_latency_empty_q_tc(void)
5008 {
5009         return run_test_case(offload_latency_empty_q_test);
5010 }
5011
5012 static int
5013 latency_tc(void)
5014 {
5015         return run_test_case(latency_test);
5016 }
5017
5018 static int
5019 validation_tc(void)
5020 {
5021         return run_test_case(validation_test);
5022 }
5023
5024 static int
5025 interrupt_tc(void)
5026 {
5027         return run_test_case(throughput_test);
5028 }
5029
5030 static struct unit_test_suite bbdev_bler_testsuite = {
5031         .suite_name = "BBdev BLER Tests",
5032         .setup = testsuite_setup,
5033         .teardown = testsuite_teardown,
5034         .unit_test_cases = {
5035                 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
5036                 TEST_CASES_END() /**< NULL terminate unit test array */
5037         }
5038 };
5039
5040 static struct unit_test_suite bbdev_throughput_testsuite = {
5041         .suite_name = "BBdev Throughput Tests",
5042         .setup = testsuite_setup,
5043         .teardown = testsuite_teardown,
5044         .unit_test_cases = {
5045                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
5046                 TEST_CASES_END() /**< NULL terminate unit test array */
5047         }
5048 };
5049
5050 static struct unit_test_suite bbdev_validation_testsuite = {
5051         .suite_name = "BBdev Validation Tests",
5052         .setup = testsuite_setup,
5053         .teardown = testsuite_teardown,
5054         .unit_test_cases = {
5055                 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc),
5056                 TEST_CASES_END() /**< NULL terminate unit test array */
5057         }
5058 };
5059
5060 static struct unit_test_suite bbdev_latency_testsuite = {
5061         .suite_name = "BBdev Latency Tests",
5062         .setup = testsuite_setup,
5063         .teardown = testsuite_teardown,
5064         .unit_test_cases = {
5065                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
5066                 TEST_CASES_END() /**< NULL terminate unit test array */
5067         }
5068 };
5069
5070 static struct unit_test_suite bbdev_offload_cost_testsuite = {
5071         .suite_name = "BBdev Offload Cost Tests",
5072         .setup = testsuite_setup,
5073         .teardown = testsuite_teardown,
5074         .unit_test_cases = {
5075                 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
5076                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
5077                 TEST_CASES_END() /**< NULL terminate unit test array */
5078         }
5079 };
5080
5081 static struct unit_test_suite bbdev_interrupt_testsuite = {
5082         .suite_name = "BBdev Interrupt Tests",
5083         .setup = interrupt_testsuite_setup,
5084         .teardown = testsuite_teardown,
5085         .unit_test_cases = {
5086                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
5087                 TEST_CASES_END() /**< NULL terminate unit test array */
5088         }
5089 };
5090
5091 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
5092 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
5093 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
5094 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
5095 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
5096 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);