app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19 #include <rte_interrupts.h>
  20
  21 #include "main.h"
  22 #include "test_bbdev_vector.h"
  23
  24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  25
  26 #define MAX_QUEUES RTE_MAX_LCORE
  27 #define TEST_REPETITIONS 100
  28 #define WAIT_OFFLOAD_US 1000
  29
  30 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
  31 #include <fpga_lte_fec.h>
  32 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
  33 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
  34 #define VF_UL_4G_QUEUE_VALUE 4
  35 #define VF_DL_4G_QUEUE_VALUE 4
  36 #define UL_4G_BANDWIDTH 3
  37 #define DL_4G_BANDWIDTH 3
  38 #define UL_4G_LOAD_BALANCE 128
  39 #define DL_4G_LOAD_BALANCE 128
  40 #define FLR_4G_TIMEOUT 610
  41 #endif
  42
  43 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
  44 #include <rte_pmd_fpga_5gnr_fec.h>
  45 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
  46 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
  47 #define VF_UL_5G_QUEUE_VALUE 4
  48 #define VF_DL_5G_QUEUE_VALUE 4
  49 #define UL_5G_BANDWIDTH 3
  50 #define DL_5G_BANDWIDTH 3
  51 #define UL_5G_LOAD_BALANCE 128
  52 #define DL_5G_LOAD_BALANCE 128
  53 #endif
  54
  55 #ifdef RTE_BASEBAND_ACC100
  56 #include <rte_acc100_cfg.h>
  57 #define ACC100PF_DRIVER_NAME   ("intel_acc100_pf")
  58 #define ACC100VF_DRIVER_NAME   ("intel_acc100_vf")
  59 #define ACC100_QMGR_NUM_AQS 16
  60 #define ACC100_QMGR_NUM_QGS 2
  61 #define ACC100_QMGR_AQ_DEPTH 5
  62 #define ACC100_QMGR_INVALID_IDX -1
  63 #define ACC100_QMGR_RR 1
  64 #define ACC100_QOS_GBR 0
  65 #endif
  66
  67 #define OPS_CACHE_SIZE 256U
  68 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  69
  70 #define SYNC_WAIT 0
  71 #define SYNC_START 1
  72 #define INVALID_OPAQUE -1
  73
  74 #define INVALID_QUEUE_ID -1
  75 /* Increment for next code block in external HARQ memory */
  76 #define HARQ_INCR 32768
  77 /* Headroom for filler LLRs insertion in HARQ buffer */
  78 #define FILLER_HEADROOM 1024
  79 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
  80 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
  81 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
  82 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
  83 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
  84 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
  85 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
  86 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
  87 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
  88
  89 static struct test_bbdev_vector test_vector;
  90
  91 /* Switch between PMD and Interrupt for throughput TC */
  92 static bool intr_enabled;
  93
  94 /* LLR arithmetic representation for numerical conversion */
  95 static int ldpc_llr_decimals;
  96 static int ldpc_llr_size;
  97 /* Keep track of the LDPC decoder device capability flag */
  98 static uint32_t ldpc_cap_flags;
  99
 100 /* Represents tested active devices */
 101 static struct active_device {
 102         const char *driver_name;
 103         uint8_t dev_id;
 104         uint16_t supported_ops;
 105         uint16_t queue_ids[MAX_QUEUES];
 106         uint16_t nb_queues;
 107         struct rte_mempool *ops_mempool;
 108         struct rte_mempool *in_mbuf_pool;
 109         struct rte_mempool *hard_out_mbuf_pool;
 110         struct rte_mempool *soft_out_mbuf_pool;
 111         struct rte_mempool *harq_in_mbuf_pool;
 112         struct rte_mempool *harq_out_mbuf_pool;
 113 } active_devs[RTE_BBDEV_MAX_DEVS];
 114
 115 static uint8_t nb_active_devs;
 116
 117 /* Data buffers used by BBDEV ops */
 118 struct test_buffers {
 119         struct rte_bbdev_op_data *inputs;
 120         struct rte_bbdev_op_data *hard_outputs;
 121         struct rte_bbdev_op_data *soft_outputs;
 122         struct rte_bbdev_op_data *harq_inputs;
 123         struct rte_bbdev_op_data *harq_outputs;
 124 };
 125
 126 /* Operation parameters specific for given test case */
 127 struct test_op_params {
 128         struct rte_mempool *mp;
 129         struct rte_bbdev_dec_op *ref_dec_op;
 130         struct rte_bbdev_enc_op *ref_enc_op;
 131         uint16_t burst_sz;
 132         uint16_t num_to_process;
 133         uint16_t num_lcores;
 134         int vector_mask;
 135         uint16_t sync;
 136         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
 137 };
 138
 139 /* Contains per lcore params */
 140 struct thread_params {
 141         uint8_t dev_id;
 142         uint16_t queue_id;
 143         uint32_t lcore_id;
 144         uint64_t start_time;
 145         double ops_per_sec;
 146         double mbps;
 147         uint8_t iter_count;
 148         double iter_average;
 149         double bler;
 150         uint16_t nb_dequeued;
 151         int16_t processing_status;
 152         uint16_t burst_sz;
 153         struct test_op_params *op_params;
 154         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
 155         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
 156 };
 157
 158 #ifdef RTE_BBDEV_OFFLOAD_COST
 159 /* Stores time statistics */
 160 struct test_time_stats {
 161         /* Stores software enqueue total working time */
 162         uint64_t enq_sw_total_time;
 163         /* Stores minimum value of software enqueue working time */
 164         uint64_t enq_sw_min_time;
 165         /* Stores maximum value of software enqueue working time */
 166         uint64_t enq_sw_max_time;
 167         /* Stores turbo enqueue total working time */
 168         uint64_t enq_acc_total_time;
 169         /* Stores minimum value of accelerator enqueue working time */
 170         uint64_t enq_acc_min_time;
 171         /* Stores maximum value of accelerator enqueue working time */
 172         uint64_t enq_acc_max_time;
 173         /* Stores dequeue total working time */
 174         uint64_t deq_total_time;
 175         /* Stores minimum value of dequeue working time */
 176         uint64_t deq_min_time;
 177         /* Stores maximum value of dequeue working time */
 178         uint64_t deq_max_time;
 179 };
 180 #endif
 181
 182 typedef int (test_case_function)(struct active_device *ad,
 183                 struct test_op_params *op_params);
 184
 185 static inline void
 186 mbuf_reset(struct rte_mbuf *m)
 187 {
 188         m->pkt_len = 0;
 189
 190         do {
 191                 m->data_len = 0;
 192                 m = m->next;
 193         } while (m != NULL);
 194 }
 195
 196 /* Read flag value 0/1 from bitmap */
 197 static inline bool
 198 check_bit(uint32_t bitmap, uint32_t bitmask)
 199 {
 200         return bitmap & bitmask;
 201 }
 202
 203 static inline void
 204 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 205 {
 206         ad->supported_ops |= (1 << op_type);
 207 }
 208
 209 static inline bool
 210 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 211 {
 212         return ad->supported_ops & (1 << op_type);
 213 }
 214
 215 static inline bool
 216 flags_match(uint32_t flags_req, uint32_t flags_present)
 217 {
 218         return (flags_req & flags_present) == flags_req;
 219 }
 220
 221 static void
 222 clear_soft_out_cap(uint32_t *op_flags)
 223 {
 224         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 225         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 226         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 227 }
 228
 229 /* This API is to convert all the test vector op data entries
 230  * to big endian format. It is used when the device supports
 231  * the input in the big endian format.
 232  */
 233 static inline void
 234 convert_op_data_to_be(void)
 235 {
 236         struct op_data_entries *op;
 237         enum op_data_type type;
 238         uint8_t nb_segs, *rem_data, temp;
 239         uint32_t *data, len;
 240         int complete, rem, i, j;
 241
 242         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
 243                 nb_segs = test_vector.entries[type].nb_segments;
 244                 op = &test_vector.entries[type];
 245
 246                 /* Invert byte endianness for all the segments */
 247                 for (i = 0; i < nb_segs; ++i) {
 248                         len = op->segments[i].length;
 249                         data = op->segments[i].addr;
 250
 251                         /* Swap complete u32 bytes */
 252                         complete = len / 4;
 253                         for (j = 0; j < complete; j++)
 254                                 data[j] = rte_bswap32(data[j]);
 255
 256                         /* Swap any remaining bytes */
 257                         rem = len % 4;
 258                         rem_data = (uint8_t *)&data[j];
 259                         for (j = 0; j < rem/2; j++) {
 260                                 temp = rem_data[j];
 261                                 rem_data[j] = rem_data[rem - j - 1];
 262                                 rem_data[rem - j - 1] = temp;
 263                         }
 264                 }
 265         }
 266 }
 267
 268 static int
 269 check_dev_cap(const struct rte_bbdev_info *dev_info)
 270 {
 271         unsigned int i;
 272         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
 273                 nb_harq_inputs, nb_harq_outputs;
 274         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 275         uint8_t dev_data_endianness = dev_info->drv.data_endianness;
 276
 277         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 278         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 279         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 280         nb_harq_inputs  = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
 281         nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
 282
 283         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 284                 if (op_cap->type != test_vector.op_type)
 285                         continue;
 286
 287                 if (dev_data_endianness == RTE_BIG_ENDIAN)
 288                         convert_op_data_to_be();
 289
 290                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 291                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 292                                         &op_cap->cap.turbo_dec;
 293                         /* Ignore lack of soft output capability, just skip
 294                          * checking if soft output is valid.
 295                          */
 296                         if ((test_vector.turbo_dec.op_flags &
 297                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 298                                         !(cap->capability_flags &
 299                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 300                                 printf(
 301                                         "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 302                                         dev_info->dev_name);
 303                                 clear_soft_out_cap(
 304                                         &test_vector.turbo_dec.op_flags);
 305                         }
 306
 307                         if (!flags_match(test_vector.turbo_dec.op_flags,
 308                                         cap->capability_flags))
 309                                 return TEST_FAILED;
 310                         if (nb_inputs > cap->num_buffers_src) {
 311                                 printf("Too many inputs defined: %u, max: %u\n",
 312                                         nb_inputs, cap->num_buffers_src);
 313                                 return TEST_FAILED;
 314                         }
 315                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 316                                         (test_vector.turbo_dec.op_flags &
 317                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 318                                 printf(
 319                                         "Too many soft outputs defined: %u, max: %u\n",
 320                                                 nb_soft_outputs,
 321                                                 cap->num_buffers_soft_out);
 322                                 return TEST_FAILED;
 323                         }
 324                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 325                                 printf(
 326                                         "Too many hard outputs defined: %u, max: %u\n",
 327                                                 nb_hard_outputs,
 328                                                 cap->num_buffers_hard_out);
 329                                 return TEST_FAILED;
 330                         }
 331                         if (intr_enabled && !(cap->capability_flags &
 332                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 333                                 printf(
 334                                         "Dequeue interrupts are not supported!\n");
 335                                 return TEST_FAILED;
 336                         }
 337
 338                         return TEST_SUCCESS;
 339                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 340                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 341                                         &op_cap->cap.turbo_enc;
 342
 343                         if (!flags_match(test_vector.turbo_enc.op_flags,
 344                                         cap->capability_flags))
 345                                 return TEST_FAILED;
 346                         if (nb_inputs > cap->num_buffers_src) {
 347                                 printf("Too many inputs defined: %u, max: %u\n",
 348                                         nb_inputs, cap->num_buffers_src);
 349                                 return TEST_FAILED;
 350                         }
 351                         if (nb_hard_outputs > cap->num_buffers_dst) {
 352                                 printf(
 353                                         "Too many hard outputs defined: %u, max: %u\n",
 354                                         nb_hard_outputs, cap->num_buffers_dst);
 355                                 return TEST_FAILED;
 356                         }
 357                         if (intr_enabled && !(cap->capability_flags &
 358                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 359                                 printf(
 360                                         "Dequeue interrupts are not supported!\n");
 361                                 return TEST_FAILED;
 362                         }
 363
 364                         return TEST_SUCCESS;
 365                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
 366                         const struct rte_bbdev_op_cap_ldpc_enc *cap =
 367                                         &op_cap->cap.ldpc_enc;
 368
 369                         if (!flags_match(test_vector.ldpc_enc.op_flags,
 370                                         cap->capability_flags)){
 371                                 printf("Flag Mismatch\n");
 372                                 return TEST_FAILED;
 373                         }
 374                         if (nb_inputs > cap->num_buffers_src) {
 375                                 printf("Too many inputs defined: %u, max: %u\n",
 376                                         nb_inputs, cap->num_buffers_src);
 377                                 return TEST_FAILED;
 378                         }
 379                         if (nb_hard_outputs > cap->num_buffers_dst) {
 380                                 printf(
 381                                         "Too many hard outputs defined: %u, max: %u\n",
 382                                         nb_hard_outputs, cap->num_buffers_dst);
 383                                 return TEST_FAILED;
 384                         }
 385                         if (intr_enabled && !(cap->capability_flags &
 386                                         RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
 387                                 printf(
 388                                         "Dequeue interrupts are not supported!\n");
 389                                 return TEST_FAILED;
 390                         }
 391
 392                         return TEST_SUCCESS;
 393                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
 394                         const struct rte_bbdev_op_cap_ldpc_dec *cap =
 395                                         &op_cap->cap.ldpc_dec;
 396
 397                         if (!flags_match(test_vector.ldpc_dec.op_flags,
 398                                         cap->capability_flags)){
 399                                 printf("Flag Mismatch\n");
 400                                 return TEST_FAILED;
 401                         }
 402                         if (nb_inputs > cap->num_buffers_src) {
 403                                 printf("Too many inputs defined: %u, max: %u\n",
 404                                         nb_inputs, cap->num_buffers_src);
 405                                 return TEST_FAILED;
 406                         }
 407                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 408                                 printf(
 409                                         "Too many hard outputs defined: %u, max: %u\n",
 410                                         nb_hard_outputs,
 411                                         cap->num_buffers_hard_out);
 412                                 return TEST_FAILED;
 413                         }
 414                         if (nb_harq_inputs > cap->num_buffers_hard_out) {
 415                                 printf(
 416                                         "Too many HARQ inputs defined: %u, max: %u\n",
 417                                         nb_harq_inputs,
 418                                         cap->num_buffers_hard_out);
 419                                 return TEST_FAILED;
 420                         }
 421                         if (nb_harq_outputs > cap->num_buffers_hard_out) {
 422                                 printf(
 423                                         "Too many HARQ outputs defined: %u, max: %u\n",
 424                                         nb_harq_outputs,
 425                                         cap->num_buffers_hard_out);
 426                                 return TEST_FAILED;
 427                         }
 428                         if (intr_enabled && !(cap->capability_flags &
 429                                         RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
 430                                 printf(
 431                                         "Dequeue interrupts are not supported!\n");
 432                                 return TEST_FAILED;
 433                         }
 434                         if (intr_enabled && (test_vector.ldpc_dec.op_flags &
 435                                 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
 436                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
 437                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
 438                                         ))) {
 439                                 printf("Skip loop-back with interrupt\n");
 440                                 return TEST_FAILED;
 441                         }
 442                         return TEST_SUCCESS;
 443                 }
 444         }
 445
 446         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 447                 return TEST_SUCCESS; /* Special case for NULL device */
 448
 449         return TEST_FAILED;
 450 }
 451
 452 /* calculates optimal mempool size not smaller than the val */
 453 static unsigned int
 454 optimal_mempool_size(unsigned int val)
 455 {
 456         return rte_align32pow2(val + 1) - 1;
 457 }
 458
 459 /* allocates mbuf mempool for inputs and outputs */
 460 static struct rte_mempool *
 461 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 462                 int socket_id, unsigned int mbuf_pool_size,
 463                 const char *op_type_str)
 464 {
 465         unsigned int i;
 466         uint32_t max_seg_sz = 0;
 467         char pool_name[RTE_MEMPOOL_NAMESIZE];
 468
 469         /* find max input segment size */
 470         for (i = 0; i < entries->nb_segments; ++i)
 471                 if (entries->segments[i].length > max_seg_sz)
 472                         max_seg_sz = entries->segments[i].length;
 473
 474         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 475                         dev_id);
 476         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 477                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
 478                                         + FILLER_HEADROOM,
 479                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 480 }
 481
 482 static int
 483 create_mempools(struct active_device *ad, int socket_id,
 484                 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
 485 {
 486         struct rte_mempool *mp;
 487         unsigned int ops_pool_size, mbuf_pool_size = 0;
 488         char pool_name[RTE_MEMPOOL_NAMESIZE];
 489         const char *op_type_str;
 490         enum rte_bbdev_op_type op_type = org_op_type;
 491
 492         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 493         struct op_data_entries *hard_out =
 494                         &test_vector.entries[DATA_HARD_OUTPUT];
 495         struct op_data_entries *soft_out =
 496                         &test_vector.entries[DATA_SOFT_OUTPUT];
 497         struct op_data_entries *harq_in =
 498                         &test_vector.entries[DATA_HARQ_INPUT];
 499         struct op_data_entries *harq_out =
 500                         &test_vector.entries[DATA_HARQ_OUTPUT];
 501
 502         /* allocate ops mempool */
 503         ops_pool_size = optimal_mempool_size(RTE_MAX(
 504                         /* Ops used plus 1 reference op */
 505                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 506                         /* Minimal cache size plus 1 reference op */
 507                         (unsigned int)(1.5 * rte_lcore_count() *
 508                                         OPS_CACHE_SIZE + 1)),
 509                         OPS_POOL_SIZE_MIN));
 510
 511         if (org_op_type == RTE_BBDEV_OP_NONE)
 512                 op_type = RTE_BBDEV_OP_TURBO_ENC;
 513
 514         op_type_str = rte_bbdev_op_type_str(op_type);
 515         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 516
 517         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 518                         ad->dev_id);
 519         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 520                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 521         TEST_ASSERT_NOT_NULL(mp,
 522                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 523                         ops_pool_size,
 524                         ad->dev_id,
 525                         socket_id);
 526         ad->ops_mempool = mp;
 527
 528         /* Do not create inputs and outputs mbufs for BaseBand Null Device */
 529         if (org_op_type == RTE_BBDEV_OP_NONE)
 530                 return TEST_SUCCESS;
 531
 532         /* Inputs */
 533         if (in->nb_segments > 0) {
 534                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 535                                 in->nb_segments);
 536                 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
 537                                 mbuf_pool_size, "in");
 538                 TEST_ASSERT_NOT_NULL(mp,
 539                                 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 540                                 mbuf_pool_size,
 541                                 ad->dev_id,
 542                                 socket_id);
 543                 ad->in_mbuf_pool = mp;
 544         }
 545
 546         /* Hard outputs */
 547         if (hard_out->nb_segments > 0) {
 548                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 549                                 hard_out->nb_segments);
 550                 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
 551                                 mbuf_pool_size,
 552                                 "hard_out");
 553                 TEST_ASSERT_NOT_NULL(mp,
 554                                 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 555                                 mbuf_pool_size,
 556                                 ad->dev_id,
 557                                 socket_id);
 558                 ad->hard_out_mbuf_pool = mp;
 559         }
 560
 561         /* Soft outputs */
 562         if (soft_out->nb_segments > 0) {
 563                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 564                                 soft_out->nb_segments);
 565                 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
 566                                 mbuf_pool_size,
 567                                 "soft_out");
 568                 TEST_ASSERT_NOT_NULL(mp,
 569                                 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 570                                 mbuf_pool_size,
 571                                 ad->dev_id,
 572                                 socket_id);
 573                 ad->soft_out_mbuf_pool = mp;
 574         }
 575
 576         /* HARQ inputs */
 577         if (harq_in->nb_segments > 0) {
 578                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 579                                 harq_in->nb_segments);
 580                 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
 581                                 mbuf_pool_size,
 582                                 "harq_in");
 583                 TEST_ASSERT_NOT_NULL(mp,
 584                                 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
 585                                 mbuf_pool_size,
 586                                 ad->dev_id,
 587                                 socket_id);
 588                 ad->harq_in_mbuf_pool = mp;
 589         }
 590
 591         /* HARQ outputs */
 592         if (harq_out->nb_segments > 0) {
 593                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 594                                 harq_out->nb_segments);
 595                 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
 596                                 mbuf_pool_size,
 597                                 "harq_out");
 598                 TEST_ASSERT_NOT_NULL(mp,
 599                                 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
 600                                 mbuf_pool_size,
 601                                 ad->dev_id,
 602                                 socket_id);
 603                 ad->harq_out_mbuf_pool = mp;
 604         }
 605
 606         return TEST_SUCCESS;
 607 }
 608
 609 static int
 610 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 611                 struct test_bbdev_vector *vector)
 612 {
 613         int ret;
 614         unsigned int queue_id;
 615         struct rte_bbdev_queue_conf qconf;
 616         struct active_device *ad = &active_devs[nb_active_devs];
 617         unsigned int nb_queues;
 618         enum rte_bbdev_op_type op_type = vector->op_type;
 619
 620 /* Configure fpga lte fec with PF & VF values
 621  * if '-i' flag is set and using fpga device
 622  */
 623 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
 624         if ((get_init_device() == true) &&
 625                 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
 626                 struct rte_fpga_lte_fec_conf conf;
 627                 unsigned int i;
 628
 629                 printf("Configure FPGA LTE FEC Driver %s with default values\n",
 630                                 info->drv.driver_name);
 631
 632                 /* clear default configuration before initialization */
 633                 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf));
 634
 635                 /* Set PF mode :
 636                  * true if PF is used for data plane
 637                  * false for VFs
 638                  */
 639                 conf.pf_mode_en = true;
 640
 641                 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
 642                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 643                         conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
 644                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 645                         conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
 646                 }
 647
 648                 /* UL bandwidth. Needed for schedule algorithm */
 649                 conf.ul_bandwidth = UL_4G_BANDWIDTH;
 650                 /* DL bandwidth */
 651                 conf.dl_bandwidth = DL_4G_BANDWIDTH;
 652
 653                 /* UL & DL load Balance Factor to 64 */
 654                 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
 655                 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
 656
 657                 /**< FLR timeout value */
 658                 conf.flr_time_out = FLR_4G_TIMEOUT;
 659
 660                 /* setup FPGA PF with configuration information */
 661                 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf);
 662                 TEST_ASSERT_SUCCESS(ret,
 663                                 "Failed to configure 4G FPGA PF for bbdev %s",
 664                                 info->dev_name);
 665         }
 666 #endif
 667 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
 668         if ((get_init_device() == true) &&
 669                 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
 670                 struct rte_fpga_5gnr_fec_conf conf;
 671                 unsigned int i;
 672
 673                 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
 674                                 info->drv.driver_name);
 675
 676                 /* clear default configuration before initialization */
 677                 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
 678
 679                 /* Set PF mode :
 680                  * true if PF is used for data plane
 681                  * false for VFs
 682                  */
 683                 conf.pf_mode_en = true;
 684
 685                 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
 686                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 687                         conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
 688                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 689                         conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
 690                 }
 691
 692                 /* UL bandwidth. Needed for schedule algorithm */
 693                 conf.ul_bandwidth = UL_5G_BANDWIDTH;
 694                 /* DL bandwidth */
 695                 conf.dl_bandwidth = DL_5G_BANDWIDTH;
 696
 697                 /* UL & DL load Balance Factor to 64 */
 698                 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
 699                 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
 700
 701                 /* setup FPGA PF with configuration information */
 702                 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf);
 703                 TEST_ASSERT_SUCCESS(ret,
 704                                 "Failed to configure 5G FPGA PF for bbdev %s",
 705                                 info->dev_name);
 706         }
 707 #endif
 708 #ifdef RTE_BASEBAND_ACC100
 709         if ((get_init_device() == true) &&
 710                 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) {
 711                 struct rte_acc100_conf conf;
 712                 unsigned int i;
 713
 714                 printf("Configure ACC100 FEC Driver %s with default values\n",
 715                                 info->drv.driver_name);
 716
 717                 /* clear default configuration before initialization */
 718                 memset(&conf, 0, sizeof(struct rte_acc100_conf));
 719
 720                 /* Always set in PF mode for built-in configuration */
 721                 conf.pf_mode_en = true;
 722                 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) {
 723                         conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 724                         conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 725                         conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR;
 726                         conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 727                         conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
 728                         conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR;
 729                         conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 730                         conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 731                         conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR;
 732                         conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 733                         conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
 734                         conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR;
 735                 }
 736
 737                 conf.input_pos_llr_1_bit = true;
 738                 conf.output_pos_llr_1_bit = true;
 739                 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */
 740
 741                 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
 742                 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 743                 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 744                 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 745                 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
 746                 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 747                 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 748                 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 749                 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
 750                 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 751                 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 752                 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 753                 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
 754                 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
 755                 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
 756                 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
 757
 758                 /* setup PF with configuration information */
 759                 ret = rte_acc100_configure(info->dev_name, &conf);
 760                 TEST_ASSERT_SUCCESS(ret,
 761                                 "Failed to configure ACC100 PF for bbdev %s",
 762                                 info->dev_name);
 763         }
 764 #endif
 765         /* Let's refresh this now this is configured */
 766         rte_bbdev_info_get(dev_id, info);
 767         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 768         nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
 769
 770         /* setup device */
 771         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 772         if (ret < 0) {
 773                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 774                                 dev_id, nb_queues, info->socket_id, ret);
 775                 return TEST_FAILED;
 776         }
 777
 778         /* configure interrupts if needed */
 779         if (intr_enabled) {
 780                 ret = rte_bbdev_intr_enable(dev_id);
 781                 if (ret < 0) {
 782                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 783                                         ret);
 784                         return TEST_FAILED;
 785                 }
 786         }
 787
 788         /* setup device queues */
 789         qconf.socket = info->socket_id;
 790         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 791         qconf.priority = 0;
 792         qconf.deferred_start = 0;
 793         qconf.op_type = op_type;
 794
 795         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 796                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 797                 if (ret != 0) {
 798                         printf(
 799                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 800                                         queue_id, qconf.priority, dev_id);
 801                         qconf.priority++;
 802                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 803                                         &qconf);
 804                 }
 805                 if (ret != 0) {
 806                         printf("All queues on dev %u allocated: %u\n",
 807                                         dev_id, queue_id);
 808                         break;
 809                 }
 810                 ad->queue_ids[queue_id] = queue_id;
 811         }
 812         TEST_ASSERT(queue_id != 0,
 813                         "ERROR Failed to configure any queues on dev %u",
 814                         dev_id);
 815         ad->nb_queues = queue_id;
 816
 817         set_avail_op(ad, op_type);
 818
 819         return TEST_SUCCESS;
 820 }
 821
 822 static int
 823 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 824                 struct test_bbdev_vector *vector)
 825 {
 826         int ret;
 827
 828         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 829         active_devs[nb_active_devs].dev_id = dev_id;
 830
 831         ret = add_bbdev_dev(dev_id, info, vector);
 832         if (ret == TEST_SUCCESS)
 833                 ++nb_active_devs;
 834         return ret;
 835 }
 836
 837 static uint8_t
 838 populate_active_devices(void)
 839 {
 840         int ret;
 841         uint8_t dev_id;
 842         uint8_t nb_devs_added = 0;
 843         struct rte_bbdev_info info;
 844
 845         RTE_BBDEV_FOREACH(dev_id) {
 846                 rte_bbdev_info_get(dev_id, &info);
 847
 848                 if (check_dev_cap(&info)) {
 849                         printf(
 850                                 "Device %d (%s) does not support specified capabilities\n",
 851                                         dev_id, info.dev_name);
 852                         continue;
 853                 }
 854
 855                 ret = add_active_device(dev_id, &info, &test_vector);
 856                 if (ret != 0) {
 857                         printf("Adding active bbdev %s skipped\n",
 858                                         info.dev_name);
 859                         continue;
 860                 }
 861                 nb_devs_added++;
 862         }
 863
 864         return nb_devs_added;
 865 }
 866
 867 static int
 868 read_test_vector(void)
 869 {
 870         int ret;
 871
 872         memset(&test_vector, 0, sizeof(test_vector));
 873         printf("Test vector file = %s\n", get_vector_filename());
 874         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 875         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 876                         get_vector_filename());
 877
 878         return TEST_SUCCESS;
 879 }
 880
 881 static int
 882 testsuite_setup(void)
 883 {
 884         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 885
 886         if (populate_active_devices() == 0) {
 887                 printf("No suitable devices found!\n");
 888                 return TEST_SKIPPED;
 889         }
 890
 891         return TEST_SUCCESS;
 892 }
 893
 894 static int
 895 interrupt_testsuite_setup(void)
 896 {
 897         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 898
 899         /* Enable interrupts */
 900         intr_enabled = true;
 901
 902         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 903         if (populate_active_devices() == 0 ||
 904                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 905                 intr_enabled = false;
 906                 printf("No suitable devices found!\n");
 907                 return TEST_SKIPPED;
 908         }
 909
 910         return TEST_SUCCESS;
 911 }
 912
 913 static void
 914 testsuite_teardown(void)
 915 {
 916         uint8_t dev_id;
 917
 918         /* Unconfigure devices */
 919         RTE_BBDEV_FOREACH(dev_id)
 920                 rte_bbdev_close(dev_id);
 921
 922         /* Clear active devices structs. */
 923         memset(active_devs, 0, sizeof(active_devs));
 924         nb_active_devs = 0;
 925
 926         /* Disable interrupts */
 927         intr_enabled = false;
 928 }
 929
 930 static int
 931 ut_setup(void)
 932 {
 933         uint8_t i, dev_id;
 934
 935         for (i = 0; i < nb_active_devs; i++) {
 936                 dev_id = active_devs[i].dev_id;
 937                 /* reset bbdev stats */
 938                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 939                                 "Failed to reset stats of bbdev %u", dev_id);
 940                 /* start the device */
 941                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 942                                 "Failed to start bbdev %u", dev_id);
 943         }
 944
 945         return TEST_SUCCESS;
 946 }
 947
 948 static void
 949 ut_teardown(void)
 950 {
 951         uint8_t i, dev_id;
 952         struct rte_bbdev_stats stats;
 953
 954         for (i = 0; i < nb_active_devs; i++) {
 955                 dev_id = active_devs[i].dev_id;
 956                 /* read stats and print */
 957                 rte_bbdev_stats_get(dev_id, &stats);
 958                 /* Stop the device */
 959                 rte_bbdev_stop(dev_id);
 960         }
 961 }
 962
 963 static int
 964 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 965                 struct op_data_entries *ref_entries,
 966                 struct rte_mempool *mbuf_pool, const uint16_t n,
 967                 enum op_data_type op_type, uint16_t min_alignment)
 968 {
 969         int ret;
 970         unsigned int i, j;
 971         bool large_input = false;
 972
 973         for (i = 0; i < n; ++i) {
 974                 char *data;
 975                 struct op_data_buf *seg = &ref_entries->segments[0];
 976                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 977                 TEST_ASSERT_NOT_NULL(m_head,
 978                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 979                                 op_type, n * ref_entries->nb_segments,
 980                                 mbuf_pool->size);
 981
 982                 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
 983                         /*
 984                          * Special case when DPDK mbuf cannot handle
 985                          * the required input size
 986                          */
 987                         printf("Warning: Larger input size than DPDK mbuf %d\n",
 988                                         seg->length);
 989                         large_input = true;
 990                 }
 991                 bufs[i].data = m_head;
 992                 bufs[i].offset = 0;
 993                 bufs[i].length = 0;
 994
 995                 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
 996                         if ((op_type == DATA_INPUT) && large_input) {
 997                                 /* Allocate a fake overused mbuf */
 998                                 data = rte_malloc(NULL, seg->length, 0);
 999                                 TEST_ASSERT_NOT_NULL(data,
1000                                         "rte malloc failed with %u bytes",
1001                                         seg->length);
1002                                 memcpy(data, seg->addr, seg->length);
1003                                 m_head->buf_addr = data;
1004                                 m_head->buf_iova = rte_malloc_virt2iova(data);
1005                                 m_head->data_off = 0;
1006                                 m_head->data_len = seg->length;
1007                         } else {
1008                                 data = rte_pktmbuf_append(m_head, seg->length);
1009                                 TEST_ASSERT_NOT_NULL(data,
1010                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
1011                                         seg->length, op_type);
1012
1013                                 TEST_ASSERT(data == RTE_PTR_ALIGN(
1014                                                 data, min_alignment),
1015                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
1016                                         data, min_alignment);
1017                                 rte_memcpy(data, seg->addr, seg->length);
1018                         }
1019
1020                         bufs[i].length += seg->length;
1021
1022                         for (j = 1; j < ref_entries->nb_segments; ++j) {
1023                                 struct rte_mbuf *m_tail =
1024                                                 rte_pktmbuf_alloc(mbuf_pool);
1025                                 TEST_ASSERT_NOT_NULL(m_tail,
1026                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1027                                                 op_type,
1028                                                 n * ref_entries->nb_segments,
1029                                                 mbuf_pool->size);
1030                                 seg += 1;
1031
1032                                 data = rte_pktmbuf_append(m_tail, seg->length);
1033                                 TEST_ASSERT_NOT_NULL(data,
1034                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
1035                                                 seg->length, op_type);
1036
1037                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
1038                                                 min_alignment),
1039                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
1040                                                 data, min_alignment);
1041                                 rte_memcpy(data, seg->addr, seg->length);
1042                                 bufs[i].length += seg->length;
1043
1044                                 ret = rte_pktmbuf_chain(m_head, m_tail);
1045                                 TEST_ASSERT_SUCCESS(ret,
1046                                                 "Couldn't chain mbufs from %d data type mbuf pool",
1047                                                 op_type);
1048                         }
1049                 } else {
1050
1051                         /* allocate chained-mbuf for output buffer */
1052                         for (j = 1; j < ref_entries->nb_segments; ++j) {
1053                                 struct rte_mbuf *m_tail =
1054                                                 rte_pktmbuf_alloc(mbuf_pool);
1055                                 TEST_ASSERT_NOT_NULL(m_tail,
1056                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1057                                                 op_type,
1058                                                 n * ref_entries->nb_segments,
1059                                                 mbuf_pool->size);
1060
1061                                 ret = rte_pktmbuf_chain(m_head, m_tail);
1062                                 TEST_ASSERT_SUCCESS(ret,
1063                                                 "Couldn't chain mbufs from %d data type mbuf pool",
1064                                                 op_type);
1065                         }
1066                 }
1067         }
1068
1069         return 0;
1070 }
1071
1072 static int
1073 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
1074                 const int socket)
1075 {
1076         int i;
1077
1078         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
1079         if (*buffers == NULL) {
1080                 printf("WARNING: Failed to allocate op_data on socket %d\n",
1081                                 socket);
1082                 /* try to allocate memory on other detected sockets */
1083                 for (i = 0; i < socket; i++) {
1084                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
1085                         if (*buffers != NULL)
1086                                 break;
1087                 }
1088         }
1089
1090         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
1091 }
1092
1093 static void
1094 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
1095                 const uint16_t n, const int8_t max_llr_modulus)
1096 {
1097         uint16_t i, byte_idx;
1098
1099         for (i = 0; i < n; ++i) {
1100                 struct rte_mbuf *m = input_ops[i].data;
1101                 while (m != NULL) {
1102                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1103                                         input_ops[i].offset);
1104                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1105                                         ++byte_idx)
1106                                 llr[byte_idx] = round((double)max_llr_modulus *
1107                                                 llr[byte_idx] / INT8_MAX);
1108
1109                         m = m->next;
1110                 }
1111         }
1112 }
1113
1114 /*
1115  * We may have to insert filler bits
1116  * when they are required by the HARQ assumption
1117  */
1118 static void
1119 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1120                 const uint16_t n, struct test_op_params *op_params)
1121 {
1122         struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1123
1124         if (input_ops == NULL)
1125                 return;
1126         /* No need to add filler if not required by device */
1127         if (!(ldpc_cap_flags &
1128                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1129                 return;
1130         /* No need to add filler for loopback operation */
1131         if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1132                 return;
1133
1134         uint16_t i, j, parity_offset;
1135         for (i = 0; i < n; ++i) {
1136                 struct rte_mbuf *m = input_ops[i].data;
1137                 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1138                                 input_ops[i].offset);
1139                 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1140                                 * dec.z_c - dec.n_filler;
1141                 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1142                 m->data_len = new_hin_size;
1143                 input_ops[i].length = new_hin_size;
1144                 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1145                                 j--)
1146                         llr[j] = llr[j - dec.n_filler];
1147                 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1148                 for (j = 0; j < dec.n_filler; j++)
1149                         llr[parity_offset + j] = llr_max_pre_scaling;
1150         }
1151 }
1152
1153 static void
1154 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1155                 const uint16_t n, const int8_t llr_size,
1156                 const int8_t llr_decimals)
1157 {
1158         if (input_ops == NULL)
1159                 return;
1160
1161         uint16_t i, byte_idx;
1162
1163         int16_t llr_max, llr_min, llr_tmp;
1164         llr_max = (1 << (llr_size - 1)) - 1;
1165         llr_min = -llr_max;
1166         for (i = 0; i < n; ++i) {
1167                 struct rte_mbuf *m = input_ops[i].data;
1168                 while (m != NULL) {
1169                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1170                                         input_ops[i].offset);
1171                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1172                                         ++byte_idx) {
1173
1174                                 llr_tmp = llr[byte_idx];
1175                                 if (llr_decimals == 4)
1176                                         llr_tmp *= 8;
1177                                 else if (llr_decimals == 2)
1178                                         llr_tmp *= 2;
1179                                 else if (llr_decimals == 0)
1180                                         llr_tmp /= 2;
1181                                 llr_tmp = RTE_MIN(llr_max,
1182                                                 RTE_MAX(llr_min, llr_tmp));
1183                                 llr[byte_idx] = (int8_t) llr_tmp;
1184                         }
1185
1186                         m = m->next;
1187                 }
1188         }
1189 }
1190
1191
1192
1193 static int
1194 fill_queue_buffers(struct test_op_params *op_params,
1195                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1196                 struct rte_mempool *soft_out_mp,
1197                 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1198                 uint16_t queue_id,
1199                 const struct rte_bbdev_op_cap *capabilities,
1200                 uint16_t min_alignment, const int socket_id)
1201 {
1202         int ret;
1203         enum op_data_type type;
1204         const uint16_t n = op_params->num_to_process;
1205
1206         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1207                 in_mp,
1208                 soft_out_mp,
1209                 hard_out_mp,
1210                 harq_in_mp,
1211                 harq_out_mp,
1212         };
1213
1214         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1215                 &op_params->q_bufs[socket_id][queue_id].inputs,
1216                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1217                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1218                 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1219                 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1220         };
1221
1222         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1223                 struct op_data_entries *ref_entries =
1224                                 &test_vector.entries[type];
1225                 if (ref_entries->nb_segments == 0)
1226                         continue;
1227
1228                 ret = allocate_buffers_on_socket(queue_ops[type],
1229                                 n * sizeof(struct rte_bbdev_op_data),
1230                                 socket_id);
1231                 TEST_ASSERT_SUCCESS(ret,
1232                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
1233
1234                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1235                                 mbuf_pools[type], n, type, min_alignment);
1236                 TEST_ASSERT_SUCCESS(ret,
1237                                 "Couldn't init rte_bbdev_op_data structs");
1238         }
1239
1240         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1241                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1242                         capabilities->cap.turbo_dec.max_llr_modulus);
1243
1244         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1245                 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1246                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1247                 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1248                                 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1249                 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1250                                 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1251                 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1252                 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1253                 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1254                 if (!loopback && !llr_comp)
1255                         ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1256                                         ldpc_llr_size, ldpc_llr_decimals);
1257                 if (!loopback && !harq_comp)
1258                         ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1259                                         ldpc_llr_size, ldpc_llr_decimals);
1260                 if (!loopback)
1261                         ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1262                                         op_params);
1263         }
1264
1265         return 0;
1266 }
1267
1268 static void
1269 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1270 {
1271         unsigned int i, j;
1272
1273         rte_mempool_free(ad->ops_mempool);
1274         rte_mempool_free(ad->in_mbuf_pool);
1275         rte_mempool_free(ad->hard_out_mbuf_pool);
1276         rte_mempool_free(ad->soft_out_mbuf_pool);
1277         rte_mempool_free(ad->harq_in_mbuf_pool);
1278         rte_mempool_free(ad->harq_out_mbuf_pool);
1279
1280         for (i = 0; i < rte_lcore_count(); ++i) {
1281                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1282                         rte_free(op_params->q_bufs[j][i].inputs);
1283                         rte_free(op_params->q_bufs[j][i].hard_outputs);
1284                         rte_free(op_params->q_bufs[j][i].soft_outputs);
1285                         rte_free(op_params->q_bufs[j][i].harq_inputs);
1286                         rte_free(op_params->q_bufs[j][i].harq_outputs);
1287                 }
1288         }
1289 }
1290
1291 static void
1292 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1293                 unsigned int start_idx,
1294                 struct rte_bbdev_op_data *inputs,
1295                 struct rte_bbdev_op_data *hard_outputs,
1296                 struct rte_bbdev_op_data *soft_outputs,
1297                 struct rte_bbdev_dec_op *ref_op)
1298 {
1299         unsigned int i;
1300         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1301
1302         for (i = 0; i < n; ++i) {
1303                 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1304                         ops[i]->turbo_dec.tb_params.ea =
1305                                         turbo_dec->tb_params.ea;
1306                         ops[i]->turbo_dec.tb_params.eb =
1307                                         turbo_dec->tb_params.eb;
1308                         ops[i]->turbo_dec.tb_params.k_pos =
1309                                         turbo_dec->tb_params.k_pos;
1310                         ops[i]->turbo_dec.tb_params.k_neg =
1311                                         turbo_dec->tb_params.k_neg;
1312                         ops[i]->turbo_dec.tb_params.c =
1313                                         turbo_dec->tb_params.c;
1314                         ops[i]->turbo_dec.tb_params.c_neg =
1315                                         turbo_dec->tb_params.c_neg;
1316                         ops[i]->turbo_dec.tb_params.cab =
1317                                         turbo_dec->tb_params.cab;
1318                         ops[i]->turbo_dec.tb_params.r =
1319                                         turbo_dec->tb_params.r;
1320                 } else {
1321                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1322                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1323                 }
1324
1325                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1326                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1327                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1328                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1329                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1330                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1331                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1332
1333                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1334                 ops[i]->turbo_dec.input = inputs[start_idx + i];
1335                 if (soft_outputs != NULL)
1336                         ops[i]->turbo_dec.soft_output =
1337                                 soft_outputs[start_idx + i];
1338         }
1339 }
1340
1341 static void
1342 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1343                 unsigned int start_idx,
1344                 struct rte_bbdev_op_data *inputs,
1345                 struct rte_bbdev_op_data *outputs,
1346                 struct rte_bbdev_enc_op *ref_op)
1347 {
1348         unsigned int i;
1349         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1350         for (i = 0; i < n; ++i) {
1351                 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1352                         ops[i]->turbo_enc.tb_params.ea =
1353                                         turbo_enc->tb_params.ea;
1354                         ops[i]->turbo_enc.tb_params.eb =
1355                                         turbo_enc->tb_params.eb;
1356                         ops[i]->turbo_enc.tb_params.k_pos =
1357                                         turbo_enc->tb_params.k_pos;
1358                         ops[i]->turbo_enc.tb_params.k_neg =
1359                                         turbo_enc->tb_params.k_neg;
1360                         ops[i]->turbo_enc.tb_params.c =
1361                                         turbo_enc->tb_params.c;
1362                         ops[i]->turbo_enc.tb_params.c_neg =
1363                                         turbo_enc->tb_params.c_neg;
1364                         ops[i]->turbo_enc.tb_params.cab =
1365                                         turbo_enc->tb_params.cab;
1366                         ops[i]->turbo_enc.tb_params.ncb_pos =
1367                                         turbo_enc->tb_params.ncb_pos;
1368                         ops[i]->turbo_enc.tb_params.ncb_neg =
1369                                         turbo_enc->tb_params.ncb_neg;
1370                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1371                 } else {
1372                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1373                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1374                         ops[i]->turbo_enc.cb_params.ncb =
1375                                         turbo_enc->cb_params.ncb;
1376                 }
1377                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1378                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1379                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1380
1381                 ops[i]->turbo_enc.output = outputs[start_idx + i];
1382                 ops[i]->turbo_enc.input = inputs[start_idx + i];
1383         }
1384 }
1385
1386
1387 /* Returns a random number drawn from a normal distribution
1388  * with mean of 0 and variance of 1
1389  * Marsaglia algorithm
1390  */
1391 static double
1392 randn(int n)
1393 {
1394         double S, Z, U1, U2, u, v, fac;
1395
1396         do {
1397                 U1 = (double)rand() / RAND_MAX;
1398                 U2 = (double)rand() / RAND_MAX;
1399                 u = 2. * U1 - 1.;
1400                 v = 2. * U2 - 1.;
1401                 S = u * u + v * v;
1402         } while (S >= 1 || S == 0);
1403         fac = sqrt(-2. * log(S) / S);
1404         Z = (n % 2) ? u * fac : v * fac;
1405         return Z;
1406 }
1407
1408 static inline double
1409 maxstar(double A, double B)
1410 {
1411         if (fabs(A - B) > 5)
1412                 return RTE_MAX(A, B);
1413         else
1414                 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1415 }
1416
1417 /*
1418  * Generate Qm LLRS for Qm==8
1419  * Modulation, AWGN and LLR estimation from max log development
1420  */
1421 static void
1422 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1423 {
1424         int qm = 8;
1425         int qam = 256;
1426         int m, k;
1427         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1428         /* 5.1.4 of TS38.211 */
1429         const double symbols_I[256] = {
1430                         5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1431                         5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1432                         11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1433                         15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1434                         15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1435                         1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1436                         1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1437                         15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1438                         13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1439                         -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1440                         -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1441                         -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1442                         -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1443                         -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1444                         -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1445                         -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1446                         -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1447                         -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1448                         -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1449                         -13, -13, -15, -15, -13, -13, -15, -15};
1450         const double symbols_Q[256] = {
1451                         5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1452                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1453                         15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1454                         11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1455                         15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1456                         -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1457                         -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1458                         -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1459                         -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1460                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1461                         7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1462                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1463                         13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1464                         3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1465                         13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1466                         -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1467                         -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1468                         -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1469                         -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1470                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1471         /* Average constellation point energy */
1472         N0 *= 170.0;
1473         for (k = 0; k < qm; k++)
1474                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1475         /* 5.1.4 of TS38.211 */
1476         I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1477                         (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1478         Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1479                         (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1480         /* AWGN channel */
1481         I += sqrt(N0 / 2) * randn(0);
1482         Q += sqrt(N0 / 2) * randn(1);
1483         /*
1484          * Calculate the log of the probability that each of
1485          * the constellation points was transmitted
1486          */
1487         for (m = 0; m < qam; m++)
1488                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1489                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1490         /* Calculate an LLR for each of the k_64QAM bits in the set */
1491         for (k = 0; k < qm; k++) {
1492                 p0 = -999999;
1493                 p1 = -999999;
1494                 /* For each constellation point */
1495                 for (m = 0; m < qam; m++) {
1496                         if ((m >> (qm - k - 1)) & 1)
1497                                 p1 = maxstar(p1, log_syml_prob[m]);
1498                         else
1499                                 p0 = maxstar(p0, log_syml_prob[m]);
1500                 }
1501                 /* Calculate the LLR */
1502                 llr_ = p0 - p1;
1503                 llr_ *= (1 << ldpc_llr_decimals);
1504                 llr_ = round(llr_);
1505                 if (llr_ > llr_max)
1506                         llr_ = llr_max;
1507                 if (llr_ < -llr_max)
1508                         llr_ = -llr_max;
1509                 llrs[qm * i + k] = (int8_t) llr_;
1510         }
1511 }
1512
1513
1514 /*
1515  * Generate Qm LLRS for Qm==6
1516  * Modulation, AWGN and LLR estimation from max log development
1517  */
1518 static void
1519 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1520 {
1521         int qm = 6;
1522         int qam = 64;
1523         int m, k;
1524         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1525         /* 5.1.4 of TS38.211 */
1526         const double symbols_I[64] = {
1527                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1528                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1529                         -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1530                         -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1531                         -5, -5, -7, -7, -5, -5, -7, -7};
1532         const double symbols_Q[64] = {
1533                         3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1534                         -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1535                         -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1536                         5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1537                         -3, -1, -3, -1, -5, -7, -5, -7};
1538         /* Average constellation point energy */
1539         N0 *= 42.0;
1540         for (k = 0; k < qm; k++)
1541                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1542         /* 5.1.4 of TS38.211 */
1543         I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1544         Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1545         /* AWGN channel */
1546         I += sqrt(N0 / 2) * randn(0);
1547         Q += sqrt(N0 / 2) * randn(1);
1548         /*
1549          * Calculate the log of the probability that each of
1550          * the constellation points was transmitted
1551          */
1552         for (m = 0; m < qam; m++)
1553                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1554                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1555         /* Calculate an LLR for each of the k_64QAM bits in the set */
1556         for (k = 0; k < qm; k++) {
1557                 p0 = -999999;
1558                 p1 = -999999;
1559                 /* For each constellation point */
1560                 for (m = 0; m < qam; m++) {
1561                         if ((m >> (qm - k - 1)) & 1)
1562                                 p1 = maxstar(p1, log_syml_prob[m]);
1563                         else
1564                                 p0 = maxstar(p0, log_syml_prob[m]);
1565                 }
1566                 /* Calculate the LLR */
1567                 llr_ = p0 - p1;
1568                 llr_ *= (1 << ldpc_llr_decimals);
1569                 llr_ = round(llr_);
1570                 if (llr_ > llr_max)
1571                         llr_ = llr_max;
1572                 if (llr_ < -llr_max)
1573                         llr_ = -llr_max;
1574                 llrs[qm * i + k] = (int8_t) llr_;
1575         }
1576 }
1577
1578 /*
1579  * Generate Qm LLRS for Qm==4
1580  * Modulation, AWGN and LLR estimation from max log development
1581  */
1582 static void
1583 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1584 {
1585         int qm = 4;
1586         int qam = 16;
1587         int m, k;
1588         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1589         /* 5.1.4 of TS38.211 */
1590         const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1591                         -1, -1, -3, -3, -1, -1, -3, -3};
1592         const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1593                         1, 3, 1, 3, -1, -3, -1, -3};
1594         /* Average constellation point energy */
1595         N0 *= 10.0;
1596         for (k = 0; k < qm; k++)
1597                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1598         /* 5.1.4 of TS38.211 */
1599         I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1600         Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1601         /* AWGN channel */
1602         I += sqrt(N0 / 2) * randn(0);
1603         Q += sqrt(N0 / 2) * randn(1);
1604         /*
1605          * Calculate the log of the probability that each of
1606          * the constellation points was transmitted
1607          */
1608         for (m = 0; m < qam; m++)
1609                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1610                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1611         /* Calculate an LLR for each of the k_64QAM bits in the set */
1612         for (k = 0; k < qm; k++) {
1613                 p0 = -999999;
1614                 p1 = -999999;
1615                 /* For each constellation point */
1616                 for (m = 0; m < qam; m++) {
1617                         if ((m >> (qm - k - 1)) & 1)
1618                                 p1 = maxstar(p1, log_syml_prob[m]);
1619                         else
1620                                 p0 = maxstar(p0, log_syml_prob[m]);
1621                 }
1622                 /* Calculate the LLR */
1623                 llr_ = p0 - p1;
1624                 llr_ *= (1 << ldpc_llr_decimals);
1625                 llr_ = round(llr_);
1626                 if (llr_ > llr_max)
1627                         llr_ = llr_max;
1628                 if (llr_ < -llr_max)
1629                         llr_ = -llr_max;
1630                 llrs[qm * i + k] = (int8_t) llr_;
1631         }
1632 }
1633
1634 static void
1635 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1636 {
1637         double b, b1, n;
1638         double coeff = 2.0 * sqrt(N0);
1639
1640         /* Ignore in vectors rare quasi null LLRs not to be saturated */
1641         if (llrs[j] < 8 && llrs[j] > -8)
1642                 return;
1643
1644         /* Note don't change sign here */
1645         n = randn(j % 2);
1646         b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1647                         + coeff * n) / N0;
1648         b = b1 * (1 << ldpc_llr_decimals);
1649         b = round(b);
1650         if (b > llr_max)
1651                 b = llr_max;
1652         if (b < -llr_max)
1653                 b = -llr_max;
1654         llrs[j] = (int8_t) b;
1655 }
1656
1657 /* Generate LLR for a given SNR */
1658 static void
1659 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1660                 struct rte_bbdev_dec_op *ref_op)
1661 {
1662         struct rte_mbuf *m;
1663         uint16_t qm;
1664         uint32_t i, j, e, range;
1665         double N0, llr_max;
1666
1667         e = ref_op->ldpc_dec.cb_params.e;
1668         qm = ref_op->ldpc_dec.q_m;
1669         llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1670         range = e / qm;
1671         N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1672
1673         for (i = 0; i < n; ++i) {
1674                 m = inputs[i].data;
1675                 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1676                 if (qm == 8) {
1677                         for (j = 0; j < range; ++j)
1678                                 gen_qm8_llr(llrs, j, N0, llr_max);
1679                 } else if (qm == 6) {
1680                         for (j = 0; j < range; ++j)
1681                                 gen_qm6_llr(llrs, j, N0, llr_max);
1682                 } else if (qm == 4) {
1683                         for (j = 0; j < range; ++j)
1684                                 gen_qm4_llr(llrs, j, N0, llr_max);
1685                 } else {
1686                         for (j = 0; j < e; ++j)
1687                                 gen_qm2_llr(llrs, j, N0, llr_max);
1688                 }
1689         }
1690 }
1691
1692 static void
1693 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1694                 unsigned int start_idx,
1695                 struct rte_bbdev_op_data *inputs,
1696                 struct rte_bbdev_op_data *hard_outputs,
1697                 struct rte_bbdev_op_data *soft_outputs,
1698                 struct rte_bbdev_op_data *harq_inputs,
1699                 struct rte_bbdev_op_data *harq_outputs,
1700                 struct rte_bbdev_dec_op *ref_op)
1701 {
1702         unsigned int i;
1703         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1704
1705         for (i = 0; i < n; ++i) {
1706                 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1707                         ops[i]->ldpc_dec.tb_params.ea =
1708                                         ldpc_dec->tb_params.ea;
1709                         ops[i]->ldpc_dec.tb_params.eb =
1710                                         ldpc_dec->tb_params.eb;
1711                         ops[i]->ldpc_dec.tb_params.c =
1712                                         ldpc_dec->tb_params.c;
1713                         ops[i]->ldpc_dec.tb_params.cab =
1714                                         ldpc_dec->tb_params.cab;
1715                         ops[i]->ldpc_dec.tb_params.r =
1716                                         ldpc_dec->tb_params.r;
1717                 } else {
1718                         ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1719                 }
1720
1721                 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1722                 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1723                 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1724                 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1725                 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1726                 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1727                 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1728                 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1729                 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1730
1731                 if (hard_outputs != NULL)
1732                         ops[i]->ldpc_dec.hard_output =
1733                                         hard_outputs[start_idx + i];
1734                 if (inputs != NULL)
1735                         ops[i]->ldpc_dec.input =
1736                                         inputs[start_idx + i];
1737                 if (soft_outputs != NULL)
1738                         ops[i]->ldpc_dec.soft_output =
1739                                         soft_outputs[start_idx + i];
1740                 if (harq_inputs != NULL)
1741                         ops[i]->ldpc_dec.harq_combined_input =
1742                                         harq_inputs[start_idx + i];
1743                 if (harq_outputs != NULL)
1744                         ops[i]->ldpc_dec.harq_combined_output =
1745                                         harq_outputs[start_idx + i];
1746         }
1747 }
1748
1749
1750 static void
1751 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1752                 unsigned int start_idx,
1753                 struct rte_bbdev_op_data *inputs,
1754                 struct rte_bbdev_op_data *outputs,
1755                 struct rte_bbdev_enc_op *ref_op)
1756 {
1757         unsigned int i;
1758         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1759         for (i = 0; i < n; ++i) {
1760                 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1761                         ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1762                         ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1763                         ops[i]->ldpc_enc.tb_params.cab =
1764                                         ldpc_enc->tb_params.cab;
1765                         ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1766                         ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1767                 } else {
1768                         ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1769                 }
1770                 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1771                 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1772                 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1773                 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1774                 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1775                 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1776                 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1777                 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1778                 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1779                 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1780         }
1781 }
1782
1783 static int
1784 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1785                 unsigned int order_idx, const int expected_status)
1786 {
1787         int status = op->status;
1788         /* ignore parity mismatch false alarms for long iterations */
1789         if (get_iter_max() >= 10) {
1790                 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1791                                 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1792                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1793                         status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1794                 }
1795                 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1796                                 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1797                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1798                         status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1799                 }
1800         }
1801
1802         TEST_ASSERT(status == expected_status,
1803                         "op_status (%d) != expected_status (%d)",
1804                         op->status, expected_status);
1805
1806         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1807                         "Ordering error, expected %p, got %p",
1808                         (void *)(uintptr_t)order_idx, op->opaque_data);
1809
1810         return TEST_SUCCESS;
1811 }
1812
1813 static int
1814 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1815                 unsigned int order_idx, const int expected_status)
1816 {
1817         TEST_ASSERT(op->status == expected_status,
1818                         "op_status (%d) != expected_status (%d)",
1819                         op->status, expected_status);
1820
1821         if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1822                 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1823                                 "Ordering error, expected %p, got %p",
1824                                 (void *)(uintptr_t)order_idx, op->opaque_data);
1825
1826         return TEST_SUCCESS;
1827 }
1828
1829 static inline int
1830 validate_op_chain(struct rte_bbdev_op_data *op,
1831                 struct op_data_entries *orig_op)
1832 {
1833         uint8_t i;
1834         struct rte_mbuf *m = op->data;
1835         uint8_t nb_dst_segments = orig_op->nb_segments;
1836         uint32_t total_data_size = 0;
1837
1838         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1839                         "Number of segments differ in original (%u) and filled (%u) op",
1840                         nb_dst_segments, m->nb_segs);
1841
1842         /* Validate each mbuf segment length */
1843         for (i = 0; i < nb_dst_segments; ++i) {
1844                 /* Apply offset to the first mbuf segment */
1845                 uint16_t offset = (i == 0) ? op->offset : 0;
1846                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1847                 total_data_size += orig_op->segments[i].length;
1848
1849                 TEST_ASSERT(orig_op->segments[i].length == data_len,
1850                                 "Length of segment differ in original (%u) and filled (%u) op",
1851                                 orig_op->segments[i].length, data_len);
1852                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1853                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1854                                 data_len,
1855                                 "Output buffers (CB=%u) are not equal", i);
1856                 m = m->next;
1857         }
1858
1859         /* Validate total mbuf pkt length */
1860         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1861         TEST_ASSERT(total_data_size == pkt_len,
1862                         "Length of data differ in original (%u) and filled (%u) op",
1863                         total_data_size, pkt_len);
1864
1865         return TEST_SUCCESS;
1866 }
1867
1868 /*
1869  * Compute K0 for a given configuration for HARQ output length computation
1870  * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1871  */
1872 static inline uint16_t
1873 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1874 {
1875         if (rv_index == 0)
1876                 return 0;
1877         uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1878         if (n_cb == n) {
1879                 if (rv_index == 1)
1880                         return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1881                 else if (rv_index == 2)
1882                         return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1883                 else
1884                         return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1885         }
1886         /* LBRM case - includes a division by N */
1887         if (rv_index == 1)
1888                 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1889                                 / n) * z_c;
1890         else if (rv_index == 2)
1891                 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1892                                 / n) * z_c;
1893         else
1894                 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1895                                 / n) * z_c;
1896 }
1897
1898 /* HARQ output length including the Filler bits */
1899 static inline uint16_t
1900 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1901 {
1902         uint16_t k0 = 0;
1903         uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1904         k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1905         /* Compute RM out size and number of rows */
1906         uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1907                         * ops_ld->z_c - ops_ld->n_filler;
1908         uint16_t deRmOutSize = RTE_MIN(
1909                         k0 + ops_ld->cb_params.e +
1910                         ((k0 > parity_offset) ?
1911                                         0 : ops_ld->n_filler),
1912                                         ops_ld->n_cb);
1913         uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1914                         / ops_ld->z_c);
1915         uint16_t harq_output_len = numRows * ops_ld->z_c;
1916         return harq_output_len;
1917 }
1918
1919 static inline int
1920 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1921                 struct op_data_entries *orig_op,
1922                 struct rte_bbdev_op_ldpc_dec *ops_ld)
1923 {
1924         uint8_t i;
1925         uint32_t j, jj, k;
1926         struct rte_mbuf *m = op->data;
1927         uint8_t nb_dst_segments = orig_op->nb_segments;
1928         uint32_t total_data_size = 0;
1929         int8_t *harq_orig, *harq_out, abs_harq_origin;
1930         uint32_t byte_error = 0, cum_error = 0, error;
1931         int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1932         int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1933         uint16_t parity_offset;
1934
1935         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1936                         "Number of segments differ in original (%u) and filled (%u) op",
1937                         nb_dst_segments, m->nb_segs);
1938
1939         /* Validate each mbuf segment length */
1940         for (i = 0; i < nb_dst_segments; ++i) {
1941                 /* Apply offset to the first mbuf segment */
1942                 uint16_t offset = (i == 0) ? op->offset : 0;
1943                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1944                 total_data_size += orig_op->segments[i].length;
1945
1946                 TEST_ASSERT(orig_op->segments[i].length <
1947                                 (uint32_t)(data_len + 64),
1948                                 "Length of segment differ in original (%u) and filled (%u) op",
1949                                 orig_op->segments[i].length, data_len);
1950                 harq_orig = (int8_t *) orig_op->segments[i].addr;
1951                 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1952
1953                 if (!(ldpc_cap_flags &
1954                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1955                                 ) || (ops_ld->op_flags &
1956                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1957                         data_len -= ops_ld->z_c;
1958                         parity_offset = data_len;
1959                 } else {
1960                         /* Compute RM out size and number of rows */
1961                         parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1962                                         * ops_ld->z_c - ops_ld->n_filler;
1963                         uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1964                                         ops_ld->n_filler;
1965                         if (data_len > deRmOutSize)
1966                                 data_len = deRmOutSize;
1967                         if (data_len > orig_op->segments[i].length)
1968                                 data_len = orig_op->segments[i].length;
1969                 }
1970                 /*
1971                  * HARQ output can have minor differences
1972                  * due to integer representation and related scaling
1973                  */
1974                 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1975                         if (j == parity_offset) {
1976                                 /* Special Handling of the filler bits */
1977                                 for (k = 0; k < ops_ld->n_filler; k++) {
1978                                         if (harq_out[jj] !=
1979                                                         llr_max_pre_scaling) {
1980                                                 printf("HARQ Filler issue %d: %d %d\n",
1981                                                         jj, harq_out[jj],
1982                                                         llr_max);
1983                                                 byte_error++;
1984                                         }
1985                                         jj++;
1986                                 }
1987                         }
1988                         if (!(ops_ld->op_flags &
1989                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1990                                 if (ldpc_llr_decimals > 1)
1991                                         harq_out[jj] = (harq_out[jj] + 1)
1992                                                 >> (ldpc_llr_decimals - 1);
1993                                 /* Saturated to S7 */
1994                                 if (harq_orig[j] > llr_max)
1995                                         harq_orig[j] = llr_max;
1996                                 if (harq_orig[j] < -llr_max)
1997                                         harq_orig[j] = -llr_max;
1998                         }
1999                         if (harq_orig[j] != harq_out[jj]) {
2000                                 error = (harq_orig[j] > harq_out[jj]) ?
2001                                                 harq_orig[j] - harq_out[jj] :
2002                                                 harq_out[jj] - harq_orig[j];
2003                                 abs_harq_origin = harq_orig[j] > 0 ?
2004                                                         harq_orig[j] :
2005                                                         -harq_orig[j];
2006                                 /* Residual quantization error */
2007                                 if ((error > 8 && (abs_harq_origin <
2008                                                 (llr_max - 16))) ||
2009                                                 (error > 16)) {
2010                                         printf("HARQ mismatch %d: exp %d act %d => %d\n",
2011                                                         j, harq_orig[j],
2012                                                         harq_out[jj], error);
2013                                         byte_error++;
2014                                         cum_error += error;
2015                                 }
2016                         }
2017                 }
2018                 m = m->next;
2019         }
2020
2021         if (byte_error)
2022                 TEST_ASSERT(byte_error <= 1,
2023                                 "HARQ output mismatch (%d) %d",
2024                                 byte_error, cum_error);
2025
2026         /* Validate total mbuf pkt length */
2027         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
2028         TEST_ASSERT(total_data_size < pkt_len + 64,
2029                         "Length of data differ in original (%u) and filled (%u) op",
2030                         total_data_size, pkt_len);
2031
2032         return TEST_SUCCESS;
2033 }
2034
2035 static int
2036 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2037                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2038 {
2039         unsigned int i;
2040         int ret;
2041         struct op_data_entries *hard_data_orig =
2042                         &test_vector.entries[DATA_HARD_OUTPUT];
2043         struct op_data_entries *soft_data_orig =
2044                         &test_vector.entries[DATA_SOFT_OUTPUT];
2045         struct rte_bbdev_op_turbo_dec *ops_td;
2046         struct rte_bbdev_op_data *hard_output;
2047         struct rte_bbdev_op_data *soft_output;
2048         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
2049
2050         for (i = 0; i < n; ++i) {
2051                 ops_td = &ops[i]->turbo_dec;
2052                 hard_output = &ops_td->hard_output;
2053                 soft_output = &ops_td->soft_output;
2054
2055                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2056                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2057                                         "Returned iter_count (%d) > expected iter_count (%d)",
2058                                         ops_td->iter_count, ref_td->iter_count);
2059                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2060                 TEST_ASSERT_SUCCESS(ret,
2061                                 "Checking status and ordering for decoder failed");
2062
2063                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2064                                 hard_data_orig),
2065                                 "Hard output buffers (CB=%u) are not equal",
2066                                 i);
2067
2068                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
2069                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2070                                         soft_data_orig),
2071                                         "Soft output buffers (CB=%u) are not equal",
2072                                         i);
2073         }
2074
2075         return TEST_SUCCESS;
2076 }
2077
2078 /* Check Number of code blocks errors */
2079 static int
2080 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
2081 {
2082         unsigned int i;
2083         struct op_data_entries *hard_data_orig =
2084                         &test_vector.entries[DATA_HARD_OUTPUT];
2085         struct rte_bbdev_op_ldpc_dec *ops_td;
2086         struct rte_bbdev_op_data *hard_output;
2087         int errors = 0;
2088         struct rte_mbuf *m;
2089
2090         for (i = 0; i < n; ++i) {
2091                 ops_td = &ops[i]->ldpc_dec;
2092                 hard_output = &ops_td->hard_output;
2093                 m = hard_output->data;
2094                 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
2095                                 hard_data_orig->segments[0].addr,
2096                                 hard_data_orig->segments[0].length))
2097                         errors++;
2098         }
2099         return errors;
2100 }
2101
2102 static int
2103 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2104                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2105 {
2106         unsigned int i;
2107         int ret;
2108         struct op_data_entries *hard_data_orig =
2109                         &test_vector.entries[DATA_HARD_OUTPUT];
2110         struct op_data_entries *soft_data_orig =
2111                         &test_vector.entries[DATA_SOFT_OUTPUT];
2112         struct op_data_entries *harq_data_orig =
2113                                 &test_vector.entries[DATA_HARQ_OUTPUT];
2114         struct rte_bbdev_op_ldpc_dec *ops_td;
2115         struct rte_bbdev_op_data *hard_output;
2116         struct rte_bbdev_op_data *harq_output;
2117         struct rte_bbdev_op_data *soft_output;
2118         struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2119
2120         for (i = 0; i < n; ++i) {
2121                 ops_td = &ops[i]->ldpc_dec;
2122                 hard_output = &ops_td->hard_output;
2123                 harq_output = &ops_td->harq_combined_output;
2124                 soft_output = &ops_td->soft_output;
2125
2126                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2127                 TEST_ASSERT_SUCCESS(ret,
2128                                 "Checking status and ordering for decoder failed");
2129                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2130                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2131                                         "Returned iter_count (%d) > expected iter_count (%d)",
2132                                         ops_td->iter_count, ref_td->iter_count);
2133                 /*
2134                  * We can ignore output data when the decoding failed to
2135                  * converge or for loop-back cases
2136                  */
2137                 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2138                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2139                                 ) && (
2140                                 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2141                                                 )) == 0)
2142                         TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2143                                         hard_data_orig),
2144                                         "Hard output buffers (CB=%u) are not equal",
2145                                         i);
2146
2147                 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2148                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2149                                         soft_data_orig),
2150                                         "Soft output buffers (CB=%u) are not equal",
2151                                         i);
2152                 if (ref_op->ldpc_dec.op_flags &
2153                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2154                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2155                                         harq_data_orig, ops_td),
2156                                         "HARQ output buffers (CB=%u) are not equal",
2157                                         i);
2158                 }
2159                 if (ref_op->ldpc_dec.op_flags &
2160                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2161                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2162                                         harq_data_orig, ops_td),
2163                                         "HARQ output buffers (CB=%u) are not equal",
2164                                         i);
2165
2166         }
2167
2168         return TEST_SUCCESS;
2169 }
2170
2171
2172 static int
2173 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2174                 struct rte_bbdev_enc_op *ref_op)
2175 {
2176         unsigned int i;
2177         int ret;
2178         struct op_data_entries *hard_data_orig =
2179                         &test_vector.entries[DATA_HARD_OUTPUT];
2180
2181         for (i = 0; i < n; ++i) {
2182                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2183                 TEST_ASSERT_SUCCESS(ret,
2184                                 "Checking status and ordering for encoder failed");
2185                 TEST_ASSERT_SUCCESS(validate_op_chain(
2186                                 &ops[i]->turbo_enc.output,
2187                                 hard_data_orig),
2188                                 "Output buffers (CB=%u) are not equal",
2189                                 i);
2190         }
2191
2192         return TEST_SUCCESS;
2193 }
2194
2195 static int
2196 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2197                 struct rte_bbdev_enc_op *ref_op)
2198 {
2199         unsigned int i;
2200         int ret;
2201         struct op_data_entries *hard_data_orig =
2202                         &test_vector.entries[DATA_HARD_OUTPUT];
2203
2204         for (i = 0; i < n; ++i) {
2205                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2206                 TEST_ASSERT_SUCCESS(ret,
2207                                 "Checking status and ordering for encoder failed");
2208                 TEST_ASSERT_SUCCESS(validate_op_chain(
2209                                 &ops[i]->ldpc_enc.output,
2210                                 hard_data_orig),
2211                                 "Output buffers (CB=%u) are not equal",
2212                                 i);
2213         }
2214
2215         return TEST_SUCCESS;
2216 }
2217
2218 static void
2219 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2220 {
2221         unsigned int i;
2222         struct op_data_entries *entry;
2223
2224         op->turbo_dec = test_vector.turbo_dec;
2225         entry = &test_vector.entries[DATA_INPUT];
2226         for (i = 0; i < entry->nb_segments; ++i)
2227                 op->turbo_dec.input.length +=
2228                                 entry->segments[i].length;
2229 }
2230
2231 static void
2232 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2233 {
2234         unsigned int i;
2235         struct op_data_entries *entry;
2236
2237         op->ldpc_dec = test_vector.ldpc_dec;
2238         entry = &test_vector.entries[DATA_INPUT];
2239         for (i = 0; i < entry->nb_segments; ++i)
2240                 op->ldpc_dec.input.length +=
2241                                 entry->segments[i].length;
2242         if (test_vector.ldpc_dec.op_flags &
2243                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2244                 entry = &test_vector.entries[DATA_HARQ_INPUT];
2245                 for (i = 0; i < entry->nb_segments; ++i)
2246                         op->ldpc_dec.harq_combined_input.length +=
2247                                 entry->segments[i].length;
2248         }
2249 }
2250
2251
2252 static void
2253 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2254 {
2255         unsigned int i;
2256         struct op_data_entries *entry;
2257
2258         op->turbo_enc = test_vector.turbo_enc;
2259         entry = &test_vector.entries[DATA_INPUT];
2260         for (i = 0; i < entry->nb_segments; ++i)
2261                 op->turbo_enc.input.length +=
2262                                 entry->segments[i].length;
2263 }
2264
2265 static void
2266 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2267 {
2268         unsigned int i;
2269         struct op_data_entries *entry;
2270
2271         op->ldpc_enc = test_vector.ldpc_enc;
2272         entry = &test_vector.entries[DATA_INPUT];
2273         for (i = 0; i < entry->nb_segments; ++i)
2274                 op->ldpc_enc.input.length +=
2275                                 entry->segments[i].length;
2276 }
2277
2278 static uint32_t
2279 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2280 {
2281         uint8_t i;
2282         uint32_t c, r, tb_size = 0;
2283
2284         if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2285                 tb_size = op->turbo_dec.tb_params.k_neg;
2286         } else {
2287                 c = op->turbo_dec.tb_params.c;
2288                 r = op->turbo_dec.tb_params.r;
2289                 for (i = 0; i < c-r; i++)
2290                         tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2291                                 op->turbo_dec.tb_params.k_neg :
2292                                 op->turbo_dec.tb_params.k_pos;
2293         }
2294         return tb_size;
2295 }
2296
2297 static uint32_t
2298 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2299 {
2300         uint8_t i;
2301         uint32_t c, r, tb_size = 0;
2302         uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2303
2304         if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2305                 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2306         } else {
2307                 c = op->ldpc_dec.tb_params.c;
2308                 r = op->ldpc_dec.tb_params.r;
2309                 for (i = 0; i < c-r; i++)
2310                         tb_size += sys_cols * op->ldpc_dec.z_c
2311                                         - op->ldpc_dec.n_filler;
2312         }
2313         return tb_size;
2314 }
2315
2316 static uint32_t
2317 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2318 {
2319         uint8_t i;
2320         uint32_t c, r, tb_size = 0;
2321
2322         if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2323                 tb_size = op->turbo_enc.tb_params.k_neg;
2324         } else {
2325                 c = op->turbo_enc.tb_params.c;
2326                 r = op->turbo_enc.tb_params.r;
2327                 for (i = 0; i < c-r; i++)
2328                         tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2329                                 op->turbo_enc.tb_params.k_neg :
2330                                 op->turbo_enc.tb_params.k_pos;
2331         }
2332         return tb_size;
2333 }
2334
2335 static uint32_t
2336 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2337 {
2338         uint8_t i;
2339         uint32_t c, r, tb_size = 0;
2340         uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2341
2342         if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2343                 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2344         } else {
2345                 c = op->turbo_enc.tb_params.c;
2346                 r = op->turbo_enc.tb_params.r;
2347                 for (i = 0; i < c-r; i++)
2348                         tb_size += sys_cols * op->ldpc_enc.z_c
2349                                         - op->ldpc_enc.n_filler;
2350         }
2351         return tb_size;
2352 }
2353
2354
2355 static int
2356 init_test_op_params(struct test_op_params *op_params,
2357                 enum rte_bbdev_op_type op_type, const int expected_status,
2358                 const int vector_mask, struct rte_mempool *ops_mp,
2359                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2360 {
2361         int ret = 0;
2362         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2363                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2364                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2365                                 &op_params->ref_dec_op, 1);
2366         else
2367                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2368                                 &op_params->ref_enc_op, 1);
2369
2370         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2371
2372         op_params->mp = ops_mp;
2373         op_params->burst_sz = burst_sz;
2374         op_params->num_to_process = num_to_process;
2375         op_params->num_lcores = num_lcores;
2376         op_params->vector_mask = vector_mask;
2377         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2378                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2379                 op_params->ref_dec_op->status = expected_status;
2380         else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2381                         || op_type == RTE_BBDEV_OP_LDPC_ENC)
2382                 op_params->ref_enc_op->status = expected_status;
2383         return 0;
2384 }
2385
2386 static int
2387 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2388                 struct test_op_params *op_params)
2389 {
2390         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2391         unsigned int i;
2392         struct active_device *ad;
2393         unsigned int burst_sz = get_burst_sz();
2394         enum rte_bbdev_op_type op_type = test_vector.op_type;
2395         const struct rte_bbdev_op_cap *capabilities = NULL;
2396
2397         ad = &active_devs[dev_id];
2398
2399         /* Check if device supports op_type */
2400         if (!is_avail_op(ad, test_vector.op_type))
2401                 return TEST_SUCCESS;
2402
2403         struct rte_bbdev_info info;
2404         rte_bbdev_info_get(ad->dev_id, &info);
2405         socket_id = GET_SOCKET(info.socket_id);
2406
2407         f_ret = create_mempools(ad, socket_id, op_type,
2408                         get_num_ops());
2409         if (f_ret != TEST_SUCCESS) {
2410                 printf("Couldn't create mempools");
2411                 goto fail;
2412         }
2413         if (op_type == RTE_BBDEV_OP_NONE)
2414                 op_type = RTE_BBDEV_OP_TURBO_ENC;
2415
2416         f_ret = init_test_op_params(op_params, test_vector.op_type,
2417                         test_vector.expected_status,
2418                         test_vector.mask,
2419                         ad->ops_mempool,
2420                         burst_sz,
2421                         get_num_ops(),
2422                         get_num_lcores());
2423         if (f_ret != TEST_SUCCESS) {
2424                 printf("Couldn't init test op params");
2425                 goto fail;
2426         }
2427
2428
2429         /* Find capabilities */
2430         const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2431         for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2432                 if (cap->type == test_vector.op_type) {
2433                         capabilities = cap;
2434                         break;
2435                 }
2436                 cap++;
2437         }
2438         TEST_ASSERT_NOT_NULL(capabilities,
2439                         "Couldn't find capabilities");
2440
2441         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2442                 create_reference_dec_op(op_params->ref_dec_op);
2443         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2444                 create_reference_enc_op(op_params->ref_enc_op);
2445         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2446                 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2447         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2448                 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2449
2450         for (i = 0; i < ad->nb_queues; ++i) {
2451                 f_ret = fill_queue_buffers(op_params,
2452                                 ad->in_mbuf_pool,
2453                                 ad->hard_out_mbuf_pool,
2454                                 ad->soft_out_mbuf_pool,
2455                                 ad->harq_in_mbuf_pool,
2456                                 ad->harq_out_mbuf_pool,
2457                                 ad->queue_ids[i],
2458                                 capabilities,
2459                                 info.drv.min_alignment,
2460                                 socket_id);
2461                 if (f_ret != TEST_SUCCESS) {
2462                         printf("Couldn't init queue buffers");
2463                         goto fail;
2464                 }
2465         }
2466
2467         /* Run test case function */
2468         t_ret = test_case_func(ad, op_params);
2469
2470         /* Free active device resources and return */
2471         free_buffers(ad, op_params);
2472         return t_ret;
2473
2474 fail:
2475         free_buffers(ad, op_params);
2476         return TEST_FAILED;
2477 }
2478
2479 /* Run given test function per active device per supported op type
2480  * per burst size.
2481  */
2482 static int
2483 run_test_case(test_case_function *test_case_func)
2484 {
2485         int ret = 0;
2486         uint8_t dev;
2487
2488         /* Alloc op_params */
2489         struct test_op_params *op_params = rte_zmalloc(NULL,
2490                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2491         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2492                         RTE_ALIGN(sizeof(struct test_op_params),
2493                                 RTE_CACHE_LINE_SIZE));
2494
2495         /* For each device run test case function */
2496         for (dev = 0; dev < nb_active_devs; ++dev)
2497                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2498
2499         rte_free(op_params);
2500
2501         return ret;
2502 }
2503
2504
2505 /* Push back the HARQ output from DDR to host */
2506 static void
2507 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2508                 struct rte_bbdev_dec_op **ops,
2509                 const uint16_t n)
2510 {
2511         uint16_t j;
2512         int save_status, ret;
2513         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2514         struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2515         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2516         bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2517         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2518         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2519         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2520         for (j = 0; j < n; ++j) {
2521                 if ((loopback && mem_out) || hc_out) {
2522                         save_status = ops[j]->status;
2523                         ops[j]->ldpc_dec.op_flags =
2524                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2525                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2526                         if (h_comp)
2527                                 ops[j]->ldpc_dec.op_flags +=
2528                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2529                         ops[j]->ldpc_dec.harq_combined_input.offset =
2530                                         harq_offset;
2531                         ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2532                         harq_offset += HARQ_INCR;
2533                         if (!loopback)
2534                                 ops[j]->ldpc_dec.harq_combined_input.length =
2535                                 ops[j]->ldpc_dec.harq_combined_output.length;
2536                         rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2537                                         &ops[j], 1);
2538                         ret = 0;
2539                         while (ret == 0)
2540                                 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2541                                                 dev_id, queue_id,
2542                                                 &ops_deq[j], 1);
2543                         ops[j]->ldpc_dec.op_flags = flags;
2544                         ops[j]->status = save_status;
2545                 }
2546         }
2547 }
2548
2549 /*
2550  * Push back the HARQ output from HW DDR to Host
2551  * Preload HARQ memory input and adjust HARQ offset
2552  */
2553 static void
2554 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2555                 struct rte_bbdev_dec_op **ops, const uint16_t n,
2556                 bool preload)
2557 {
2558         uint16_t j;
2559         int deq;
2560         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2561         struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS];
2562         struct rte_bbdev_dec_op *ops_deq[MAX_OPS];
2563         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2564         bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2565         bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2566         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2567         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2568         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2569         if ((mem_in || hc_in) && preload) {
2570                 for (j = 0; j < n; ++j) {
2571                         save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input;
2572                         save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output;
2573                         ops[j]->ldpc_dec.op_flags =
2574                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2575                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2576                         if (h_comp)
2577                                 ops[j]->ldpc_dec.op_flags +=
2578                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2579                         ops[j]->ldpc_dec.harq_combined_output.offset =
2580                                         harq_offset;
2581                         ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2582                         harq_offset += HARQ_INCR;
2583                 }
2584                 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n);
2585                 deq = 0;
2586                 while (deq != n)
2587                         deq += rte_bbdev_dequeue_ldpc_dec_ops(
2588                                         dev_id, queue_id, &ops_deq[deq],
2589                                         n - deq);
2590                 /* Restore the operations */
2591                 for (j = 0; j < n; ++j) {
2592                         ops[j]->ldpc_dec.op_flags = flags;
2593                         ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j];
2594                         ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j];
2595                 }
2596         }
2597         harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2598         for (j = 0; j < n; ++j) {
2599                 /* Adjust HARQ offset when we reach external DDR */
2600                 if (mem_in || hc_in)
2601                         ops[j]->ldpc_dec.harq_combined_input.offset
2602                                 = harq_offset;
2603                 if (mem_out || hc_out)
2604                         ops[j]->ldpc_dec.harq_combined_output.offset
2605                                 = harq_offset;
2606                 harq_offset += HARQ_INCR;
2607         }
2608 }
2609
2610 static void
2611 dequeue_event_callback(uint16_t dev_id,
2612                 enum rte_bbdev_event_type event, void *cb_arg,
2613                 void *ret_param)
2614 {
2615         int ret;
2616         uint16_t i;
2617         uint64_t total_time;
2618         uint16_t deq, burst_sz, num_ops;
2619         uint16_t queue_id = *(uint16_t *) ret_param;
2620         struct rte_bbdev_info info;
2621         double tb_len_bits;
2622         struct thread_params *tp = cb_arg;
2623
2624         /* Find matching thread params using queue_id */
2625         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2626                 if (tp->queue_id == queue_id)
2627                         break;
2628
2629         if (i == MAX_QUEUES) {
2630                 printf("%s: Queue_id from interrupt details was not found!\n",
2631                                 __func__);
2632                 return;
2633         }
2634
2635         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2636                 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
2637                 printf(
2638                         "Dequeue interrupt handler called for incorrect event!\n");
2639                 return;
2640         }
2641
2642         burst_sz = __atomic_load_n(&tp->burst_sz, __ATOMIC_RELAXED);
2643         num_ops = tp->op_params->num_to_process;
2644
2645         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2646                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2647                                 &tp->dec_ops[
2648                                         __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)],
2649                                 burst_sz);
2650         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2651                 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2652                                 &tp->dec_ops[
2653                                         __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)],
2654                                 burst_sz);
2655         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2656                 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2657                                 &tp->enc_ops[
2658                                         __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)],
2659                                 burst_sz);
2660         else /*RTE_BBDEV_OP_TURBO_ENC*/
2661                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2662                                 &tp->enc_ops[
2663                                         __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)],
2664                                 burst_sz);
2665
2666         if (deq < burst_sz) {
2667                 printf(
2668                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2669                         burst_sz, deq);
2670                 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
2671                 return;
2672         }
2673
2674         if (__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) + deq < num_ops) {
2675                 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED);
2676                 return;
2677         }
2678
2679         total_time = rte_rdtsc_precise() - tp->start_time;
2680
2681         rte_bbdev_info_get(dev_id, &info);
2682
2683         ret = TEST_SUCCESS;
2684
2685         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2686                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2687                 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2688                                 tp->op_params->vector_mask);
2689                 /* get the max of iter_count for all dequeued ops */
2690                 for (i = 0; i < num_ops; ++i)
2691                         tp->iter_count = RTE_MAX(
2692                                         tp->dec_ops[i]->turbo_dec.iter_count,
2693                                         tp->iter_count);
2694                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2695         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2696                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2697                 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2698                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2699         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2700                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2701                 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2702                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2703         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2704                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2705                 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2706                                 tp->op_params->vector_mask);
2707                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2708         }
2709
2710         if (ret) {
2711                 printf("Buffers validation failed\n");
2712                 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
2713         }
2714
2715         switch (test_vector.op_type) {
2716         case RTE_BBDEV_OP_TURBO_DEC:
2717                 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2718                 break;
2719         case RTE_BBDEV_OP_TURBO_ENC:
2720                 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2721                 break;
2722         case RTE_BBDEV_OP_LDPC_DEC:
2723                 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2724                 break;
2725         case RTE_BBDEV_OP_LDPC_ENC:
2726                 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2727                 break;
2728         case RTE_BBDEV_OP_NONE:
2729                 tb_len_bits = 0.0;
2730                 break;
2731         default:
2732                 printf("Unknown op type: %d\n", test_vector.op_type);
2733                 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
2734                 return;
2735         }
2736
2737         tp->ops_per_sec += ((double)num_ops) /
2738                         ((double)total_time / (double)rte_get_tsc_hz());
2739         tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2740                         ((double)total_time / (double)rte_get_tsc_hz());
2741
2742         __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED);
2743 }
2744
2745 static int
2746 throughput_intr_lcore_ldpc_dec(void *arg)
2747 {
2748         struct thread_params *tp = arg;
2749         unsigned int enqueued;
2750         const uint16_t queue_id = tp->queue_id;
2751         const uint16_t burst_sz = tp->op_params->burst_sz;
2752         const uint16_t num_to_process = tp->op_params->num_to_process;
2753         struct rte_bbdev_dec_op *ops[num_to_process];
2754         struct test_buffers *bufs = NULL;
2755         struct rte_bbdev_info info;
2756         int ret, i, j;
2757         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2758         uint16_t num_to_enq, enq;
2759
2760         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2761                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2762         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2763                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2764
2765         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2766                         "BURST_SIZE should be <= %u", MAX_BURST);
2767
2768         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2769                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2770                         tp->dev_id, queue_id);
2771
2772         rte_bbdev_info_get(tp->dev_id, &info);
2773
2774         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2775                         "NUM_OPS cannot exceed %u for this device",
2776                         info.drv.queue_size_lim);
2777
2778         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2779
2780         __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
2781         __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2782
2783         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
2784
2785         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2786                                 num_to_process);
2787         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2788                         num_to_process);
2789         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2790                 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2791                                 bufs->hard_outputs, bufs->soft_outputs,
2792                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2793
2794         /* Set counter to validate the ordering */
2795         for (j = 0; j < num_to_process; ++j)
2796                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2797
2798         for (j = 0; j < TEST_REPETITIONS; ++j) {
2799                 for (i = 0; i < num_to_process; ++i) {
2800                         if (!loopback)
2801                                 rte_pktmbuf_reset(
2802                                         ops[i]->ldpc_dec.hard_output.data);
2803                         if (hc_out || loopback)
2804                                 mbuf_reset(
2805                                 ops[i]->ldpc_dec.harq_combined_output.data);
2806                 }
2807
2808                 tp->start_time = rte_rdtsc_precise();
2809                 for (enqueued = 0; enqueued < num_to_process;) {
2810                         num_to_enq = burst_sz;
2811
2812                         if (unlikely(num_to_process - enqueued < num_to_enq))
2813                                 num_to_enq = num_to_process - enqueued;
2814
2815                         enq = 0;
2816                         do {
2817                                 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2818                                                 tp->dev_id,
2819                                                 queue_id, &ops[enqueued],
2820                                                 num_to_enq);
2821                         } while (unlikely(num_to_enq != enq));
2822                         enqueued += enq;
2823
2824                         /* Write to thread burst_sz current number of enqueued
2825                          * descriptors. It ensures that proper number of
2826                          * descriptors will be dequeued in callback
2827                          * function - needed for last batch in case where
2828                          * the number of operations is not a multiple of
2829                          * burst size.
2830                          */
2831                         __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
2832
2833                         /* Wait until processing of previous batch is
2834                          * completed
2835                          */
2836                         rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
2837                 }
2838                 if (j != TEST_REPETITIONS - 1)
2839                         __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2840         }
2841
2842         return TEST_SUCCESS;
2843 }
2844
2845 static int
2846 throughput_intr_lcore_dec(void *arg)
2847 {
2848         struct thread_params *tp = arg;
2849         unsigned int enqueued;
2850         const uint16_t queue_id = tp->queue_id;
2851         const uint16_t burst_sz = tp->op_params->burst_sz;
2852         const uint16_t num_to_process = tp->op_params->num_to_process;
2853         struct rte_bbdev_dec_op *ops[num_to_process];
2854         struct test_buffers *bufs = NULL;
2855         struct rte_bbdev_info info;
2856         int ret, i, j;
2857         uint16_t num_to_enq, enq;
2858
2859         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2860                         "BURST_SIZE should be <= %u", MAX_BURST);
2861
2862         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2863                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2864                         tp->dev_id, queue_id);
2865
2866         rte_bbdev_info_get(tp->dev_id, &info);
2867
2868         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2869                         "NUM_OPS cannot exceed %u for this device",
2870                         info.drv.queue_size_lim);
2871
2872         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2873
2874         __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
2875         __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2876
2877         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
2878
2879         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2880                                 num_to_process);
2881         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2882                         num_to_process);
2883         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2884                 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2885                                 bufs->hard_outputs, bufs->soft_outputs,
2886                                 tp->op_params->ref_dec_op);
2887
2888         /* Set counter to validate the ordering */
2889         for (j = 0; j < num_to_process; ++j)
2890                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2891
2892         for (j = 0; j < TEST_REPETITIONS; ++j) {
2893                 for (i = 0; i < num_to_process; ++i)
2894                         rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2895
2896                 tp->start_time = rte_rdtsc_precise();
2897                 for (enqueued = 0; enqueued < num_to_process;) {
2898                         num_to_enq = burst_sz;
2899
2900                         if (unlikely(num_to_process - enqueued < num_to_enq))
2901                                 num_to_enq = num_to_process - enqueued;
2902
2903                         enq = 0;
2904                         do {
2905                                 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2906                                                 queue_id, &ops[enqueued],
2907                                                 num_to_enq);
2908                         } while (unlikely(num_to_enq != enq));
2909                         enqueued += enq;
2910
2911                         /* Write to thread burst_sz current number of enqueued
2912                          * descriptors. It ensures that proper number of
2913                          * descriptors will be dequeued in callback
2914                          * function - needed for last batch in case where
2915                          * the number of operations is not a multiple of
2916                          * burst size.
2917                          */
2918                         __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
2919
2920                         /* Wait until processing of previous batch is
2921                          * completed
2922                          */
2923                         rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
2924                 }
2925                 if (j != TEST_REPETITIONS - 1)
2926                         __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2927         }
2928
2929         return TEST_SUCCESS;
2930 }
2931
2932 static int
2933 throughput_intr_lcore_enc(void *arg)
2934 {
2935         struct thread_params *tp = arg;
2936         unsigned int enqueued;
2937         const uint16_t queue_id = tp->queue_id;
2938         const uint16_t burst_sz = tp->op_params->burst_sz;
2939         const uint16_t num_to_process = tp->op_params->num_to_process;
2940         struct rte_bbdev_enc_op *ops[num_to_process];
2941         struct test_buffers *bufs = NULL;
2942         struct rte_bbdev_info info;
2943         int ret, i, j;
2944         uint16_t num_to_enq, enq;
2945
2946         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2947                         "BURST_SIZE should be <= %u", MAX_BURST);
2948
2949         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2950                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2951                         tp->dev_id, queue_id);
2952
2953         rte_bbdev_info_get(tp->dev_id, &info);
2954
2955         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2956                         "NUM_OPS cannot exceed %u for this device",
2957                         info.drv.queue_size_lim);
2958
2959         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2960
2961         __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
2962         __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2963
2964         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
2965
2966         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2967                         num_to_process);
2968         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2969                         num_to_process);
2970         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2971                 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2972                                 bufs->hard_outputs, tp->op_params->ref_enc_op);
2973
2974         /* Set counter to validate the ordering */
2975         for (j = 0; j < num_to_process; ++j)
2976                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2977
2978         for (j = 0; j < TEST_REPETITIONS; ++j) {
2979                 for (i = 0; i < num_to_process; ++i)
2980                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2981
2982                 tp->start_time = rte_rdtsc_precise();
2983                 for (enqueued = 0; enqueued < num_to_process;) {
2984                         num_to_enq = burst_sz;
2985
2986                         if (unlikely(num_to_process - enqueued < num_to_enq))
2987                                 num_to_enq = num_to_process - enqueued;
2988
2989                         enq = 0;
2990                         do {
2991                                 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2992                                                 queue_id, &ops[enqueued],
2993                                                 num_to_enq);
2994                         } while (unlikely(enq != num_to_enq));
2995                         enqueued += enq;
2996
2997                         /* Write to thread burst_sz current number of enqueued
2998                          * descriptors. It ensures that proper number of
2999                          * descriptors will be dequeued in callback
3000                          * function - needed for last batch in case where
3001                          * the number of operations is not a multiple of
3002                          * burst size.
3003                          */
3004                         __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
3005
3006                         /* Wait until processing of previous batch is
3007                          * completed
3008                          */
3009                         rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
3010                 }
3011                 if (j != TEST_REPETITIONS - 1)
3012                         __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
3013         }
3014
3015         return TEST_SUCCESS;
3016 }
3017
3018
3019 static int
3020 throughput_intr_lcore_ldpc_enc(void *arg)
3021 {
3022         struct thread_params *tp = arg;
3023         unsigned int enqueued;
3024         const uint16_t queue_id = tp->queue_id;
3025         const uint16_t burst_sz = tp->op_params->burst_sz;
3026         const uint16_t num_to_process = tp->op_params->num_to_process;
3027         struct rte_bbdev_enc_op *ops[num_to_process];
3028         struct test_buffers *bufs = NULL;
3029         struct rte_bbdev_info info;
3030         int ret, i, j;
3031         uint16_t num_to_enq, enq;
3032
3033         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3034                         "BURST_SIZE should be <= %u", MAX_BURST);
3035
3036         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
3037                         "Failed to enable interrupts for dev: %u, queue_id: %u",
3038                         tp->dev_id, queue_id);
3039
3040         rte_bbdev_info_get(tp->dev_id, &info);
3041
3042         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
3043                         "NUM_OPS cannot exceed %u for this device",
3044                         info.drv.queue_size_lim);
3045
3046         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3047
3048         __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
3049         __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
3050
3051         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3052
3053         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
3054                         num_to_process);
3055         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3056                         num_to_process);
3057         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3058                 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
3059                                 bufs->inputs, bufs->hard_outputs,
3060                                 tp->op_params->ref_enc_op);
3061
3062         /* Set counter to validate the ordering */
3063         for (j = 0; j < num_to_process; ++j)
3064                 ops[j]->opaque_data = (void *)(uintptr_t)j;
3065
3066         for (j = 0; j < TEST_REPETITIONS; ++j) {
3067                 for (i = 0; i < num_to_process; ++i)
3068                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
3069
3070                 tp->start_time = rte_rdtsc_precise();
3071                 for (enqueued = 0; enqueued < num_to_process;) {
3072                         num_to_enq = burst_sz;
3073
3074                         if (unlikely(num_to_process - enqueued < num_to_enq))
3075                                 num_to_enq = num_to_process - enqueued;
3076
3077                         enq = 0;
3078                         do {
3079                                 enq += rte_bbdev_enqueue_ldpc_enc_ops(
3080                                                 tp->dev_id,
3081                                                 queue_id, &ops[enqueued],
3082                                                 num_to_enq);
3083                         } while (unlikely(enq != num_to_enq));
3084                         enqueued += enq;
3085
3086                         /* Write to thread burst_sz current number of enqueued
3087                          * descriptors. It ensures that proper number of
3088                          * descriptors will be dequeued in callback
3089                          * function - needed for last batch in case where
3090                          * the number of operations is not a multiple of
3091                          * burst size.
3092                          */
3093                         __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
3094
3095                         /* Wait until processing of previous batch is
3096                          * completed
3097                          */
3098                         rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
3099                 }
3100                 if (j != TEST_REPETITIONS - 1)
3101                         __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
3102         }
3103
3104         return TEST_SUCCESS;
3105 }
3106
3107 static int
3108 throughput_pmd_lcore_dec(void *arg)
3109 {
3110         struct thread_params *tp = arg;
3111         uint16_t enq, deq;
3112         uint64_t total_time = 0, start_time;
3113         const uint16_t queue_id = tp->queue_id;
3114         const uint16_t burst_sz = tp->op_params->burst_sz;
3115         const uint16_t num_ops = tp->op_params->num_to_process;
3116         struct rte_bbdev_dec_op *ops_enq[num_ops];
3117         struct rte_bbdev_dec_op *ops_deq[num_ops];
3118         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3119         struct test_buffers *bufs = NULL;
3120         int i, j, ret;
3121         struct rte_bbdev_info info;
3122         uint16_t num_to_enq;
3123
3124         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3125                         "BURST_SIZE should be <= %u", MAX_BURST);
3126
3127         rte_bbdev_info_get(tp->dev_id, &info);
3128
3129         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3130                         "NUM_OPS cannot exceed %u for this device",
3131                         info.drv.queue_size_lim);
3132
3133         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3134
3135         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3136
3137         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3138         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3139
3140         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3141                 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3142                                 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3143
3144         /* Set counter to validate the ordering */
3145         for (j = 0; j < num_ops; ++j)
3146                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3147
3148         for (i = 0; i < TEST_REPETITIONS; ++i) {
3149
3150                 for (j = 0; j < num_ops; ++j)
3151                         mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3152
3153                 start_time = rte_rdtsc_precise();
3154
3155                 for (enq = 0, deq = 0; enq < num_ops;) {
3156                         num_to_enq = burst_sz;
3157
3158                         if (unlikely(num_ops - enq < num_to_enq))
3159                                 num_to_enq = num_ops - enq;
3160
3161                         enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3162                                         queue_id, &ops_enq[enq], num_to_enq);
3163
3164                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3165                                         queue_id, &ops_deq[deq], enq - deq);
3166                 }
3167
3168                 /* dequeue the remaining */
3169                 while (deq < enq) {
3170                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3171                                         queue_id, &ops_deq[deq], enq - deq);
3172                 }
3173
3174                 total_time += rte_rdtsc_precise() - start_time;
3175         }
3176
3177         tp->iter_count = 0;
3178         /* get the max of iter_count for all dequeued ops */
3179         for (i = 0; i < num_ops; ++i) {
3180                 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3181                                 tp->iter_count);
3182         }
3183
3184         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3185                 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3186                                 tp->op_params->vector_mask);
3187                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3188         }
3189
3190         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3191
3192         double tb_len_bits = calc_dec_TB_size(ref_op);
3193
3194         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3195                         ((double)total_time / (double)rte_get_tsc_hz());
3196         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3197                         1000000.0) / ((double)total_time /
3198                         (double)rte_get_tsc_hz());
3199
3200         return TEST_SUCCESS;
3201 }
3202
3203 static int
3204 bler_pmd_lcore_ldpc_dec(void *arg)
3205 {
3206         struct thread_params *tp = arg;
3207         uint16_t enq, deq;
3208         uint64_t total_time = 0, start_time;
3209         const uint16_t queue_id = tp->queue_id;
3210         const uint16_t burst_sz = tp->op_params->burst_sz;
3211         const uint16_t num_ops = tp->op_params->num_to_process;
3212         struct rte_bbdev_dec_op *ops_enq[num_ops];
3213         struct rte_bbdev_dec_op *ops_deq[num_ops];
3214         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3215         struct test_buffers *bufs = NULL;
3216         int i, j, ret;
3217         float parity_bler = 0;
3218         struct rte_bbdev_info info;
3219         uint16_t num_to_enq;
3220         bool extDdr = check_bit(ldpc_cap_flags,
3221                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3222         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3223                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3224         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3225                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3226
3227         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3228                         "BURST_SIZE should be <= %u", MAX_BURST);
3229
3230         rte_bbdev_info_get(tp->dev_id, &info);
3231
3232         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3233                         "NUM_OPS cannot exceed %u for this device",
3234                         info.drv.queue_size_lim);
3235
3236         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3237
3238         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3239
3240         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3241         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3242
3243         /* For BLER tests we need to enable early termination */
3244         if (!check_bit(ref_op->ldpc_dec.op_flags,
3245                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3246                 ref_op->ldpc_dec.op_flags +=
3247                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3248         ref_op->ldpc_dec.iter_max = get_iter_max();
3249         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3250
3251         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3252                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3253                                 bufs->hard_outputs, bufs->soft_outputs,
3254                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3255         generate_llr_input(num_ops, bufs->inputs, ref_op);
3256
3257         /* Set counter to validate the ordering */
3258         for (j = 0; j < num_ops; ++j)
3259                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3260
3261         for (i = 0; i < 1; ++i) { /* Could add more iterations */
3262                 for (j = 0; j < num_ops; ++j) {
3263                         if (!loopback)
3264                                 mbuf_reset(
3265                                 ops_enq[j]->ldpc_dec.hard_output.data);
3266                         if (hc_out || loopback)
3267                                 mbuf_reset(
3268                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3269                 }
3270                 if (extDdr)
3271                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3272                                         num_ops, true);
3273                 start_time = rte_rdtsc_precise();
3274
3275                 for (enq = 0, deq = 0; enq < num_ops;) {
3276                         num_to_enq = burst_sz;
3277
3278                         if (unlikely(num_ops - enq < num_to_enq))
3279                                 num_to_enq = num_ops - enq;
3280
3281                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3282                                         queue_id, &ops_enq[enq], num_to_enq);
3283
3284                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3285                                         queue_id, &ops_deq[deq], enq - deq);
3286                 }
3287
3288                 /* dequeue the remaining */
3289                 while (deq < enq) {
3290                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3291                                         queue_id, &ops_deq[deq], enq - deq);
3292                 }
3293
3294                 total_time += rte_rdtsc_precise() - start_time;
3295         }
3296
3297         tp->iter_count = 0;
3298         tp->iter_average = 0;
3299         /* get the max of iter_count for all dequeued ops */
3300         for (i = 0; i < num_ops; ++i) {
3301                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3302                                 tp->iter_count);
3303                 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3304                 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3305                         parity_bler += 1.0;
3306         }
3307
3308         parity_bler /= num_ops; /* This one is based on SYND */
3309         tp->iter_average /= num_ops;
3310         tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3311
3312         if (test_vector.op_type != RTE_BBDEV_OP_NONE
3313                         && tp->bler == 0
3314                         && parity_bler == 0
3315                         && !hc_out) {
3316                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3317                                 tp->op_params->vector_mask);
3318                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3319         }
3320
3321         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3322
3323         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3324         tp->ops_per_sec = ((double)num_ops * 1) /
3325                         ((double)total_time / (double)rte_get_tsc_hz());
3326         tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3327                         1000000.0) / ((double)total_time /
3328                         (double)rte_get_tsc_hz());
3329
3330         return TEST_SUCCESS;
3331 }
3332
3333 static int
3334 throughput_pmd_lcore_ldpc_dec(void *arg)
3335 {
3336         struct thread_params *tp = arg;
3337         uint16_t enq, deq;
3338         uint64_t total_time = 0, start_time;
3339         const uint16_t queue_id = tp->queue_id;
3340         const uint16_t burst_sz = tp->op_params->burst_sz;
3341         const uint16_t num_ops = tp->op_params->num_to_process;
3342         struct rte_bbdev_dec_op *ops_enq[num_ops];
3343         struct rte_bbdev_dec_op *ops_deq[num_ops];
3344         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3345         struct test_buffers *bufs = NULL;
3346         int i, j, ret;
3347         struct rte_bbdev_info info;
3348         uint16_t num_to_enq;
3349         bool extDdr = check_bit(ldpc_cap_flags,
3350                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3351         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3352                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3353         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3354                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3355
3356         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3357                         "BURST_SIZE should be <= %u", MAX_BURST);
3358
3359         rte_bbdev_info_get(tp->dev_id, &info);
3360
3361         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3362                         "NUM_OPS cannot exceed %u for this device",
3363                         info.drv.queue_size_lim);
3364
3365         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3366
3367         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3368
3369         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3370         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3371
3372         /* For throughput tests we need to disable early termination */
3373         if (check_bit(ref_op->ldpc_dec.op_flags,
3374                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3375                 ref_op->ldpc_dec.op_flags -=
3376                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3377         ref_op->ldpc_dec.iter_max = get_iter_max();
3378         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3379
3380         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3381                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3382                                 bufs->hard_outputs, bufs->soft_outputs,
3383                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3384
3385         /* Set counter to validate the ordering */
3386         for (j = 0; j < num_ops; ++j)
3387                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3388
3389         for (i = 0; i < TEST_REPETITIONS; ++i) {
3390                 for (j = 0; j < num_ops; ++j) {
3391                         if (!loopback)
3392                                 mbuf_reset(
3393                                 ops_enq[j]->ldpc_dec.hard_output.data);
3394                         if (hc_out || loopback)
3395                                 mbuf_reset(
3396                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3397                 }
3398                 if (extDdr)
3399                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3400                                         num_ops, true);
3401                 start_time = rte_rdtsc_precise();
3402
3403                 for (enq = 0, deq = 0; enq < num_ops;) {
3404                         num_to_enq = burst_sz;
3405
3406                         if (unlikely(num_ops - enq < num_to_enq))
3407                                 num_to_enq = num_ops - enq;
3408
3409                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3410                                         queue_id, &ops_enq[enq], num_to_enq);
3411
3412                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3413                                         queue_id, &ops_deq[deq], enq - deq);
3414                 }
3415
3416                 /* dequeue the remaining */
3417                 while (deq < enq) {
3418                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3419                                         queue_id, &ops_deq[deq], enq - deq);
3420                 }
3421
3422                 total_time += rte_rdtsc_precise() - start_time;
3423         }
3424
3425         tp->iter_count = 0;
3426         /* get the max of iter_count for all dequeued ops */
3427         for (i = 0; i < num_ops; ++i) {
3428                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3429                                 tp->iter_count);
3430         }
3431         if (extDdr) {
3432                 /* Read loopback is not thread safe */
3433                 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3434         }
3435
3436         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3437                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3438                                 tp->op_params->vector_mask);
3439                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3440         }
3441
3442         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3443
3444         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3445
3446         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3447                         ((double)total_time / (double)rte_get_tsc_hz());
3448         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3449                         1000000.0) / ((double)total_time /
3450                         (double)rte_get_tsc_hz());
3451
3452         return TEST_SUCCESS;
3453 }
3454
3455 static int
3456 throughput_pmd_lcore_enc(void *arg)
3457 {
3458         struct thread_params *tp = arg;
3459         uint16_t enq, deq;
3460         uint64_t total_time = 0, start_time;
3461         const uint16_t queue_id = tp->queue_id;
3462         const uint16_t burst_sz = tp->op_params->burst_sz;
3463         const uint16_t num_ops = tp->op_params->num_to_process;
3464         struct rte_bbdev_enc_op *ops_enq[num_ops];
3465         struct rte_bbdev_enc_op *ops_deq[num_ops];
3466         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3467         struct test_buffers *bufs = NULL;
3468         int i, j, ret;
3469         struct rte_bbdev_info info;
3470         uint16_t num_to_enq;
3471
3472         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3473                         "BURST_SIZE should be <= %u", MAX_BURST);
3474
3475         rte_bbdev_info_get(tp->dev_id, &info);
3476
3477         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3478                         "NUM_OPS cannot exceed %u for this device",
3479                         info.drv.queue_size_lim);
3480
3481         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3482
3483         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3484
3485         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3486                         num_ops);
3487         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3488                         num_ops);
3489         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3490                 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3491                                 bufs->hard_outputs, ref_op);
3492
3493         /* Set counter to validate the ordering */
3494         for (j = 0; j < num_ops; ++j)
3495                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3496
3497         for (i = 0; i < TEST_REPETITIONS; ++i) {
3498
3499                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3500                         for (j = 0; j < num_ops; ++j)
3501                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3502
3503                 start_time = rte_rdtsc_precise();
3504
3505                 for (enq = 0, deq = 0; enq < num_ops;) {
3506                         num_to_enq = burst_sz;
3507
3508                         if (unlikely(num_ops - enq < num_to_enq))
3509                                 num_to_enq = num_ops - enq;
3510
3511                         enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3512                                         queue_id, &ops_enq[enq], num_to_enq);
3513
3514                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3515                                         queue_id, &ops_deq[deq], enq - deq);
3516                 }
3517
3518                 /* dequeue the remaining */
3519                 while (deq < enq) {
3520                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3521                                         queue_id, &ops_deq[deq], enq - deq);
3522                 }
3523
3524                 total_time += rte_rdtsc_precise() - start_time;
3525         }
3526
3527         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3528                 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3529                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3530         }
3531
3532         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3533
3534         double tb_len_bits = calc_enc_TB_size(ref_op);
3535
3536         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3537                         ((double)total_time / (double)rte_get_tsc_hz());
3538         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3539                         / 1000000.0) / ((double)total_time /
3540                         (double)rte_get_tsc_hz());
3541
3542         return TEST_SUCCESS;
3543 }
3544
3545 static int
3546 throughput_pmd_lcore_ldpc_enc(void *arg)
3547 {
3548         struct thread_params *tp = arg;
3549         uint16_t enq, deq;
3550         uint64_t total_time = 0, start_time;
3551         const uint16_t queue_id = tp->queue_id;
3552         const uint16_t burst_sz = tp->op_params->burst_sz;
3553         const uint16_t num_ops = tp->op_params->num_to_process;
3554         struct rte_bbdev_enc_op *ops_enq[num_ops];
3555         struct rte_bbdev_enc_op *ops_deq[num_ops];
3556         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3557         struct test_buffers *bufs = NULL;
3558         int i, j, ret;
3559         struct rte_bbdev_info info;
3560         uint16_t num_to_enq;
3561
3562         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3563                         "BURST_SIZE should be <= %u", MAX_BURST);
3564
3565         rte_bbdev_info_get(tp->dev_id, &info);
3566
3567         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3568                         "NUM_OPS cannot exceed %u for this device",
3569                         info.drv.queue_size_lim);
3570
3571         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3572
3573         rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3574
3575         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3576                         num_ops);
3577         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3578                         num_ops);
3579         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3580                 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3581                                 bufs->hard_outputs, ref_op);
3582
3583         /* Set counter to validate the ordering */
3584         for (j = 0; j < num_ops; ++j)
3585                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3586
3587         for (i = 0; i < TEST_REPETITIONS; ++i) {
3588
3589                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3590                         for (j = 0; j < num_ops; ++j)
3591                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3592
3593                 start_time = rte_rdtsc_precise();
3594
3595                 for (enq = 0, deq = 0; enq < num_ops;) {
3596                         num_to_enq = burst_sz;
3597
3598                         if (unlikely(num_ops - enq < num_to_enq))
3599                                 num_to_enq = num_ops - enq;
3600
3601                         enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3602                                         queue_id, &ops_enq[enq], num_to_enq);
3603
3604                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3605                                         queue_id, &ops_deq[deq], enq - deq);
3606                 }
3607
3608                 /* dequeue the remaining */
3609                 while (deq < enq) {
3610                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3611                                         queue_id, &ops_deq[deq], enq - deq);
3612                 }
3613
3614                 total_time += rte_rdtsc_precise() - start_time;
3615         }
3616
3617         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3618                 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3619                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3620         }
3621
3622         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3623
3624         double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3625
3626         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3627                         ((double)total_time / (double)rte_get_tsc_hz());
3628         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3629                         / 1000000.0) / ((double)total_time /
3630                         (double)rte_get_tsc_hz());
3631
3632         return TEST_SUCCESS;
3633 }
3634
3635 static void
3636 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3637 {
3638         unsigned int iter = 0;
3639         double total_mops = 0, total_mbps = 0;
3640
3641         for (iter = 0; iter < used_cores; iter++) {
3642                 printf(
3643                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3644                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3645                         t_params[iter].mbps);
3646                 total_mops += t_params[iter].ops_per_sec;
3647                 total_mbps += t_params[iter].mbps;
3648         }
3649         printf(
3650                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3651                 used_cores, total_mops, total_mbps);
3652 }
3653
3654 /* Aggregate the performance results over the number of cores used */
3655 static void
3656 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3657 {
3658         unsigned int core_idx = 0;
3659         double total_mops = 0, total_mbps = 0;
3660         uint8_t iter_count = 0;
3661
3662         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3663                 printf(
3664                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3665                         t_params[core_idx].lcore_id,
3666                         t_params[core_idx].ops_per_sec,
3667                         t_params[core_idx].mbps,
3668                         t_params[core_idx].iter_count);
3669                 total_mops += t_params[core_idx].ops_per_sec;
3670                 total_mbps += t_params[core_idx].mbps;
3671                 iter_count = RTE_MAX(iter_count,
3672                                 t_params[core_idx].iter_count);
3673         }
3674         printf(
3675                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3676                 used_cores, total_mops, total_mbps, iter_count);
3677 }
3678
3679 /* Aggregate the performance results over the number of cores used */
3680 static void
3681 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3682 {
3683         unsigned int core_idx = 0;
3684         double total_mbps = 0, total_bler = 0, total_iter = 0;
3685         double snr = get_snr();
3686
3687         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3688                 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3689                                 t_params[core_idx].lcore_id,
3690                                 t_params[core_idx].bler * 100,
3691                                 t_params[core_idx].iter_average,
3692                                 t_params[core_idx].mbps,
3693                                 get_vector_filename());
3694                 total_mbps += t_params[core_idx].mbps;
3695                 total_bler += t_params[core_idx].bler;
3696                 total_iter += t_params[core_idx].iter_average;
3697         }
3698         total_bler /= used_cores;
3699         total_iter /= used_cores;
3700
3701         printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3702                         snr, total_bler * 100, total_iter, get_iter_max(),
3703                         total_mbps, get_vector_filename());
3704 }
3705
3706 /*
3707  * Test function that determines BLER wireless performance
3708  */
3709 static int
3710 bler_test(struct active_device *ad,
3711                 struct test_op_params *op_params)
3712 {
3713         int ret;
3714         unsigned int lcore_id, used_cores = 0;
3715         struct thread_params *t_params;
3716         struct rte_bbdev_info info;
3717         lcore_function_t *bler_function;
3718         uint16_t num_lcores;
3719         const char *op_type_str;
3720
3721         rte_bbdev_info_get(ad->dev_id, &info);
3722
3723         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3724         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3725                         test_vector.op_type);
3726
3727         printf("+ ------------------------------------------------------- +\n");
3728         printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3729                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3730                         op_params->num_to_process, op_params->num_lcores,
3731                         op_type_str,
3732                         intr_enabled ? "Interrupt mode" : "PMD mode",
3733                         (double)rte_get_tsc_hz() / 1000000000.0);
3734
3735         /* Set number of lcores */
3736         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3737                         ? ad->nb_queues
3738                         : op_params->num_lcores;
3739
3740         /* Allocate memory for thread parameters structure */
3741         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3742                         RTE_CACHE_LINE_SIZE);
3743         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3744                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3745                                 RTE_CACHE_LINE_SIZE));
3746
3747         if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) &&
3748                         !check_bit(test_vector.ldpc_dec.op_flags,
3749                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
3750                         && !check_bit(test_vector.ldpc_dec.op_flags,
3751                         RTE_BBDEV_LDPC_LLR_COMPRESSION))
3752                 bler_function = bler_pmd_lcore_ldpc_dec;
3753         else
3754                 return TEST_SKIPPED;
3755
3756         __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED);
3757
3758         /* Main core is set at first entry */
3759         t_params[0].dev_id = ad->dev_id;
3760         t_params[0].lcore_id = rte_lcore_id();
3761         t_params[0].op_params = op_params;
3762         t_params[0].queue_id = ad->queue_ids[used_cores++];
3763         t_params[0].iter_count = 0;
3764
3765         RTE_LCORE_FOREACH_WORKER(lcore_id) {
3766                 if (used_cores >= num_lcores)
3767                         break;
3768
3769                 t_params[used_cores].dev_id = ad->dev_id;
3770                 t_params[used_cores].lcore_id = lcore_id;
3771                 t_params[used_cores].op_params = op_params;
3772                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3773                 t_params[used_cores].iter_count = 0;
3774
3775                 rte_eal_remote_launch(bler_function,
3776                                 &t_params[used_cores++], lcore_id);
3777         }
3778
3779         __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3780         ret = bler_function(&t_params[0]);
3781
3782         /* Main core is always used */
3783         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3784                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3785
3786         print_dec_bler(t_params, num_lcores);
3787
3788         /* Return if test failed */
3789         if (ret) {
3790                 rte_free(t_params);
3791                 return ret;
3792         }
3793
3794         /* Function to print something  here*/
3795         rte_free(t_params);
3796         return ret;
3797 }
3798
3799 /*
3800  * Test function that determines how long an enqueue + dequeue of a burst
3801  * takes on available lcores.
3802  */
3803 static int
3804 throughput_test(struct active_device *ad,
3805                 struct test_op_params *op_params)
3806 {
3807         int ret;
3808         unsigned int lcore_id, used_cores = 0;
3809         struct thread_params *t_params, *tp;
3810         struct rte_bbdev_info info;
3811         lcore_function_t *throughput_function;
3812         uint16_t num_lcores;
3813         const char *op_type_str;
3814
3815         rte_bbdev_info_get(ad->dev_id, &info);
3816
3817         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3818         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3819                         test_vector.op_type);
3820
3821         printf("+ ------------------------------------------------------- +\n");
3822         printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3823                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3824                         op_params->num_to_process, op_params->num_lcores,
3825                         op_type_str,
3826                         intr_enabled ? "Interrupt mode" : "PMD mode",
3827                         (double)rte_get_tsc_hz() / 1000000000.0);
3828
3829         /* Set number of lcores */
3830         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3831                         ? ad->nb_queues
3832                         : op_params->num_lcores;
3833
3834         /* Allocate memory for thread parameters structure */
3835         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3836                         RTE_CACHE_LINE_SIZE);
3837         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3838                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3839                                 RTE_CACHE_LINE_SIZE));
3840
3841         if (intr_enabled) {
3842                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3843                         throughput_function = throughput_intr_lcore_dec;
3844                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3845                         throughput_function = throughput_intr_lcore_ldpc_dec;
3846                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3847                         throughput_function = throughput_intr_lcore_enc;
3848                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3849                         throughput_function = throughput_intr_lcore_ldpc_enc;
3850                 else
3851                         throughput_function = throughput_intr_lcore_enc;
3852
3853                 /* Dequeue interrupt callback registration */
3854                 ret = rte_bbdev_callback_register(ad->dev_id,
3855                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3856                                 t_params);
3857                 if (ret < 0) {
3858                         rte_free(t_params);
3859                         return ret;
3860                 }
3861         } else {
3862                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3863                         throughput_function = throughput_pmd_lcore_dec;
3864                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3865                         throughput_function = throughput_pmd_lcore_ldpc_dec;
3866                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3867                         throughput_function = throughput_pmd_lcore_enc;
3868                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3869                         throughput_function = throughput_pmd_lcore_ldpc_enc;
3870                 else
3871                         throughput_function = throughput_pmd_lcore_enc;
3872         }
3873
3874         __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED);
3875
3876         /* Main core is set at first entry */
3877         t_params[0].dev_id = ad->dev_id;
3878         t_params[0].lcore_id = rte_lcore_id();
3879         t_params[0].op_params = op_params;
3880         t_params[0].queue_id = ad->queue_ids[used_cores++];
3881         t_params[0].iter_count = 0;
3882
3883         RTE_LCORE_FOREACH_WORKER(lcore_id) {
3884                 if (used_cores >= num_lcores)
3885                         break;
3886
3887                 t_params[used_cores].dev_id = ad->dev_id;
3888                 t_params[used_cores].lcore_id = lcore_id;
3889                 t_params[used_cores].op_params = op_params;
3890                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3891                 t_params[used_cores].iter_count = 0;
3892
3893                 rte_eal_remote_launch(throughput_function,
3894                                 &t_params[used_cores++], lcore_id);
3895         }
3896
3897         __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3898         ret = throughput_function(&t_params[0]);
3899
3900         /* Main core is always used */
3901         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3902                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3903
3904         /* Return if test failed */
3905         if (ret) {
3906                 rte_free(t_params);
3907                 return ret;
3908         }
3909
3910         /* Print throughput if interrupts are disabled and test passed */
3911         if (!intr_enabled) {
3912                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3913                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3914                         print_dec_throughput(t_params, num_lcores);
3915                 else
3916                         print_enc_throughput(t_params, num_lcores);
3917                 rte_free(t_params);
3918                 return ret;
3919         }
3920
3921         /* In interrupt TC we need to wait for the interrupt callback to deqeue
3922          * all pending operations. Skip waiting for queues which reported an
3923          * error using processing_status variable.
3924          * Wait for main lcore operations.
3925          */
3926         tp = &t_params[0];
3927         while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) <
3928                 op_params->num_to_process) &&
3929                 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) !=
3930                 TEST_FAILED))
3931                 rte_pause();
3932
3933         tp->ops_per_sec /= TEST_REPETITIONS;
3934         tp->mbps /= TEST_REPETITIONS;
3935         ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED);
3936
3937         /* Wait for worker lcores operations */
3938         for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3939                 tp = &t_params[used_cores];
3940
3941                 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) <
3942                         op_params->num_to_process) &&
3943                         (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) !=
3944                         TEST_FAILED))
3945                         rte_pause();
3946
3947                 tp->ops_per_sec /= TEST_REPETITIONS;
3948                 tp->mbps /= TEST_REPETITIONS;
3949                 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED);
3950         }
3951
3952         /* Print throughput if test passed */
3953         if (!ret) {
3954                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3955                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3956                         print_dec_throughput(t_params, num_lcores);
3957                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3958                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3959                         print_enc_throughput(t_params, num_lcores);
3960         }
3961
3962         rte_free(t_params);
3963         return ret;
3964 }
3965
3966 static int
3967 latency_test_dec(struct rte_mempool *mempool,
3968                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3969                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3970                 const uint16_t num_to_process, uint16_t burst_sz,
3971                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3972 {
3973         int ret = TEST_SUCCESS;
3974         uint16_t i, j, dequeued;
3975         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3976         uint64_t start_time = 0, last_time = 0;
3977
3978         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3979                 uint16_t enq = 0, deq = 0;
3980                 bool first_time = true;
3981                 last_time = 0;
3982
3983                 if (unlikely(num_to_process - dequeued < burst_sz))
3984                         burst_sz = num_to_process - dequeued;
3985
3986                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3987                 TEST_ASSERT_SUCCESS(ret,
3988                                 "rte_bbdev_dec_op_alloc_bulk() failed");
3989                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3990                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3991                                         bufs->inputs,
3992                                         bufs->hard_outputs,
3993                                         bufs->soft_outputs,
3994                                         ref_op);
3995
3996                 /* Set counter to validate the ordering */
3997                 for (j = 0; j < burst_sz; ++j)
3998                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3999
4000                 start_time = rte_rdtsc_precise();
4001
4002                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
4003                                 burst_sz);
4004                 TEST_ASSERT(enq == burst_sz,
4005                                 "Error enqueueing burst, expected %u, got %u",
4006                                 burst_sz, enq);
4007
4008                 /* Dequeue */
4009                 do {
4010                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4011                                         &ops_deq[deq], burst_sz - deq);
4012                         if (likely(first_time && (deq > 0))) {
4013                                 last_time = rte_rdtsc_precise() - start_time;
4014                                 first_time = false;
4015                         }
4016                 } while (unlikely(burst_sz != deq));
4017
4018                 *max_time = RTE_MAX(*max_time, last_time);
4019                 *min_time = RTE_MIN(*min_time, last_time);
4020                 *total_time += last_time;
4021
4022                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4023                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
4024                                         vector_mask);
4025                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4026                 }
4027
4028                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4029                 dequeued += deq;
4030         }
4031
4032         return i;
4033 }
4034
4035 /* Test case for latency/validation for LDPC Decoder */
4036 static int
4037 latency_test_ldpc_dec(struct rte_mempool *mempool,
4038                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
4039                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
4040                 const uint16_t num_to_process, uint16_t burst_sz,
4041                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time,
4042                 bool disable_et)
4043 {
4044         int ret = TEST_SUCCESS;
4045         uint16_t i, j, dequeued;
4046         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4047         uint64_t start_time = 0, last_time = 0;
4048         bool extDdr = ldpc_cap_flags &
4049                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4050
4051         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4052                 uint16_t enq = 0, deq = 0;
4053                 bool first_time = true;
4054                 last_time = 0;
4055
4056                 if (unlikely(num_to_process - dequeued < burst_sz))
4057                         burst_sz = num_to_process - dequeued;
4058
4059                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4060                 TEST_ASSERT_SUCCESS(ret,
4061                                 "rte_bbdev_dec_op_alloc_bulk() failed");
4062
4063                 /* For latency tests we need to disable early termination */
4064                 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags,
4065                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
4066                         ref_op->ldpc_dec.op_flags -=
4067                                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
4068                 ref_op->ldpc_dec.iter_max = get_iter_max();
4069                 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
4070
4071                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4072                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4073                                         bufs->inputs,
4074                                         bufs->hard_outputs,
4075                                         bufs->soft_outputs,
4076                                         bufs->harq_inputs,
4077                                         bufs->harq_outputs,
4078                                         ref_op);
4079
4080                 if (extDdr)
4081                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4082                                         burst_sz, true);
4083
4084                 /* Set counter to validate the ordering */
4085                 for (j = 0; j < burst_sz; ++j)
4086                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4087
4088                 start_time = rte_rdtsc_precise();
4089
4090                 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4091                                 &ops_enq[enq], burst_sz);
4092                 TEST_ASSERT(enq == burst_sz,
4093                                 "Error enqueueing burst, expected %u, got %u",
4094                                 burst_sz, enq);
4095
4096                 /* Dequeue */
4097                 do {
4098                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4099                                         &ops_deq[deq], burst_sz - deq);
4100                         if (likely(first_time && (deq > 0))) {
4101                                 last_time = rte_rdtsc_precise() - start_time;
4102                                 first_time = false;
4103                         }
4104                 } while (unlikely(burst_sz != deq));
4105
4106                 *max_time = RTE_MAX(*max_time, last_time);
4107                 *min_time = RTE_MIN(*min_time, last_time);
4108                 *total_time += last_time;
4109
4110                 if (extDdr)
4111                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4112
4113                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4114                         ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4115                                         vector_mask);
4116                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4117                 }
4118
4119                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4120                 dequeued += deq;
4121         }
4122         return i;
4123 }
4124
4125 static int
4126 latency_test_enc(struct rte_mempool *mempool,
4127                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4128                 uint16_t dev_id, uint16_t queue_id,
4129                 const uint16_t num_to_process, uint16_t burst_sz,
4130                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4131 {
4132         int ret = TEST_SUCCESS;
4133         uint16_t i, j, dequeued;
4134         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4135         uint64_t start_time = 0, last_time = 0;
4136
4137         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4138                 uint16_t enq = 0, deq = 0;
4139                 bool first_time = true;
4140                 last_time = 0;
4141
4142                 if (unlikely(num_to_process - dequeued < burst_sz))
4143                         burst_sz = num_to_process - dequeued;
4144
4145                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4146                 TEST_ASSERT_SUCCESS(ret,
4147                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4148                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4149                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4150                                         bufs->inputs,
4151                                         bufs->hard_outputs,
4152                                         ref_op);
4153
4154                 /* Set counter to validate the ordering */
4155                 for (j = 0; j < burst_sz; ++j)
4156                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4157
4158                 start_time = rte_rdtsc_precise();
4159
4160                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4161                                 burst_sz);
4162                 TEST_ASSERT(enq == burst_sz,
4163                                 "Error enqueueing burst, expected %u, got %u",
4164                                 burst_sz, enq);
4165
4166                 /* Dequeue */
4167                 do {
4168                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4169                                         &ops_deq[deq], burst_sz - deq);
4170                         if (likely(first_time && (deq > 0))) {
4171                                 last_time += rte_rdtsc_precise() - start_time;
4172                                 first_time = false;
4173                         }
4174                 } while (unlikely(burst_sz != deq));
4175
4176                 *max_time = RTE_MAX(*max_time, last_time);
4177                 *min_time = RTE_MIN(*min_time, last_time);
4178                 *total_time += last_time;
4179
4180                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4181                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4182                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4183                 }
4184
4185                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4186                 dequeued += deq;
4187         }
4188
4189         return i;
4190 }
4191
4192 static int
4193 latency_test_ldpc_enc(struct rte_mempool *mempool,
4194                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4195                 uint16_t dev_id, uint16_t queue_id,
4196                 const uint16_t num_to_process, uint16_t burst_sz,
4197                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4198 {
4199         int ret = TEST_SUCCESS;
4200         uint16_t i, j, dequeued;
4201         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4202         uint64_t start_time = 0, last_time = 0;
4203
4204         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4205                 uint16_t enq = 0, deq = 0;
4206                 bool first_time = true;
4207                 last_time = 0;
4208
4209                 if (unlikely(num_to_process - dequeued < burst_sz))
4210                         burst_sz = num_to_process - dequeued;
4211
4212                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4213                 TEST_ASSERT_SUCCESS(ret,
4214                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4215                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4216                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4217                                         bufs->inputs,
4218                                         bufs->hard_outputs,
4219                                         ref_op);
4220
4221                 /* Set counter to validate the ordering */
4222                 for (j = 0; j < burst_sz; ++j)
4223                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4224
4225                 start_time = rte_rdtsc_precise();
4226
4227                 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4228                                 &ops_enq[enq], burst_sz);
4229                 TEST_ASSERT(enq == burst_sz,
4230                                 "Error enqueueing burst, expected %u, got %u",
4231                                 burst_sz, enq);
4232
4233                 /* Dequeue */
4234                 do {
4235                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4236                                         &ops_deq[deq], burst_sz - deq);
4237                         if (likely(first_time && (deq > 0))) {
4238                                 last_time += rte_rdtsc_precise() - start_time;
4239                                 first_time = false;
4240                         }
4241                 } while (unlikely(burst_sz != deq));
4242
4243                 *max_time = RTE_MAX(*max_time, last_time);
4244                 *min_time = RTE_MIN(*min_time, last_time);
4245                 *total_time += last_time;
4246
4247                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4248                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4249                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4250                 }
4251
4252                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4253                 dequeued += deq;
4254         }
4255
4256         return i;
4257 }
4258
4259 /* Common function for running validation and latency test cases */
4260 static int
4261 validation_latency_test(struct active_device *ad,
4262                 struct test_op_params *op_params, bool latency_flag)
4263 {
4264         int iter;
4265         uint16_t burst_sz = op_params->burst_sz;
4266         const uint16_t num_to_process = op_params->num_to_process;
4267         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4268         const uint16_t queue_id = ad->queue_ids[0];
4269         struct test_buffers *bufs = NULL;
4270         struct rte_bbdev_info info;
4271         uint64_t total_time, min_time, max_time;
4272         const char *op_type_str;
4273
4274         total_time = max_time = 0;
4275         min_time = UINT64_MAX;
4276
4277         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4278                         "BURST_SIZE should be <= %u", MAX_BURST);
4279
4280         rte_bbdev_info_get(ad->dev_id, &info);
4281         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4282
4283         op_type_str = rte_bbdev_op_type_str(op_type);
4284         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4285
4286         printf("+ ------------------------------------------------------- +\n");
4287         if (latency_flag)
4288                 printf("== test: latency\ndev:");
4289         else
4290                 printf("== test: validation\ndev:");
4291         printf("%s, burst size: %u, num ops: %u, op type: %s\n",
4292                         info.dev_name, burst_sz, num_to_process, op_type_str);
4293
4294         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4295                 iter = latency_test_dec(op_params->mp, bufs,
4296                                 op_params->ref_dec_op, op_params->vector_mask,
4297                                 ad->dev_id, queue_id, num_to_process,
4298                                 burst_sz, &total_time, &min_time, &max_time);
4299         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4300                 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4301                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4302                                 num_to_process, burst_sz, &total_time,
4303                                 &min_time, &max_time);
4304         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4305                 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4306                                 op_params->ref_dec_op, op_params->vector_mask,
4307                                 ad->dev_id, queue_id, num_to_process,
4308                                 burst_sz, &total_time, &min_time, &max_time,
4309                                 latency_flag);
4310         else /* RTE_BBDEV_OP_TURBO_ENC */
4311                 iter = latency_test_enc(op_params->mp, bufs,
4312                                 op_params->ref_enc_op,
4313                                 ad->dev_id, queue_id,
4314                                 num_to_process, burst_sz, &total_time,
4315                                 &min_time, &max_time);
4316
4317         if (iter <= 0)
4318                 return TEST_FAILED;
4319
4320         printf("Operation latency:\n"
4321                         "\tavg: %lg cycles, %lg us\n"
4322                         "\tmin: %lg cycles, %lg us\n"
4323                         "\tmax: %lg cycles, %lg us\n",
4324                         (double)total_time / (double)iter,
4325                         (double)(total_time * 1000000) / (double)iter /
4326                         (double)rte_get_tsc_hz(), (double)min_time,
4327                         (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4328                         (double)max_time, (double)(max_time * 1000000) /
4329                         (double)rte_get_tsc_hz());
4330
4331         return TEST_SUCCESS;
4332 }
4333
4334 static int
4335 latency_test(struct active_device *ad, struct test_op_params *op_params)
4336 {
4337         return validation_latency_test(ad, op_params, true);
4338 }
4339
4340 static int
4341 validation_test(struct active_device *ad, struct test_op_params *op_params)
4342 {
4343         return validation_latency_test(ad, op_params, false);
4344 }
4345
4346 #ifdef RTE_BBDEV_OFFLOAD_COST
4347 static int
4348 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4349                 struct rte_bbdev_stats *stats)
4350 {
4351         struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4352         struct rte_bbdev_stats *q_stats;
4353
4354         if (queue_id >= dev->data->num_queues)
4355                 return -1;
4356
4357         q_stats = &dev->data->queues[queue_id].queue_stats;
4358
4359         stats->enqueued_count = q_stats->enqueued_count;
4360         stats->dequeued_count = q_stats->dequeued_count;
4361         stats->enqueue_err_count = q_stats->enqueue_err_count;
4362         stats->dequeue_err_count = q_stats->dequeue_err_count;
4363         stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4364
4365         return 0;
4366 }
4367
4368 static int
4369 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4370                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4371                 uint16_t queue_id, const uint16_t num_to_process,
4372                 uint16_t burst_sz, struct test_time_stats *time_st)
4373 {
4374         int i, dequeued, ret;
4375         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4376         uint64_t enq_start_time, deq_start_time;
4377         uint64_t enq_sw_last_time, deq_last_time;
4378         struct rte_bbdev_stats stats;
4379
4380         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4381                 uint16_t enq = 0, deq = 0;
4382
4383                 if (unlikely(num_to_process - dequeued < burst_sz))
4384                         burst_sz = num_to_process - dequeued;
4385
4386                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4387                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4388                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4389                                         bufs->inputs,
4390                                         bufs->hard_outputs,
4391                                         bufs->soft_outputs,
4392                                         ref_op);
4393
4394                 /* Start time meas for enqueue function offload latency */
4395                 enq_start_time = rte_rdtsc_precise();
4396                 do {
4397                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4398                                         &ops_enq[enq], burst_sz - enq);
4399                 } while (unlikely(burst_sz != enq));
4400
4401                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4402                 TEST_ASSERT_SUCCESS(ret,
4403                                 "Failed to get stats for queue (%u) of device (%u)",
4404                                 queue_id, dev_id);
4405
4406                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4407                                 stats.acc_offload_cycles;
4408                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4409                                 enq_sw_last_time);
4410                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4411                                 enq_sw_last_time);
4412                 time_st->enq_sw_total_time += enq_sw_last_time;
4413
4414                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4415                                 stats.acc_offload_cycles);
4416                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4417                                 stats.acc_offload_cycles);
4418                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4419
4420                 /* give time for device to process ops */
4421                 rte_delay_us(WAIT_OFFLOAD_US);
4422
4423                 /* Start time meas for dequeue function offload latency */
4424                 deq_start_time = rte_rdtsc_precise();
4425                 /* Dequeue one operation */
4426                 do {
4427                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4428                                         &ops_deq[deq], enq);
4429                 } while (unlikely(deq == 0));
4430
4431                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4432                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4433                                 deq_last_time);
4434                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4435                                 deq_last_time);
4436                 time_st->deq_total_time += deq_last_time;
4437
4438                 /* Dequeue remaining operations if needed*/
4439                 while (burst_sz != deq)
4440                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4441                                         &ops_deq[deq], burst_sz - deq);
4442
4443                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4444                 dequeued += deq;
4445         }
4446
4447         return i;
4448 }
4449
4450 static int
4451 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4452                 struct test_buffers *bufs,
4453                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4454                 uint16_t queue_id, const uint16_t num_to_process,
4455                 uint16_t burst_sz, struct test_time_stats *time_st)
4456 {
4457         int i, dequeued, ret;
4458         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4459         uint64_t enq_start_time, deq_start_time;
4460         uint64_t enq_sw_last_time, deq_last_time;
4461         struct rte_bbdev_stats stats;
4462         bool extDdr = ldpc_cap_flags &
4463                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4464
4465         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4466                 uint16_t enq = 0, deq = 0;
4467
4468                 if (unlikely(num_to_process - dequeued < burst_sz))
4469                         burst_sz = num_to_process - dequeued;
4470
4471                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4472                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4473                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4474                                         bufs->inputs,
4475                                         bufs->hard_outputs,
4476                                         bufs->soft_outputs,
4477                                         bufs->harq_inputs,
4478                                         bufs->harq_outputs,
4479                                         ref_op);
4480
4481                 if (extDdr)
4482                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4483                                         burst_sz, true);
4484
4485                 /* Start time meas for enqueue function offload latency */
4486                 enq_start_time = rte_rdtsc_precise();
4487                 do {
4488                         enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4489                                         &ops_enq[enq], burst_sz - enq);
4490                 } while (unlikely(burst_sz != enq));
4491
4492                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4493                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4494                 TEST_ASSERT_SUCCESS(ret,
4495                                 "Failed to get stats for queue (%u) of device (%u)",
4496                                 queue_id, dev_id);
4497
4498                 enq_sw_last_time -= stats.acc_offload_cycles;
4499                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4500                                 enq_sw_last_time);
4501                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4502                                 enq_sw_last_time);
4503                 time_st->enq_sw_total_time += enq_sw_last_time;
4504
4505                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4506                                 stats.acc_offload_cycles);
4507                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4508                                 stats.acc_offload_cycles);
4509                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4510
4511                 /* give time for device to process ops */
4512                 rte_delay_us(WAIT_OFFLOAD_US);
4513
4514                 /* Start time meas for dequeue function offload latency */
4515                 deq_start_time = rte_rdtsc_precise();
4516                 /* Dequeue one operation */
4517                 do {
4518                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4519                                         &ops_deq[deq], enq);
4520                 } while (unlikely(deq == 0));
4521
4522                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4523                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4524                                 deq_last_time);
4525                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4526                                 deq_last_time);
4527                 time_st->deq_total_time += deq_last_time;
4528
4529                 /* Dequeue remaining operations if needed*/
4530                 while (burst_sz != deq)
4531                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4532                                         &ops_deq[deq], burst_sz - deq);
4533
4534                 if (extDdr) {
4535                         /* Read loopback is not thread safe */
4536                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4537                 }
4538
4539                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4540                 dequeued += deq;
4541         }
4542
4543         return i;
4544 }
4545
4546 static int
4547 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4548                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4549                 uint16_t queue_id, const uint16_t num_to_process,
4550                 uint16_t burst_sz, struct test_time_stats *time_st)
4551 {
4552         int i, dequeued, ret;
4553         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4554         uint64_t enq_start_time, deq_start_time;
4555         uint64_t enq_sw_last_time, deq_last_time;
4556         struct rte_bbdev_stats stats;
4557
4558         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4559                 uint16_t enq = 0, deq = 0;
4560
4561                 if (unlikely(num_to_process - dequeued < burst_sz))
4562                         burst_sz = num_to_process - dequeued;
4563
4564                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4565                 TEST_ASSERT_SUCCESS(ret,
4566                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4567                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4568                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4569                                         bufs->inputs,
4570                                         bufs->hard_outputs,
4571                                         ref_op);
4572
4573                 /* Start time meas for enqueue function offload latency */
4574                 enq_start_time = rte_rdtsc_precise();
4575                 do {
4576                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4577                                         &ops_enq[enq], burst_sz - enq);
4578                 } while (unlikely(burst_sz != enq));
4579
4580                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4581
4582                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4583                 TEST_ASSERT_SUCCESS(ret,
4584                                 "Failed to get stats for queue (%u) of device (%u)",
4585                                 queue_id, dev_id);
4586                 enq_sw_last_time -= stats.acc_offload_cycles;
4587                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4588                                 enq_sw_last_time);
4589                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4590                                 enq_sw_last_time);
4591                 time_st->enq_sw_total_time += enq_sw_last_time;
4592
4593                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4594                                 stats.acc_offload_cycles);
4595                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4596                                 stats.acc_offload_cycles);
4597                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4598
4599                 /* give time for device to process ops */
4600                 rte_delay_us(WAIT_OFFLOAD_US);
4601
4602                 /* Start time meas for dequeue function offload latency */
4603                 deq_start_time = rte_rdtsc_precise();
4604                 /* Dequeue one operation */
4605                 do {
4606                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4607                                         &ops_deq[deq], enq);
4608                 } while (unlikely(deq == 0));
4609
4610                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4611                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4612                                 deq_last_time);
4613                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4614                                 deq_last_time);
4615                 time_st->deq_total_time += deq_last_time;
4616
4617                 while (burst_sz != deq)
4618                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4619                                         &ops_deq[deq], burst_sz - deq);
4620
4621                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4622                 dequeued += deq;
4623         }
4624
4625         return i;
4626 }
4627
4628 static int
4629 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4630                 struct test_buffers *bufs,
4631                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4632                 uint16_t queue_id, const uint16_t num_to_process,
4633                 uint16_t burst_sz, struct test_time_stats *time_st)
4634 {
4635         int i, dequeued, ret;
4636         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4637         uint64_t enq_start_time, deq_start_time;
4638         uint64_t enq_sw_last_time, deq_last_time;
4639         struct rte_bbdev_stats stats;
4640
4641         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4642                 uint16_t enq = 0, deq = 0;
4643
4644                 if (unlikely(num_to_process - dequeued < burst_sz))
4645                         burst_sz = num_to_process - dequeued;
4646
4647                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4648                 TEST_ASSERT_SUCCESS(ret,
4649                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4650                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4651                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4652                                         bufs->inputs,
4653                                         bufs->hard_outputs,
4654                                         ref_op);
4655
4656                 /* Start time meas for enqueue function offload latency */
4657                 enq_start_time = rte_rdtsc_precise();
4658                 do {
4659                         enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4660                                         &ops_enq[enq], burst_sz - enq);
4661                 } while (unlikely(burst_sz != enq));
4662
4663                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4664                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4665                 TEST_ASSERT_SUCCESS(ret,
4666                                 "Failed to get stats for queue (%u) of device (%u)",
4667                                 queue_id, dev_id);
4668
4669                 enq_sw_last_time -= stats.acc_offload_cycles;
4670                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4671                                 enq_sw_last_time);
4672                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4673                                 enq_sw_last_time);
4674                 time_st->enq_sw_total_time += enq_sw_last_time;
4675
4676                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4677                                 stats.acc_offload_cycles);
4678                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4679                                 stats.acc_offload_cycles);
4680                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4681
4682                 /* give time for device to process ops */
4683                 rte_delay_us(WAIT_OFFLOAD_US);
4684
4685                 /* Start time meas for dequeue function offload latency */
4686                 deq_start_time = rte_rdtsc_precise();
4687                 /* Dequeue one operation */
4688                 do {
4689                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4690                                         &ops_deq[deq], enq);
4691                 } while (unlikely(deq == 0));
4692
4693                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4694                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4695                                 deq_last_time);
4696                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4697                                 deq_last_time);
4698                 time_st->deq_total_time += deq_last_time;
4699
4700                 while (burst_sz != deq)
4701                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4702                                         &ops_deq[deq], burst_sz - deq);
4703
4704                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4705                 dequeued += deq;
4706         }
4707
4708         return i;
4709 }
4710 #endif
4711
4712 static int
4713 offload_cost_test(struct active_device *ad,
4714                 struct test_op_params *op_params)
4715 {
4716 #ifndef RTE_BBDEV_OFFLOAD_COST
4717         RTE_SET_USED(ad);
4718         RTE_SET_USED(op_params);
4719         printf("Offload latency test is disabled.\n");
4720         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4721         return TEST_SKIPPED;
4722 #else
4723         int iter;
4724         uint16_t burst_sz = op_params->burst_sz;
4725         const uint16_t num_to_process = op_params->num_to_process;
4726         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4727         const uint16_t queue_id = ad->queue_ids[0];
4728         struct test_buffers *bufs = NULL;
4729         struct rte_bbdev_info info;
4730         const char *op_type_str;
4731         struct test_time_stats time_st;
4732
4733         memset(&time_st, 0, sizeof(struct test_time_stats));
4734         time_st.enq_sw_min_time = UINT64_MAX;
4735         time_st.enq_acc_min_time = UINT64_MAX;
4736         time_st.deq_min_time = UINT64_MAX;
4737
4738         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4739                         "BURST_SIZE should be <= %u", MAX_BURST);
4740
4741         rte_bbdev_info_get(ad->dev_id, &info);
4742         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4743
4744         op_type_str = rte_bbdev_op_type_str(op_type);
4745         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4746
4747         printf("+ ------------------------------------------------------- +\n");
4748         printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4749                         info.dev_name, burst_sz, num_to_process, op_type_str);
4750
4751         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4752                 iter = offload_latency_test_dec(op_params->mp, bufs,
4753                                 op_params->ref_dec_op, ad->dev_id, queue_id,
4754                                 num_to_process, burst_sz, &time_st);
4755         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4756                 iter = offload_latency_test_enc(op_params->mp, bufs,
4757                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4758                                 num_to_process, burst_sz, &time_st);
4759         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4760                 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4761                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4762                                 num_to_process, burst_sz, &time_st);
4763         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4764                 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4765                         op_params->ref_dec_op, ad->dev_id, queue_id,
4766                         num_to_process, burst_sz, &time_st);
4767         else
4768                 iter = offload_latency_test_enc(op_params->mp, bufs,
4769                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4770                                 num_to_process, burst_sz, &time_st);
4771
4772         if (iter <= 0)
4773                 return TEST_FAILED;
4774
4775         printf("Enqueue driver offload cost latency:\n"
4776                         "\tavg: %lg cycles, %lg us\n"
4777                         "\tmin: %lg cycles, %lg us\n"
4778                         "\tmax: %lg cycles, %lg us\n"
4779                         "Enqueue accelerator offload cost latency:\n"
4780                         "\tavg: %lg cycles, %lg us\n"
4781                         "\tmin: %lg cycles, %lg us\n"
4782                         "\tmax: %lg cycles, %lg us\n",
4783                         (double)time_st.enq_sw_total_time / (double)iter,
4784                         (double)(time_st.enq_sw_total_time * 1000000) /
4785                         (double)iter / (double)rte_get_tsc_hz(),
4786                         (double)time_st.enq_sw_min_time,
4787                         (double)(time_st.enq_sw_min_time * 1000000) /
4788                         rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4789                         (double)(time_st.enq_sw_max_time * 1000000) /
4790                         rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4791                         (double)iter,
4792                         (double)(time_st.enq_acc_total_time * 1000000) /
4793                         (double)iter / (double)rte_get_tsc_hz(),
4794                         (double)time_st.enq_acc_min_time,
4795                         (double)(time_st.enq_acc_min_time * 1000000) /
4796                         rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4797                         (double)(time_st.enq_acc_max_time * 1000000) /
4798                         rte_get_tsc_hz());
4799
4800         printf("Dequeue offload cost latency - one op:\n"
4801                         "\tavg: %lg cycles, %lg us\n"
4802                         "\tmin: %lg cycles, %lg us\n"
4803                         "\tmax: %lg cycles, %lg us\n",
4804                         (double)time_st.deq_total_time / (double)iter,
4805                         (double)(time_st.deq_total_time * 1000000) /
4806                         (double)iter / (double)rte_get_tsc_hz(),
4807                         (double)time_st.deq_min_time,
4808                         (double)(time_st.deq_min_time * 1000000) /
4809                         rte_get_tsc_hz(), (double)time_st.deq_max_time,
4810                         (double)(time_st.deq_max_time * 1000000) /
4811                         rte_get_tsc_hz());
4812
4813         struct rte_bbdev_stats stats = {0};
4814         get_bbdev_queue_stats(ad->dev_id, queue_id, &stats);
4815         if (op_type != RTE_BBDEV_OP_LDPC_DEC) {
4816                 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process,
4817                                 "Mismatch in enqueue count %10"PRIu64" %d",
4818                                 stats.enqueued_count, num_to_process);
4819                 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process,
4820                                 "Mismatch in dequeue count %10"PRIu64" %d",
4821                                 stats.dequeued_count, num_to_process);
4822         }
4823         TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0,
4824                         "Enqueue count Error %10"PRIu64"",
4825                         stats.enqueue_err_count);
4826         TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0,
4827                         "Dequeue count Error (%10"PRIu64"",
4828                         stats.dequeue_err_count);
4829
4830         return TEST_SUCCESS;
4831 #endif
4832 }
4833
4834 #ifdef RTE_BBDEV_OFFLOAD_COST
4835 static int
4836 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4837                 const uint16_t num_to_process, uint16_t burst_sz,
4838                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4839                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4840 {
4841         int i, deq_total;
4842         struct rte_bbdev_dec_op *ops[MAX_BURST];
4843         uint64_t deq_start_time, deq_last_time;
4844
4845         /* Test deq offload latency from an empty queue */
4846
4847         for (i = 0, deq_total = 0; deq_total < num_to_process;
4848                         ++i, deq_total += burst_sz) {
4849                 deq_start_time = rte_rdtsc_precise();
4850
4851                 if (unlikely(num_to_process - deq_total < burst_sz))
4852                         burst_sz = num_to_process - deq_total;
4853                 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4854                         rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4855                                         burst_sz);
4856                 else
4857                         rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4858                                         burst_sz);
4859
4860                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4861                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4862                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4863                 *deq_total_time += deq_last_time;
4864         }
4865
4866         return i;
4867 }
4868
4869 static int
4870 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4871                 const uint16_t num_to_process, uint16_t burst_sz,
4872                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4873                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4874 {
4875         int i, deq_total;
4876         struct rte_bbdev_enc_op *ops[MAX_BURST];
4877         uint64_t deq_start_time, deq_last_time;
4878
4879         /* Test deq offload latency from an empty queue */
4880         for (i = 0, deq_total = 0; deq_total < num_to_process;
4881                         ++i, deq_total += burst_sz) {
4882                 deq_start_time = rte_rdtsc_precise();
4883
4884                 if (unlikely(num_to_process - deq_total < burst_sz))
4885                         burst_sz = num_to_process - deq_total;
4886                 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4887                         rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4888                                         burst_sz);
4889                 else
4890                         rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4891                                         burst_sz);
4892
4893                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4894                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4895                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4896                 *deq_total_time += deq_last_time;
4897         }
4898
4899         return i;
4900 }
4901
4902 #endif
4903
4904 static int
4905 offload_latency_empty_q_test(struct active_device *ad,
4906                 struct test_op_params *op_params)
4907 {
4908 #ifndef RTE_BBDEV_OFFLOAD_COST
4909         RTE_SET_USED(ad);
4910         RTE_SET_USED(op_params);
4911         printf("Offload latency empty dequeue test is disabled.\n");
4912         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4913         return TEST_SKIPPED;
4914 #else
4915         int iter;
4916         uint64_t deq_total_time, deq_min_time, deq_max_time;
4917         uint16_t burst_sz = op_params->burst_sz;
4918         const uint16_t num_to_process = op_params->num_to_process;
4919         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4920         const uint16_t queue_id = ad->queue_ids[0];
4921         struct rte_bbdev_info info;
4922         const char *op_type_str;
4923
4924         deq_total_time = deq_max_time = 0;
4925         deq_min_time = UINT64_MAX;
4926
4927         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4928                         "BURST_SIZE should be <= %u", MAX_BURST);
4929
4930         rte_bbdev_info_get(ad->dev_id, &info);
4931
4932         op_type_str = rte_bbdev_op_type_str(op_type);
4933         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4934
4935         printf("+ ------------------------------------------------------- +\n");
4936         printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4937                         info.dev_name, burst_sz, num_to_process, op_type_str);
4938
4939         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4940                         op_type == RTE_BBDEV_OP_LDPC_DEC)
4941                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4942                                 num_to_process, burst_sz, &deq_total_time,
4943                                 &deq_min_time, &deq_max_time, op_type);
4944         else
4945                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4946                                 num_to_process, burst_sz, &deq_total_time,
4947                                 &deq_min_time, &deq_max_time, op_type);
4948
4949         if (iter <= 0)
4950                 return TEST_FAILED;
4951
4952         printf("Empty dequeue offload:\n"
4953                         "\tavg: %lg cycles, %lg us\n"
4954                         "\tmin: %lg cycles, %lg us\n"
4955                         "\tmax: %lg cycles, %lg us\n",
4956                         (double)deq_total_time / (double)iter,
4957                         (double)(deq_total_time * 1000000) / (double)iter /
4958                         (double)rte_get_tsc_hz(), (double)deq_min_time,
4959                         (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4960                         (double)deq_max_time, (double)(deq_max_time * 1000000) /
4961                         rte_get_tsc_hz());
4962
4963         return TEST_SUCCESS;
4964 #endif
4965 }
4966
4967 static int
4968 bler_tc(void)
4969 {
4970         return run_test_case(bler_test);
4971 }
4972
4973 static int
4974 throughput_tc(void)
4975 {
4976         return run_test_case(throughput_test);
4977 }
4978
4979 static int
4980 offload_cost_tc(void)
4981 {
4982         return run_test_case(offload_cost_test);
4983 }
4984
4985 static int
4986 offload_latency_empty_q_tc(void)
4987 {
4988         return run_test_case(offload_latency_empty_q_test);
4989 }
4990
4991 static int
4992 latency_tc(void)
4993 {
4994         return run_test_case(latency_test);
4995 }
4996
4997 static int
4998 validation_tc(void)
4999 {
5000         return run_test_case(validation_test);
5001 }
5002
5003 static int
5004 interrupt_tc(void)
5005 {
5006         return run_test_case(throughput_test);
5007 }
5008
5009 static struct unit_test_suite bbdev_bler_testsuite = {
5010         .suite_name = "BBdev BLER Tests",
5011         .setup = testsuite_setup,
5012         .teardown = testsuite_teardown,
5013         .unit_test_cases = {
5014                 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
5015                 TEST_CASES_END() /**< NULL terminate unit test array */
5016         }
5017 };
5018
5019 static struct unit_test_suite bbdev_throughput_testsuite = {
5020         .suite_name = "BBdev Throughput Tests",
5021         .setup = testsuite_setup,
5022         .teardown = testsuite_teardown,
5023         .unit_test_cases = {
5024                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
5025                 TEST_CASES_END() /**< NULL terminate unit test array */
5026         }
5027 };
5028
5029 static struct unit_test_suite bbdev_validation_testsuite = {
5030         .suite_name = "BBdev Validation Tests",
5031         .setup = testsuite_setup,
5032         .teardown = testsuite_teardown,
5033         .unit_test_cases = {
5034                 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc),
5035                 TEST_CASES_END() /**< NULL terminate unit test array */
5036         }
5037 };
5038
5039 static struct unit_test_suite bbdev_latency_testsuite = {
5040         .suite_name = "BBdev Latency Tests",
5041         .setup = testsuite_setup,
5042         .teardown = testsuite_teardown,
5043         .unit_test_cases = {
5044                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
5045                 TEST_CASES_END() /**< NULL terminate unit test array */
5046         }
5047 };
5048
5049 static struct unit_test_suite bbdev_offload_cost_testsuite = {
5050         .suite_name = "BBdev Offload Cost Tests",
5051         .setup = testsuite_setup,
5052         .teardown = testsuite_teardown,
5053         .unit_test_cases = {
5054                 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
5055                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
5056                 TEST_CASES_END() /**< NULL terminate unit test array */
5057         }
5058 };
5059
5060 static struct unit_test_suite bbdev_interrupt_testsuite = {
5061         .suite_name = "BBdev Interrupt Tests",
5062         .setup = interrupt_testsuite_setup,
5063         .teardown = testsuite_teardown,
5064         .unit_test_cases = {
5065                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
5066                 TEST_CASES_END() /**< NULL terminate unit test array */
5067         }
5068 };
5069
5070 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
5071 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
5072 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
5073 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
5074 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
5075 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);