app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <math.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_common.h>
  11 #include <rte_dev.h>
  12 #include <rte_launch.h>
  13 #include <rte_bbdev.h>
  14 #include <rte_cycles.h>
  15 #include <rte_lcore.h>
  16 #include <rte_malloc.h>
  17 #include <rte_random.h>
  18 #include <rte_hexdump.h>
  19 #include <rte_interrupts.h>
  20
  21 #include "main.h"
  22 #include "test_bbdev_vector.h"
  23
  24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  25
  26 #define MAX_QUEUES RTE_MAX_LCORE
  27 #define TEST_REPETITIONS 1000
  28
  29 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
  30 #include <fpga_lte_fec.h>
  31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
  32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
  33 #define VF_UL_4G_QUEUE_VALUE 4
  34 #define VF_DL_4G_QUEUE_VALUE 4
  35 #define UL_4G_BANDWIDTH 3
  36 #define DL_4G_BANDWIDTH 3
  37 #define UL_4G_LOAD_BALANCE 128
  38 #define DL_4G_LOAD_BALANCE 128
  39 #define FLR_4G_TIMEOUT 610
  40 #endif
  41
  42 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC
  43 #include <rte_pmd_fpga_5gnr_fec.h>
  44 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
  45 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
  46 #define VF_UL_5G_QUEUE_VALUE 4
  47 #define VF_DL_5G_QUEUE_VALUE 4
  48 #define UL_5G_BANDWIDTH 3
  49 #define DL_5G_BANDWIDTH 3
  50 #define UL_5G_LOAD_BALANCE 128
  51 #define DL_5G_LOAD_BALANCE 128
  52 #define FLR_5G_TIMEOUT 610
  53 #endif
  54
  55 #define OPS_CACHE_SIZE 256U
  56 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  57
  58 #define SYNC_WAIT 0
  59 #define SYNC_START 1
  60 #define INVALID_OPAQUE -1
  61
  62 #define INVALID_QUEUE_ID -1
  63 /* Increment for next code block in external HARQ memory */
  64 #define HARQ_INCR 32768
  65 /* Headroom for filler LLRs insertion in HARQ buffer */
  66 #define FILLER_HEADROOM 1024
  67 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
  68 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
  69 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
  70 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
  71 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
  72 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
  73 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
  74 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
  75 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
  76
  77 static struct test_bbdev_vector test_vector;
  78
  79 /* Switch between PMD and Interrupt for throughput TC */
  80 static bool intr_enabled;
  81
  82 /* LLR arithmetic representation for numerical conversion */
  83 static int ldpc_llr_decimals;
  84 static int ldpc_llr_size;
  85 /* Keep track of the LDPC decoder device capability flag */
  86 static uint32_t ldpc_cap_flags;
  87
  88 /* Represents tested active devices */
  89 static struct active_device {
  90         const char *driver_name;
  91         uint8_t dev_id;
  92         uint16_t supported_ops;
  93         uint16_t queue_ids[MAX_QUEUES];
  94         uint16_t nb_queues;
  95         struct rte_mempool *ops_mempool;
  96         struct rte_mempool *in_mbuf_pool;
  97         struct rte_mempool *hard_out_mbuf_pool;
  98         struct rte_mempool *soft_out_mbuf_pool;
  99         struct rte_mempool *harq_in_mbuf_pool;
 100         struct rte_mempool *harq_out_mbuf_pool;
 101 } active_devs[RTE_BBDEV_MAX_DEVS];
 102
 103 static uint8_t nb_active_devs;
 104
 105 /* Data buffers used by BBDEV ops */
 106 struct test_buffers {
 107         struct rte_bbdev_op_data *inputs;
 108         struct rte_bbdev_op_data *hard_outputs;
 109         struct rte_bbdev_op_data *soft_outputs;
 110         struct rte_bbdev_op_data *harq_inputs;
 111         struct rte_bbdev_op_data *harq_outputs;
 112 };
 113
 114 /* Operation parameters specific for given test case */
 115 struct test_op_params {
 116         struct rte_mempool *mp;
 117         struct rte_bbdev_dec_op *ref_dec_op;
 118         struct rte_bbdev_enc_op *ref_enc_op;
 119         uint16_t burst_sz;
 120         uint16_t num_to_process;
 121         uint16_t num_lcores;
 122         int vector_mask;
 123         rte_atomic16_t sync;
 124         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
 125 };
 126
 127 /* Contains per lcore params */
 128 struct thread_params {
 129         uint8_t dev_id;
 130         uint16_t queue_id;
 131         uint32_t lcore_id;
 132         uint64_t start_time;
 133         double ops_per_sec;
 134         double mbps;
 135         uint8_t iter_count;
 136         double iter_average;
 137         double bler;
 138         rte_atomic16_t nb_dequeued;
 139         rte_atomic16_t processing_status;
 140         rte_atomic16_t burst_sz;
 141         struct test_op_params *op_params;
 142         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
 143         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
 144 };
 145
 146 #ifdef RTE_BBDEV_OFFLOAD_COST
 147 /* Stores time statistics */
 148 struct test_time_stats {
 149         /* Stores software enqueue total working time */
 150         uint64_t enq_sw_total_time;
 151         /* Stores minimum value of software enqueue working time */
 152         uint64_t enq_sw_min_time;
 153         /* Stores maximum value of software enqueue working time */
 154         uint64_t enq_sw_max_time;
 155         /* Stores turbo enqueue total working time */
 156         uint64_t enq_acc_total_time;
 157         /* Stores minimum value of accelerator enqueue working time */
 158         uint64_t enq_acc_min_time;
 159         /* Stores maximum value of accelerator enqueue working time */
 160         uint64_t enq_acc_max_time;
 161         /* Stores dequeue total working time */
 162         uint64_t deq_total_time;
 163         /* Stores minimum value of dequeue working time */
 164         uint64_t deq_min_time;
 165         /* Stores maximum value of dequeue working time */
 166         uint64_t deq_max_time;
 167 };
 168 #endif
 169
 170 typedef int (test_case_function)(struct active_device *ad,
 171                 struct test_op_params *op_params);
 172
 173 static inline void
 174 mbuf_reset(struct rte_mbuf *m)
 175 {
 176         m->pkt_len = 0;
 177
 178         do {
 179                 m->data_len = 0;
 180                 m = m->next;
 181         } while (m != NULL);
 182 }
 183
 184 /* Read flag value 0/1 from bitmap */
 185 static inline bool
 186 check_bit(uint32_t bitmap, uint32_t bitmask)
 187 {
 188         return bitmap & bitmask;
 189 }
 190
 191 static inline void
 192 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 193 {
 194         ad->supported_ops |= (1 << op_type);
 195 }
 196
 197 static inline bool
 198 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
 199 {
 200         return ad->supported_ops & (1 << op_type);
 201 }
 202
 203 static inline bool
 204 flags_match(uint32_t flags_req, uint32_t flags_present)
 205 {
 206         return (flags_req & flags_present) == flags_req;
 207 }
 208
 209 static void
 210 clear_soft_out_cap(uint32_t *op_flags)
 211 {
 212         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 213         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 214         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 215 }
 216
 217 static int
 218 check_dev_cap(const struct rte_bbdev_info *dev_info)
 219 {
 220         unsigned int i;
 221         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
 222                 nb_harq_inputs, nb_harq_outputs;
 223         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 224
 225         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 226         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 227         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 228         nb_harq_inputs  = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
 229         nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
 230
 231         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 232                 if (op_cap->type != test_vector.op_type)
 233                         continue;
 234
 235                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 236                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 237                                         &op_cap->cap.turbo_dec;
 238                         /* Ignore lack of soft output capability, just skip
 239                          * checking if soft output is valid.
 240                          */
 241                         if ((test_vector.turbo_dec.op_flags &
 242                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 243                                         !(cap->capability_flags &
 244                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 245                                 printf(
 246                                         "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 247                                         dev_info->dev_name);
 248                                 clear_soft_out_cap(
 249                                         &test_vector.turbo_dec.op_flags);
 250                         }
 251
 252                         if (!flags_match(test_vector.turbo_dec.op_flags,
 253                                         cap->capability_flags))
 254                                 return TEST_FAILED;
 255                         if (nb_inputs > cap->num_buffers_src) {
 256                                 printf("Too many inputs defined: %u, max: %u\n",
 257                                         nb_inputs, cap->num_buffers_src);
 258                                 return TEST_FAILED;
 259                         }
 260                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 261                                         (test_vector.turbo_dec.op_flags &
 262                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 263                                 printf(
 264                                         "Too many soft outputs defined: %u, max: %u\n",
 265                                                 nb_soft_outputs,
 266                                                 cap->num_buffers_soft_out);
 267                                 return TEST_FAILED;
 268                         }
 269                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 270                                 printf(
 271                                         "Too many hard outputs defined: %u, max: %u\n",
 272                                                 nb_hard_outputs,
 273                                                 cap->num_buffers_hard_out);
 274                                 return TEST_FAILED;
 275                         }
 276                         if (intr_enabled && !(cap->capability_flags &
 277                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 278                                 printf(
 279                                         "Dequeue interrupts are not supported!\n");
 280                                 return TEST_FAILED;
 281                         }
 282
 283                         return TEST_SUCCESS;
 284                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 285                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 286                                         &op_cap->cap.turbo_enc;
 287
 288                         if (!flags_match(test_vector.turbo_enc.op_flags,
 289                                         cap->capability_flags))
 290                                 return TEST_FAILED;
 291                         if (nb_inputs > cap->num_buffers_src) {
 292                                 printf("Too many inputs defined: %u, max: %u\n",
 293                                         nb_inputs, cap->num_buffers_src);
 294                                 return TEST_FAILED;
 295                         }
 296                         if (nb_hard_outputs > cap->num_buffers_dst) {
 297                                 printf(
 298                                         "Too many hard outputs defined: %u, max: %u\n",
 299                                         nb_hard_outputs, cap->num_buffers_dst);
 300                                 return TEST_FAILED;
 301                         }
 302                         if (intr_enabled && !(cap->capability_flags &
 303                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 304                                 printf(
 305                                         "Dequeue interrupts are not supported!\n");
 306                                 return TEST_FAILED;
 307                         }
 308
 309                         return TEST_SUCCESS;
 310                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
 311                         const struct rte_bbdev_op_cap_ldpc_enc *cap =
 312                                         &op_cap->cap.ldpc_enc;
 313
 314                         if (!flags_match(test_vector.ldpc_enc.op_flags,
 315                                         cap->capability_flags)){
 316                                 printf("Flag Mismatch\n");
 317                                 return TEST_FAILED;
 318                         }
 319                         if (nb_inputs > cap->num_buffers_src) {
 320                                 printf("Too many inputs defined: %u, max: %u\n",
 321                                         nb_inputs, cap->num_buffers_src);
 322                                 return TEST_FAILED;
 323                         }
 324                         if (nb_hard_outputs > cap->num_buffers_dst) {
 325                                 printf(
 326                                         "Too many hard outputs defined: %u, max: %u\n",
 327                                         nb_hard_outputs, cap->num_buffers_dst);
 328                                 return TEST_FAILED;
 329                         }
 330                         if (intr_enabled && !(cap->capability_flags &
 331                                         RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
 332                                 printf(
 333                                         "Dequeue interrupts are not supported!\n");
 334                                 return TEST_FAILED;
 335                         }
 336
 337                         return TEST_SUCCESS;
 338                 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
 339                         const struct rte_bbdev_op_cap_ldpc_dec *cap =
 340                                         &op_cap->cap.ldpc_dec;
 341
 342                         if (!flags_match(test_vector.ldpc_dec.op_flags,
 343                                         cap->capability_flags)){
 344                                 printf("Flag Mismatch\n");
 345                                 return TEST_FAILED;
 346                         }
 347                         if (nb_inputs > cap->num_buffers_src) {
 348                                 printf("Too many inputs defined: %u, max: %u\n",
 349                                         nb_inputs, cap->num_buffers_src);
 350                                 return TEST_FAILED;
 351                         }
 352                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 353                                 printf(
 354                                         "Too many hard outputs defined: %u, max: %u\n",
 355                                         nb_hard_outputs,
 356                                         cap->num_buffers_hard_out);
 357                                 return TEST_FAILED;
 358                         }
 359                         if (nb_harq_inputs > cap->num_buffers_hard_out) {
 360                                 printf(
 361                                         "Too many HARQ inputs defined: %u, max: %u\n",
 362                                         nb_hard_outputs,
 363                                         cap->num_buffers_hard_out);
 364                                 return TEST_FAILED;
 365                         }
 366                         if (nb_harq_outputs > cap->num_buffers_hard_out) {
 367                                 printf(
 368                                         "Too many HARQ outputs defined: %u, max: %u\n",
 369                                         nb_hard_outputs,
 370                                         cap->num_buffers_hard_out);
 371                                 return TEST_FAILED;
 372                         }
 373                         if (intr_enabled && !(cap->capability_flags &
 374                                         RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
 375                                 printf(
 376                                         "Dequeue interrupts are not supported!\n");
 377                                 return TEST_FAILED;
 378                         }
 379                         if (intr_enabled && (test_vector.ldpc_dec.op_flags &
 380                                 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
 381                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
 382                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
 383                                         ))) {
 384                                 printf("Skip loop-back with interrupt\n");
 385                                 return TEST_FAILED;
 386                         }
 387                         return TEST_SUCCESS;
 388                 }
 389         }
 390
 391         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 392                 return TEST_SUCCESS; /* Special case for NULL device */
 393
 394         return TEST_FAILED;
 395 }
 396
 397 /* calculates optimal mempool size not smaller than the val */
 398 static unsigned int
 399 optimal_mempool_size(unsigned int val)
 400 {
 401         return rte_align32pow2(val + 1) - 1;
 402 }
 403
 404 /* allocates mbuf mempool for inputs and outputs */
 405 static struct rte_mempool *
 406 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 407                 int socket_id, unsigned int mbuf_pool_size,
 408                 const char *op_type_str)
 409 {
 410         unsigned int i;
 411         uint32_t max_seg_sz = 0;
 412         char pool_name[RTE_MEMPOOL_NAMESIZE];
 413
 414         /* find max input segment size */
 415         for (i = 0; i < entries->nb_segments; ++i)
 416                 if (entries->segments[i].length > max_seg_sz)
 417                         max_seg_sz = entries->segments[i].length;
 418
 419         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 420                         dev_id);
 421         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 422                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
 423                                         + FILLER_HEADROOM,
 424                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 425 }
 426
 427 static int
 428 create_mempools(struct active_device *ad, int socket_id,
 429                 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
 430 {
 431         struct rte_mempool *mp;
 432         unsigned int ops_pool_size, mbuf_pool_size = 0;
 433         char pool_name[RTE_MEMPOOL_NAMESIZE];
 434         const char *op_type_str;
 435         enum rte_bbdev_op_type op_type = org_op_type;
 436
 437         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 438         struct op_data_entries *hard_out =
 439                         &test_vector.entries[DATA_HARD_OUTPUT];
 440         struct op_data_entries *soft_out =
 441                         &test_vector.entries[DATA_SOFT_OUTPUT];
 442         struct op_data_entries *harq_in =
 443                         &test_vector.entries[DATA_HARQ_INPUT];
 444         struct op_data_entries *harq_out =
 445                         &test_vector.entries[DATA_HARQ_OUTPUT];
 446
 447         /* allocate ops mempool */
 448         ops_pool_size = optimal_mempool_size(RTE_MAX(
 449                         /* Ops used plus 1 reference op */
 450                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 451                         /* Minimal cache size plus 1 reference op */
 452                         (unsigned int)(1.5 * rte_lcore_count() *
 453                                         OPS_CACHE_SIZE + 1)),
 454                         OPS_POOL_SIZE_MIN));
 455
 456         if (org_op_type == RTE_BBDEV_OP_NONE)
 457                 op_type = RTE_BBDEV_OP_TURBO_ENC;
 458
 459         op_type_str = rte_bbdev_op_type_str(op_type);
 460         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 461
 462         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 463                         ad->dev_id);
 464         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 465                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 466         TEST_ASSERT_NOT_NULL(mp,
 467                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 468                         ops_pool_size,
 469                         ad->dev_id,
 470                         socket_id);
 471         ad->ops_mempool = mp;
 472
 473         /* Do not create inputs and outputs mbufs for BaseBand Null Device */
 474         if (org_op_type == RTE_BBDEV_OP_NONE)
 475                 return TEST_SUCCESS;
 476
 477         /* Inputs */
 478         if (in->nb_segments > 0) {
 479                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 480                                 in->nb_segments);
 481                 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
 482                                 mbuf_pool_size, "in");
 483                 TEST_ASSERT_NOT_NULL(mp,
 484                                 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 485                                 mbuf_pool_size,
 486                                 ad->dev_id,
 487                                 socket_id);
 488                 ad->in_mbuf_pool = mp;
 489         }
 490
 491         /* Hard outputs */
 492         if (hard_out->nb_segments > 0) {
 493                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 494                                 hard_out->nb_segments);
 495                 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
 496                                 mbuf_pool_size,
 497                                 "hard_out");
 498                 TEST_ASSERT_NOT_NULL(mp,
 499                                 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 500                                 mbuf_pool_size,
 501                                 ad->dev_id,
 502                                 socket_id);
 503                 ad->hard_out_mbuf_pool = mp;
 504         }
 505
 506         /* Soft outputs */
 507         if (soft_out->nb_segments > 0) {
 508                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 509                                 soft_out->nb_segments);
 510                 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
 511                                 mbuf_pool_size,
 512                                 "soft_out");
 513                 TEST_ASSERT_NOT_NULL(mp,
 514                                 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 515                                 mbuf_pool_size,
 516                                 ad->dev_id,
 517                                 socket_id);
 518                 ad->soft_out_mbuf_pool = mp;
 519         }
 520
 521         /* HARQ inputs */
 522         if (harq_in->nb_segments > 0) {
 523                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 524                                 harq_in->nb_segments);
 525                 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
 526                                 mbuf_pool_size,
 527                                 "harq_in");
 528                 TEST_ASSERT_NOT_NULL(mp,
 529                                 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
 530                                 mbuf_pool_size,
 531                                 ad->dev_id,
 532                                 socket_id);
 533                 ad->harq_in_mbuf_pool = mp;
 534         }
 535
 536         /* HARQ outputs */
 537         if (harq_out->nb_segments > 0) {
 538                 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 539                                 harq_out->nb_segments);
 540                 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
 541                                 mbuf_pool_size,
 542                                 "harq_out");
 543                 TEST_ASSERT_NOT_NULL(mp,
 544                                 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
 545                                 mbuf_pool_size,
 546                                 ad->dev_id,
 547                                 socket_id);
 548                 ad->harq_out_mbuf_pool = mp;
 549         }
 550
 551         return TEST_SUCCESS;
 552 }
 553
 554 static int
 555 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 556                 struct test_bbdev_vector *vector)
 557 {
 558         int ret;
 559         unsigned int queue_id;
 560         struct rte_bbdev_queue_conf qconf;
 561         struct active_device *ad = &active_devs[nb_active_devs];
 562         unsigned int nb_queues;
 563         enum rte_bbdev_op_type op_type = vector->op_type;
 564
 565 /* Configure fpga lte fec with PF & VF values
 566  * if '-i' flag is set and using fpga device
 567  */
 568 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
 569         if ((get_init_device() == true) &&
 570                 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
 571                 struct fpga_lte_fec_conf conf;
 572                 unsigned int i;
 573
 574                 printf("Configure FPGA LTE FEC Driver %s with default values\n",
 575                                 info->drv.driver_name);
 576
 577                 /* clear default configuration before initialization */
 578                 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
 579
 580                 /* Set PF mode :
 581                  * true if PF is used for data plane
 582                  * false for VFs
 583                  */
 584                 conf.pf_mode_en = true;
 585
 586                 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
 587                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 588                         conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
 589                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 590                         conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
 591                 }
 592
 593                 /* UL bandwidth. Needed for schedule algorithm */
 594                 conf.ul_bandwidth = UL_4G_BANDWIDTH;
 595                 /* DL bandwidth */
 596                 conf.dl_bandwidth = DL_4G_BANDWIDTH;
 597
 598                 /* UL & DL load Balance Factor to 64 */
 599                 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
 600                 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
 601
 602                 /**< FLR timeout value */
 603                 conf.flr_time_out = FLR_4G_TIMEOUT;
 604
 605                 /* setup FPGA PF with configuration information */
 606                 ret = fpga_lte_fec_configure(info->dev_name, &conf);
 607                 TEST_ASSERT_SUCCESS(ret,
 608                                 "Failed to configure 4G FPGA PF for bbdev %s",
 609                                 info->dev_name);
 610         }
 611 #endif
 612 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC
 613         if ((get_init_device() == true) &&
 614                 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
 615                 struct fpga_5gnr_fec_conf conf;
 616                 unsigned int i;
 617
 618                 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
 619                                 info->drv.driver_name);
 620
 621                 /* clear default configuration before initialization */
 622                 memset(&conf, 0, sizeof(struct fpga_5gnr_fec_conf));
 623
 624                 /* Set PF mode :
 625                  * true if PF is used for data plane
 626                  * false for VFs
 627                  */
 628                 conf.pf_mode_en = true;
 629
 630                 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
 631                         /* Number of UL queues per VF (fpga supports 8 VFs) */
 632                         conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
 633                         /* Number of DL queues per VF (fpga supports 8 VFs) */
 634                         conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
 635                 }
 636
 637                 /* UL bandwidth. Needed for schedule algorithm */
 638                 conf.ul_bandwidth = UL_5G_BANDWIDTH;
 639                 /* DL bandwidth */
 640                 conf.dl_bandwidth = DL_5G_BANDWIDTH;
 641
 642                 /* UL & DL load Balance Factor to 64 */
 643                 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
 644                 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
 645
 646                 /**< FLR timeout value */
 647                 conf.flr_time_out = FLR_5G_TIMEOUT;
 648
 649                 /* setup FPGA PF with configuration information */
 650                 ret = fpga_5gnr_fec_configure(info->dev_name, &conf);
 651                 TEST_ASSERT_SUCCESS(ret,
 652                                 "Failed to configure 5G FPGA PF for bbdev %s",
 653                                 info->dev_name);
 654         }
 655 #endif
 656         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 657         nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
 658
 659         /* setup device */
 660         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 661         if (ret < 0) {
 662                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 663                                 dev_id, nb_queues, info->socket_id, ret);
 664                 return TEST_FAILED;
 665         }
 666
 667         /* configure interrupts if needed */
 668         if (intr_enabled) {
 669                 ret = rte_bbdev_intr_enable(dev_id);
 670                 if (ret < 0) {
 671                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 672                                         ret);
 673                         return TEST_FAILED;
 674                 }
 675         }
 676
 677         /* setup device queues */
 678         qconf.socket = info->socket_id;
 679         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 680         qconf.priority = 0;
 681         qconf.deferred_start = 0;
 682         qconf.op_type = op_type;
 683
 684         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 685                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 686                 if (ret != 0) {
 687                         printf(
 688                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 689                                         queue_id, qconf.priority, dev_id);
 690                         qconf.priority++;
 691                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 692                                         &qconf);
 693                 }
 694                 if (ret != 0) {
 695                         printf("All queues on dev %u allocated: %u\n",
 696                                         dev_id, queue_id);
 697                         break;
 698                 }
 699                 ad->queue_ids[queue_id] = queue_id;
 700         }
 701         TEST_ASSERT(queue_id != 0,
 702                         "ERROR Failed to configure any queues on dev %u",
 703                         dev_id);
 704         ad->nb_queues = queue_id;
 705
 706         set_avail_op(ad, op_type);
 707
 708         return TEST_SUCCESS;
 709 }
 710
 711 static int
 712 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 713                 struct test_bbdev_vector *vector)
 714 {
 715         int ret;
 716
 717         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 718         active_devs[nb_active_devs].dev_id = dev_id;
 719
 720         ret = add_bbdev_dev(dev_id, info, vector);
 721         if (ret == TEST_SUCCESS)
 722                 ++nb_active_devs;
 723         return ret;
 724 }
 725
 726 static uint8_t
 727 populate_active_devices(void)
 728 {
 729         int ret;
 730         uint8_t dev_id;
 731         uint8_t nb_devs_added = 0;
 732         struct rte_bbdev_info info;
 733
 734         RTE_BBDEV_FOREACH(dev_id) {
 735                 rte_bbdev_info_get(dev_id, &info);
 736
 737                 if (check_dev_cap(&info)) {
 738                         printf(
 739                                 "Device %d (%s) does not support specified capabilities\n",
 740                                         dev_id, info.dev_name);
 741                         continue;
 742                 }
 743
 744                 ret = add_active_device(dev_id, &info, &test_vector);
 745                 if (ret != 0) {
 746                         printf("Adding active bbdev %s skipped\n",
 747                                         info.dev_name);
 748                         continue;
 749                 }
 750                 nb_devs_added++;
 751         }
 752
 753         return nb_devs_added;
 754 }
 755
 756 static int
 757 read_test_vector(void)
 758 {
 759         int ret;
 760
 761         memset(&test_vector, 0, sizeof(test_vector));
 762         printf("Test vector file = %s\n", get_vector_filename());
 763         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 764         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 765                         get_vector_filename());
 766
 767         return TEST_SUCCESS;
 768 }
 769
 770 static int
 771 testsuite_setup(void)
 772 {
 773         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 774
 775         if (populate_active_devices() == 0) {
 776                 printf("No suitable devices found!\n");
 777                 return TEST_SKIPPED;
 778         }
 779
 780         return TEST_SUCCESS;
 781 }
 782
 783 static int
 784 interrupt_testsuite_setup(void)
 785 {
 786         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 787
 788         /* Enable interrupts */
 789         intr_enabled = true;
 790
 791         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 792         if (populate_active_devices() == 0 ||
 793                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 794                 intr_enabled = false;
 795                 printf("No suitable devices found!\n");
 796                 return TEST_SKIPPED;
 797         }
 798
 799         return TEST_SUCCESS;
 800 }
 801
 802 static void
 803 testsuite_teardown(void)
 804 {
 805         uint8_t dev_id;
 806
 807         /* Unconfigure devices */
 808         RTE_BBDEV_FOREACH(dev_id)
 809                 rte_bbdev_close(dev_id);
 810
 811         /* Clear active devices structs. */
 812         memset(active_devs, 0, sizeof(active_devs));
 813         nb_active_devs = 0;
 814
 815         /* Disable interrupts */
 816         intr_enabled = false;
 817 }
 818
 819 static int
 820 ut_setup(void)
 821 {
 822         uint8_t i, dev_id;
 823
 824         for (i = 0; i < nb_active_devs; i++) {
 825                 dev_id = active_devs[i].dev_id;
 826                 /* reset bbdev stats */
 827                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 828                                 "Failed to reset stats of bbdev %u", dev_id);
 829                 /* start the device */
 830                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 831                                 "Failed to start bbdev %u", dev_id);
 832         }
 833
 834         return TEST_SUCCESS;
 835 }
 836
 837 static void
 838 ut_teardown(void)
 839 {
 840         uint8_t i, dev_id;
 841         struct rte_bbdev_stats stats;
 842
 843         for (i = 0; i < nb_active_devs; i++) {
 844                 dev_id = active_devs[i].dev_id;
 845                 /* read stats and print */
 846                 rte_bbdev_stats_get(dev_id, &stats);
 847                 /* Stop the device */
 848                 rte_bbdev_stop(dev_id);
 849         }
 850 }
 851
 852 static int
 853 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 854                 struct op_data_entries *ref_entries,
 855                 struct rte_mempool *mbuf_pool, const uint16_t n,
 856                 enum op_data_type op_type, uint16_t min_alignment)
 857 {
 858         int ret;
 859         unsigned int i, j;
 860         bool large_input = false;
 861
 862         for (i = 0; i < n; ++i) {
 863                 char *data;
 864                 struct op_data_buf *seg = &ref_entries->segments[0];
 865                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 866                 TEST_ASSERT_NOT_NULL(m_head,
 867                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 868                                 op_type, n * ref_entries->nb_segments,
 869                                 mbuf_pool->size);
 870
 871                 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
 872                         /*
 873                          * Special case when DPDK mbuf cannot handle
 874                          * the required input size
 875                          */
 876                         printf("Warning: Larger input size than DPDK mbuf %d\n",
 877                                         seg->length);
 878                         large_input = true;
 879                 }
 880                 bufs[i].data = m_head;
 881                 bufs[i].offset = 0;
 882                 bufs[i].length = 0;
 883
 884                 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
 885                         if ((op_type == DATA_INPUT) && large_input) {
 886                                 /* Allocate a fake overused mbuf */
 887                                 data = rte_malloc(NULL, seg->length, 0);
 888                                 memcpy(data, seg->addr, seg->length);
 889                                 m_head->buf_addr = data;
 890                                 m_head->buf_iova = rte_malloc_virt2iova(data);
 891                                 m_head->data_off = 0;
 892                                 m_head->data_len = seg->length;
 893                         } else {
 894                                 data = rte_pktmbuf_append(m_head, seg->length);
 895                                 TEST_ASSERT_NOT_NULL(data,
 896                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 897                                         seg->length, op_type);
 898
 899                                 TEST_ASSERT(data == RTE_PTR_ALIGN(
 900                                                 data, min_alignment),
 901                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 902                                         data, min_alignment);
 903                                 rte_memcpy(data, seg->addr, seg->length);
 904                         }
 905
 906                         bufs[i].length += seg->length;
 907
 908                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 909                                 struct rte_mbuf *m_tail =
 910                                                 rte_pktmbuf_alloc(mbuf_pool);
 911                                 TEST_ASSERT_NOT_NULL(m_tail,
 912                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 913                                                 op_type,
 914                                                 n * ref_entries->nb_segments,
 915                                                 mbuf_pool->size);
 916                                 seg += 1;
 917
 918                                 data = rte_pktmbuf_append(m_tail, seg->length);
 919                                 TEST_ASSERT_NOT_NULL(data,
 920                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 921                                                 seg->length, op_type);
 922
 923                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 924                                                 min_alignment),
 925                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 926                                                 data, min_alignment);
 927                                 rte_memcpy(data, seg->addr, seg->length);
 928                                 bufs[i].length += seg->length;
 929
 930                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 931                                 TEST_ASSERT_SUCCESS(ret,
 932                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 933                                                 op_type);
 934                         }
 935                 } else {
 936
 937                         /* allocate chained-mbuf for output buffer */
 938                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 939                                 struct rte_mbuf *m_tail =
 940                                                 rte_pktmbuf_alloc(mbuf_pool);
 941                                 TEST_ASSERT_NOT_NULL(m_tail,
 942                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 943                                                 op_type,
 944                                                 n * ref_entries->nb_segments,
 945                                                 mbuf_pool->size);
 946
 947                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 948                                 TEST_ASSERT_SUCCESS(ret,
 949                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 950                                                 op_type);
 951                         }
 952                 }
 953         }
 954
 955         return 0;
 956 }
 957
 958 static int
 959 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
 960                 const int socket)
 961 {
 962         int i;
 963
 964         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
 965         if (*buffers == NULL) {
 966                 printf("WARNING: Failed to allocate op_data on socket %d\n",
 967                                 socket);
 968                 /* try to allocate memory on other detected sockets */
 969                 for (i = 0; i < socket; i++) {
 970                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
 971                         if (*buffers != NULL)
 972                                 break;
 973                 }
 974         }
 975
 976         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
 977 }
 978
 979 static void
 980 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
 981                 const uint16_t n, const int8_t max_llr_modulus)
 982 {
 983         uint16_t i, byte_idx;
 984
 985         for (i = 0; i < n; ++i) {
 986                 struct rte_mbuf *m = input_ops[i].data;
 987                 while (m != NULL) {
 988                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
 989                                         input_ops[i].offset);
 990                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
 991                                         ++byte_idx)
 992                                 llr[byte_idx] = round((double)max_llr_modulus *
 993                                                 llr[byte_idx] / INT8_MAX);
 994
 995                         m = m->next;
 996                 }
 997         }
 998 }
 999
1000 /*
1001  * We may have to insert filler bits
1002  * when they are required by the HARQ assumption
1003  */
1004 static void
1005 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1006                 const uint16_t n, struct test_op_params *op_params)
1007 {
1008         struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1009
1010         if (input_ops == NULL)
1011                 return;
1012         /* No need to add filler if not required by device */
1013         if (!(ldpc_cap_flags &
1014                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1015                 return;
1016         /* No need to add filler for loopback operation */
1017         if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1018                 return;
1019
1020         uint16_t i, j, parity_offset;
1021         for (i = 0; i < n; ++i) {
1022                 struct rte_mbuf *m = input_ops[i].data;
1023                 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1024                                 input_ops[i].offset);
1025                 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1026                                 * dec.z_c - dec.n_filler;
1027                 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1028                 m->data_len = new_hin_size;
1029                 input_ops[i].length = new_hin_size;
1030                 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1031                                 j--)
1032                         llr[j] = llr[j - dec.n_filler];
1033                 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1034                 for (j = 0; j < dec.n_filler; j++)
1035                         llr[parity_offset + j] = llr_max_pre_scaling;
1036         }
1037 }
1038
1039 static void
1040 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1041                 const uint16_t n, const int8_t llr_size,
1042                 const int8_t llr_decimals)
1043 {
1044         if (input_ops == NULL)
1045                 return;
1046
1047         uint16_t i, byte_idx;
1048
1049         int16_t llr_max, llr_min, llr_tmp;
1050         llr_max = (1 << (llr_size - 1)) - 1;
1051         llr_min = -llr_max;
1052         for (i = 0; i < n; ++i) {
1053                 struct rte_mbuf *m = input_ops[i].data;
1054                 while (m != NULL) {
1055                         int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1056                                         input_ops[i].offset);
1057                         for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1058                                         ++byte_idx) {
1059
1060                                 llr_tmp = llr[byte_idx];
1061                                 if (llr_decimals == 4)
1062                                         llr_tmp *= 8;
1063                                 else if (llr_decimals == 2)
1064                                         llr_tmp *= 2;
1065                                 else if (llr_decimals == 0)
1066                                         llr_tmp /= 2;
1067                                 llr_tmp = RTE_MIN(llr_max,
1068                                                 RTE_MAX(llr_min, llr_tmp));
1069                                 llr[byte_idx] = (int8_t) llr_tmp;
1070                         }
1071
1072                         m = m->next;
1073                 }
1074         }
1075 }
1076
1077
1078
1079 static int
1080 fill_queue_buffers(struct test_op_params *op_params,
1081                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1082                 struct rte_mempool *soft_out_mp,
1083                 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1084                 uint16_t queue_id,
1085                 const struct rte_bbdev_op_cap *capabilities,
1086                 uint16_t min_alignment, const int socket_id)
1087 {
1088         int ret;
1089         enum op_data_type type;
1090         const uint16_t n = op_params->num_to_process;
1091
1092         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1093                 in_mp,
1094                 soft_out_mp,
1095                 hard_out_mp,
1096                 harq_in_mp,
1097                 harq_out_mp,
1098         };
1099
1100         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1101                 &op_params->q_bufs[socket_id][queue_id].inputs,
1102                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1103                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1104                 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1105                 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1106         };
1107
1108         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1109                 struct op_data_entries *ref_entries =
1110                                 &test_vector.entries[type];
1111                 if (ref_entries->nb_segments == 0)
1112                         continue;
1113
1114                 ret = allocate_buffers_on_socket(queue_ops[type],
1115                                 n * sizeof(struct rte_bbdev_op_data),
1116                                 socket_id);
1117                 TEST_ASSERT_SUCCESS(ret,
1118                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
1119
1120                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1121                                 mbuf_pools[type], n, type, min_alignment);
1122                 TEST_ASSERT_SUCCESS(ret,
1123                                 "Couldn't init rte_bbdev_op_data structs");
1124         }
1125
1126         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1127                 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1128                         capabilities->cap.turbo_dec.max_llr_modulus);
1129
1130         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1131                 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1132                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1133                 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1134                                 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1135                 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1136                                 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1137                 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1138                 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1139                 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1140                 if (!loopback && !llr_comp)
1141                         ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1142                                         ldpc_llr_size, ldpc_llr_decimals);
1143                 if (!loopback && !harq_comp)
1144                         ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1145                                         ldpc_llr_size, ldpc_llr_decimals);
1146                 if (!loopback)
1147                         ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1148                                         op_params);
1149         }
1150
1151         return 0;
1152 }
1153
1154 static void
1155 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1156 {
1157         unsigned int i, j;
1158
1159         rte_mempool_free(ad->ops_mempool);
1160         rte_mempool_free(ad->in_mbuf_pool);
1161         rte_mempool_free(ad->hard_out_mbuf_pool);
1162         rte_mempool_free(ad->soft_out_mbuf_pool);
1163         rte_mempool_free(ad->harq_in_mbuf_pool);
1164         rte_mempool_free(ad->harq_out_mbuf_pool);
1165
1166         for (i = 0; i < rte_lcore_count(); ++i) {
1167                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1168                         rte_free(op_params->q_bufs[j][i].inputs);
1169                         rte_free(op_params->q_bufs[j][i].hard_outputs);
1170                         rte_free(op_params->q_bufs[j][i].soft_outputs);
1171                         rte_free(op_params->q_bufs[j][i].harq_inputs);
1172                         rte_free(op_params->q_bufs[j][i].harq_outputs);
1173                 }
1174         }
1175 }
1176
1177 static void
1178 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1179                 unsigned int start_idx,
1180                 struct rte_bbdev_op_data *inputs,
1181                 struct rte_bbdev_op_data *hard_outputs,
1182                 struct rte_bbdev_op_data *soft_outputs,
1183                 struct rte_bbdev_dec_op *ref_op)
1184 {
1185         unsigned int i;
1186         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1187
1188         for (i = 0; i < n; ++i) {
1189                 if (turbo_dec->code_block_mode == 0) {
1190                         ops[i]->turbo_dec.tb_params.ea =
1191                                         turbo_dec->tb_params.ea;
1192                         ops[i]->turbo_dec.tb_params.eb =
1193                                         turbo_dec->tb_params.eb;
1194                         ops[i]->turbo_dec.tb_params.k_pos =
1195                                         turbo_dec->tb_params.k_pos;
1196                         ops[i]->turbo_dec.tb_params.k_neg =
1197                                         turbo_dec->tb_params.k_neg;
1198                         ops[i]->turbo_dec.tb_params.c =
1199                                         turbo_dec->tb_params.c;
1200                         ops[i]->turbo_dec.tb_params.c_neg =
1201                                         turbo_dec->tb_params.c_neg;
1202                         ops[i]->turbo_dec.tb_params.cab =
1203                                         turbo_dec->tb_params.cab;
1204                         ops[i]->turbo_dec.tb_params.r =
1205                                         turbo_dec->tb_params.r;
1206                 } else {
1207                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1208                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1209                 }
1210
1211                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1212                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1213                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1214                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1215                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1216                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1217                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1218
1219                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1220                 ops[i]->turbo_dec.input = inputs[start_idx + i];
1221                 if (soft_outputs != NULL)
1222                         ops[i]->turbo_dec.soft_output =
1223                                 soft_outputs[start_idx + i];
1224         }
1225 }
1226
1227 static void
1228 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1229                 unsigned int start_idx,
1230                 struct rte_bbdev_op_data *inputs,
1231                 struct rte_bbdev_op_data *outputs,
1232                 struct rte_bbdev_enc_op *ref_op)
1233 {
1234         unsigned int i;
1235         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1236         for (i = 0; i < n; ++i) {
1237                 if (turbo_enc->code_block_mode == 0) {
1238                         ops[i]->turbo_enc.tb_params.ea =
1239                                         turbo_enc->tb_params.ea;
1240                         ops[i]->turbo_enc.tb_params.eb =
1241                                         turbo_enc->tb_params.eb;
1242                         ops[i]->turbo_enc.tb_params.k_pos =
1243                                         turbo_enc->tb_params.k_pos;
1244                         ops[i]->turbo_enc.tb_params.k_neg =
1245                                         turbo_enc->tb_params.k_neg;
1246                         ops[i]->turbo_enc.tb_params.c =
1247                                         turbo_enc->tb_params.c;
1248                         ops[i]->turbo_enc.tb_params.c_neg =
1249                                         turbo_enc->tb_params.c_neg;
1250                         ops[i]->turbo_enc.tb_params.cab =
1251                                         turbo_enc->tb_params.cab;
1252                         ops[i]->turbo_enc.tb_params.ncb_pos =
1253                                         turbo_enc->tb_params.ncb_pos;
1254                         ops[i]->turbo_enc.tb_params.ncb_neg =
1255                                         turbo_enc->tb_params.ncb_neg;
1256                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1257                 } else {
1258                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1259                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1260                         ops[i]->turbo_enc.cb_params.ncb =
1261                                         turbo_enc->cb_params.ncb;
1262                 }
1263                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1264                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1265                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1266
1267                 ops[i]->turbo_enc.output = outputs[start_idx + i];
1268                 ops[i]->turbo_enc.input = inputs[start_idx + i];
1269         }
1270 }
1271
1272
1273 /* Returns a random number drawn from a normal distribution
1274  * with mean of 0 and variance of 1
1275  * Marsaglia algorithm
1276  */
1277 static double
1278 randn(int n)
1279 {
1280         double S, Z, U1, U2, u, v, fac;
1281
1282         do {
1283                 U1 = (double)rand() / RAND_MAX;
1284                 U2 = (double)rand() / RAND_MAX;
1285                 u = 2. * U1 - 1.;
1286                 v = 2. * U2 - 1.;
1287                 S = u * u + v * v;
1288         } while (S >= 1 || S == 0);
1289         fac = sqrt(-2. * log(S) / S);
1290         Z = (n % 2) ? u * fac : v * fac;
1291         return Z;
1292 }
1293
1294 static inline double
1295 maxstar(double A, double B)
1296 {
1297         if (fabs(A - B) > 5)
1298                 return RTE_MAX(A, B);
1299         else
1300                 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1301 }
1302
1303 /*
1304  * Generate Qm LLRS for Qm==8
1305  * Modulation, AWGN and LLR estimation from max log development
1306  */
1307 static void
1308 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1309 {
1310         int qm = 8;
1311         int qam = 256;
1312         int m, k;
1313         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1314         /* 5.1.4 of TS38.211 */
1315         const double symbols_I[256] = {
1316                         5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1317                         5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1318                         11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1319                         15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1320                         15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1321                         1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1322                         1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1323                         15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1324                         13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1325                         -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1326                         -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1327                         -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1328                         -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1329                         -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1330                         -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1331                         -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1332                         -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1333                         -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1334                         -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1335                         -13, -13, -15, -15, -13, -13, -15, -15};
1336         const double symbols_Q[256] = {
1337                         5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1338                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1339                         15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1340                         11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1341                         15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1342                         -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1343                         -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1344                         -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1345                         -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1346                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1347                         7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1348                         9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1349                         13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1350                         3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1351                         13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1352                         -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1353                         -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1354                         -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1355                         -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1356                         -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1357         /* Average constellation point energy */
1358         N0 *= 170.0;
1359         for (k = 0; k < qm; k++)
1360                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1361         /* 5.1.4 of TS38.211 */
1362         I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1363                         (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1364         Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1365                         (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1366         /* AWGN channel */
1367         I += sqrt(N0 / 2) * randn(0);
1368         Q += sqrt(N0 / 2) * randn(1);
1369         /*
1370          * Calculate the log of the probability that each of
1371          * the constellation points was transmitted
1372          */
1373         for (m = 0; m < qam; m++)
1374                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1375                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1376         /* Calculate an LLR for each of the k_64QAM bits in the set */
1377         for (k = 0; k < qm; k++) {
1378                 p0 = -999999;
1379                 p1 = -999999;
1380                 /* For each constellation point */
1381                 for (m = 0; m < qam; m++) {
1382                         if ((m >> (qm - k - 1)) & 1)
1383                                 p1 = maxstar(p1, log_syml_prob[m]);
1384                         else
1385                                 p0 = maxstar(p0, log_syml_prob[m]);
1386                 }
1387                 /* Calculate the LLR */
1388                 llr_ = p0 - p1;
1389                 llr_ *= (1 << ldpc_llr_decimals);
1390                 llr_ = round(llr_);
1391                 if (llr_ > llr_max)
1392                         llr_ = llr_max;
1393                 if (llr_ < -llr_max)
1394                         llr_ = -llr_max;
1395                 llrs[qm * i + k] = (int8_t) llr_;
1396         }
1397 }
1398
1399
1400 /*
1401  * Generate Qm LLRS for Qm==6
1402  * Modulation, AWGN and LLR estimation from max log development
1403  */
1404 static void
1405 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1406 {
1407         int qm = 6;
1408         int qam = 64;
1409         int m, k;
1410         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1411         /* 5.1.4 of TS38.211 */
1412         const double symbols_I[64] = {
1413                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1414                         3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1415                         -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1416                         -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1417                         -5, -5, -7, -7, -5, -5, -7, -7};
1418         const double symbols_Q[64] = {
1419                         3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1420                         -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1421                         -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1422                         5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1423                         -3, -1, -3, -1, -5, -7, -5, -7};
1424         /* Average constellation point energy */
1425         N0 *= 42.0;
1426         for (k = 0; k < qm; k++)
1427                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1428         /* 5.1.4 of TS38.211 */
1429         I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1430         Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1431         /* AWGN channel */
1432         I += sqrt(N0 / 2) * randn(0);
1433         Q += sqrt(N0 / 2) * randn(1);
1434         /*
1435          * Calculate the log of the probability that each of
1436          * the constellation points was transmitted
1437          */
1438         for (m = 0; m < qam; m++)
1439                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1440                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1441         /* Calculate an LLR for each of the k_64QAM bits in the set */
1442         for (k = 0; k < qm; k++) {
1443                 p0 = -999999;
1444                 p1 = -999999;
1445                 /* For each constellation point */
1446                 for (m = 0; m < qam; m++) {
1447                         if ((m >> (qm - k - 1)) & 1)
1448                                 p1 = maxstar(p1, log_syml_prob[m]);
1449                         else
1450                                 p0 = maxstar(p0, log_syml_prob[m]);
1451                 }
1452                 /* Calculate the LLR */
1453                 llr_ = p0 - p1;
1454                 llr_ *= (1 << ldpc_llr_decimals);
1455                 llr_ = round(llr_);
1456                 if (llr_ > llr_max)
1457                         llr_ = llr_max;
1458                 if (llr_ < -llr_max)
1459                         llr_ = -llr_max;
1460                 llrs[qm * i + k] = (int8_t) llr_;
1461         }
1462 }
1463
1464 /*
1465  * Generate Qm LLRS for Qm==4
1466  * Modulation, AWGN and LLR estimation from max log development
1467  */
1468 static void
1469 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1470 {
1471         int qm = 4;
1472         int qam = 16;
1473         int m, k;
1474         double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1475         /* 5.1.4 of TS38.211 */
1476         const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1477                         -1, -1, -3, -3, -1, -1, -3, -3};
1478         const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1479                         1, 3, 1, 3, -1, -3, -1, -3};
1480         /* Average constellation point energy */
1481         N0 *= 10.0;
1482         for (k = 0; k < qm; k++)
1483                 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1484         /* 5.1.4 of TS38.211 */
1485         I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1486         Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1487         /* AWGN channel */
1488         I += sqrt(N0 / 2) * randn(0);
1489         Q += sqrt(N0 / 2) * randn(1);
1490         /*
1491          * Calculate the log of the probability that each of
1492          * the constellation points was transmitted
1493          */
1494         for (m = 0; m < qam; m++)
1495                 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1496                                 + pow(Q - symbols_Q[m], 2.0)) / N0;
1497         /* Calculate an LLR for each of the k_64QAM bits in the set */
1498         for (k = 0; k < qm; k++) {
1499                 p0 = -999999;
1500                 p1 = -999999;
1501                 /* For each constellation point */
1502                 for (m = 0; m < qam; m++) {
1503                         if ((m >> (qm - k - 1)) & 1)
1504                                 p1 = maxstar(p1, log_syml_prob[m]);
1505                         else
1506                                 p0 = maxstar(p0, log_syml_prob[m]);
1507                 }
1508                 /* Calculate the LLR */
1509                 llr_ = p0 - p1;
1510                 llr_ *= (1 << ldpc_llr_decimals);
1511                 llr_ = round(llr_);
1512                 if (llr_ > llr_max)
1513                         llr_ = llr_max;
1514                 if (llr_ < -llr_max)
1515                         llr_ = -llr_max;
1516                 llrs[qm * i + k] = (int8_t) llr_;
1517         }
1518 }
1519
1520 static void
1521 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1522 {
1523         double b, b1, n;
1524         double coeff = 2.0 * sqrt(N0);
1525
1526         /* Ignore in vectors rare quasi null LLRs not to be saturated */
1527         if (llrs[j] < 8 && llrs[j] > -8)
1528                 return;
1529
1530         /* Note don't change sign here */
1531         n = randn(j % 2);
1532         b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1533                         + coeff * n) / N0;
1534         b = b1 * (1 << ldpc_llr_decimals);
1535         b = round(b);
1536         if (b > llr_max)
1537                 b = llr_max;
1538         if (b < -llr_max)
1539                 b = -llr_max;
1540         llrs[j] = (int8_t) b;
1541 }
1542
1543 /* Generate LLR for a given SNR */
1544 static void
1545 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1546                 struct rte_bbdev_dec_op *ref_op)
1547 {
1548         struct rte_mbuf *m;
1549         uint16_t qm;
1550         uint32_t i, j, e, range;
1551         double N0, llr_max;
1552
1553         e = ref_op->ldpc_dec.cb_params.e;
1554         qm = ref_op->ldpc_dec.q_m;
1555         llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1556         range = e / qm;
1557         N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1558
1559         for (i = 0; i < n; ++i) {
1560                 m = inputs[i].data;
1561                 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1562                 if (qm == 8) {
1563                         for (j = 0; j < range; ++j)
1564                                 gen_qm8_llr(llrs, j, N0, llr_max);
1565                 } else if (qm == 6) {
1566                         for (j = 0; j < range; ++j)
1567                                 gen_qm6_llr(llrs, j, N0, llr_max);
1568                 } else if (qm == 4) {
1569                         for (j = 0; j < range; ++j)
1570                                 gen_qm4_llr(llrs, j, N0, llr_max);
1571                 } else {
1572                         for (j = 0; j < e; ++j)
1573                                 gen_qm2_llr(llrs, j, N0, llr_max);
1574                 }
1575         }
1576 }
1577
1578 static void
1579 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1580                 unsigned int start_idx,
1581                 struct rte_bbdev_op_data *inputs,
1582                 struct rte_bbdev_op_data *hard_outputs,
1583                 struct rte_bbdev_op_data *soft_outputs,
1584                 struct rte_bbdev_op_data *harq_inputs,
1585                 struct rte_bbdev_op_data *harq_outputs,
1586                 struct rte_bbdev_dec_op *ref_op)
1587 {
1588         unsigned int i;
1589         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1590
1591         for (i = 0; i < n; ++i) {
1592                 if (ldpc_dec->code_block_mode == 0) {
1593                         ops[i]->ldpc_dec.tb_params.ea =
1594                                         ldpc_dec->tb_params.ea;
1595                         ops[i]->ldpc_dec.tb_params.eb =
1596                                         ldpc_dec->tb_params.eb;
1597                         ops[i]->ldpc_dec.tb_params.c =
1598                                         ldpc_dec->tb_params.c;
1599                         ops[i]->ldpc_dec.tb_params.cab =
1600                                         ldpc_dec->tb_params.cab;
1601                         ops[i]->ldpc_dec.tb_params.r =
1602                                         ldpc_dec->tb_params.r;
1603                 } else {
1604                         ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1605                 }
1606
1607                 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1608                 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1609                 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1610                 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1611                 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1612                 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1613                 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1614                 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1615                 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1616
1617                 if (hard_outputs != NULL)
1618                         ops[i]->ldpc_dec.hard_output =
1619                                         hard_outputs[start_idx + i];
1620                 if (inputs != NULL)
1621                         ops[i]->ldpc_dec.input =
1622                                         inputs[start_idx + i];
1623                 if (soft_outputs != NULL)
1624                         ops[i]->ldpc_dec.soft_output =
1625                                         soft_outputs[start_idx + i];
1626                 if (harq_inputs != NULL)
1627                         ops[i]->ldpc_dec.harq_combined_input =
1628                                         harq_inputs[start_idx + i];
1629                 if (harq_outputs != NULL)
1630                         ops[i]->ldpc_dec.harq_combined_output =
1631                                         harq_outputs[start_idx + i];
1632         }
1633 }
1634
1635
1636 static void
1637 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1638                 unsigned int start_idx,
1639                 struct rte_bbdev_op_data *inputs,
1640                 struct rte_bbdev_op_data *outputs,
1641                 struct rte_bbdev_enc_op *ref_op)
1642 {
1643         unsigned int i;
1644         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1645         for (i = 0; i < n; ++i) {
1646                 if (ldpc_enc->code_block_mode == 0) {
1647                         ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1648                         ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1649                         ops[i]->ldpc_enc.tb_params.cab =
1650                                         ldpc_enc->tb_params.cab;
1651                         ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1652                         ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1653                 } else {
1654                         ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1655                 }
1656                 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1657                 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1658                 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1659                 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1660                 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1661                 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1662                 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1663                 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1664                 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1665                 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1666         }
1667 }
1668
1669 static int
1670 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1671                 unsigned int order_idx, const int expected_status)
1672 {
1673         int status = op->status;
1674         /* ignore parity mismatch false alarms for long iterations */
1675         if (get_iter_max() >= 10) {
1676                 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1677                                 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1678                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1679                         status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1680                 }
1681                 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1682                                 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1683                         printf("WARNING: Ignore Syndrome Check mismatch\n");
1684                         status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1685                 }
1686         }
1687
1688         TEST_ASSERT(status == expected_status,
1689                         "op_status (%d) != expected_status (%d)",
1690                         op->status, expected_status);
1691
1692         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1693                         "Ordering error, expected %p, got %p",
1694                         (void *)(uintptr_t)order_idx, op->opaque_data);
1695
1696         return TEST_SUCCESS;
1697 }
1698
1699 static int
1700 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1701                 unsigned int order_idx, const int expected_status)
1702 {
1703         TEST_ASSERT(op->status == expected_status,
1704                         "op_status (%d) != expected_status (%d)",
1705                         op->status, expected_status);
1706
1707         if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1708                 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1709                                 "Ordering error, expected %p, got %p",
1710                                 (void *)(uintptr_t)order_idx, op->opaque_data);
1711
1712         return TEST_SUCCESS;
1713 }
1714
1715 static inline int
1716 validate_op_chain(struct rte_bbdev_op_data *op,
1717                 struct op_data_entries *orig_op)
1718 {
1719         uint8_t i;
1720         struct rte_mbuf *m = op->data;
1721         uint8_t nb_dst_segments = orig_op->nb_segments;
1722         uint32_t total_data_size = 0;
1723
1724         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1725                         "Number of segments differ in original (%u) and filled (%u) op",
1726                         nb_dst_segments, m->nb_segs);
1727
1728         /* Validate each mbuf segment length */
1729         for (i = 0; i < nb_dst_segments; ++i) {
1730                 /* Apply offset to the first mbuf segment */
1731                 uint16_t offset = (i == 0) ? op->offset : 0;
1732                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1733                 total_data_size += orig_op->segments[i].length;
1734
1735                 TEST_ASSERT(orig_op->segments[i].length == data_len,
1736                                 "Length of segment differ in original (%u) and filled (%u) op",
1737                                 orig_op->segments[i].length, data_len);
1738                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1739                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1740                                 data_len,
1741                                 "Output buffers (CB=%u) are not equal", i);
1742                 m = m->next;
1743         }
1744
1745         /* Validate total mbuf pkt length */
1746         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1747         TEST_ASSERT(total_data_size == pkt_len,
1748                         "Length of data differ in original (%u) and filled (%u) op",
1749                         total_data_size, pkt_len);
1750
1751         return TEST_SUCCESS;
1752 }
1753
1754 /*
1755  * Compute K0 for a given configuration for HARQ output length computation
1756  * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1757  */
1758 static inline uint16_t
1759 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1760 {
1761         if (rv_index == 0)
1762                 return 0;
1763         uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1764         if (n_cb == n) {
1765                 if (rv_index == 1)
1766                         return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1767                 else if (rv_index == 2)
1768                         return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1769                 else
1770                         return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1771         }
1772         /* LBRM case - includes a division by N */
1773         if (rv_index == 1)
1774                 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1775                                 / n) * z_c;
1776         else if (rv_index == 2)
1777                 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1778                                 / n) * z_c;
1779         else
1780                 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1781                                 / n) * z_c;
1782 }
1783
1784 /* HARQ output length including the Filler bits */
1785 static inline uint16_t
1786 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1787 {
1788         uint16_t k0 = 0;
1789         uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1790         k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1791         /* Compute RM out size and number of rows */
1792         uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1793                         * ops_ld->z_c - ops_ld->n_filler;
1794         uint16_t deRmOutSize = RTE_MIN(
1795                         k0 + ops_ld->cb_params.e +
1796                         ((k0 > parity_offset) ?
1797                                         0 : ops_ld->n_filler),
1798                                         ops_ld->n_cb);
1799         uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1800                         / ops_ld->z_c);
1801         uint16_t harq_output_len = numRows * ops_ld->z_c;
1802         return harq_output_len;
1803 }
1804
1805 static inline int
1806 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1807                 struct op_data_entries *orig_op,
1808                 struct rte_bbdev_op_ldpc_dec *ops_ld)
1809 {
1810         uint8_t i;
1811         uint32_t j, jj, k;
1812         struct rte_mbuf *m = op->data;
1813         uint8_t nb_dst_segments = orig_op->nb_segments;
1814         uint32_t total_data_size = 0;
1815         int8_t *harq_orig, *harq_out, abs_harq_origin;
1816         uint32_t byte_error = 0, cum_error = 0, error;
1817         int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1818         int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1819         uint16_t parity_offset;
1820
1821         TEST_ASSERT(nb_dst_segments == m->nb_segs,
1822                         "Number of segments differ in original (%u) and filled (%u) op",
1823                         nb_dst_segments, m->nb_segs);
1824
1825         /* Validate each mbuf segment length */
1826         for (i = 0; i < nb_dst_segments; ++i) {
1827                 /* Apply offset to the first mbuf segment */
1828                 uint16_t offset = (i == 0) ? op->offset : 0;
1829                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1830                 total_data_size += orig_op->segments[i].length;
1831
1832                 TEST_ASSERT(orig_op->segments[i].length <
1833                                 (uint32_t)(data_len + 64),
1834                                 "Length of segment differ in original (%u) and filled (%u) op",
1835                                 orig_op->segments[i].length, data_len);
1836                 harq_orig = (int8_t *) orig_op->segments[i].addr;
1837                 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1838
1839                 if (!(ldpc_cap_flags &
1840                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1841                                 ) || (ops_ld->op_flags &
1842                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1843                         data_len -= ops_ld->z_c;
1844                         parity_offset = data_len;
1845                 } else {
1846                         /* Compute RM out size and number of rows */
1847                         parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1848                                         * ops_ld->z_c - ops_ld->n_filler;
1849                         uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1850                                         ops_ld->n_filler;
1851                         if (data_len > deRmOutSize)
1852                                 data_len = deRmOutSize;
1853                         if (data_len > orig_op->segments[i].length)
1854                                 data_len = orig_op->segments[i].length;
1855                 }
1856                 /*
1857                  * HARQ output can have minor differences
1858                  * due to integer representation and related scaling
1859                  */
1860                 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1861                         if (j == parity_offset) {
1862                                 /* Special Handling of the filler bits */
1863                                 for (k = 0; k < ops_ld->n_filler; k++) {
1864                                         if (harq_out[jj] !=
1865                                                         llr_max_pre_scaling) {
1866                                                 printf("HARQ Filler issue %d: %d %d\n",
1867                                                         jj, harq_out[jj],
1868                                                         llr_max);
1869                                                 byte_error++;
1870                                         }
1871                                         jj++;
1872                                 }
1873                         }
1874                         if (!(ops_ld->op_flags &
1875                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1876                                 if (ldpc_llr_decimals > 1)
1877                                         harq_out[jj] = (harq_out[jj] + 1)
1878                                                 >> (ldpc_llr_decimals - 1);
1879                                 /* Saturated to S7 */
1880                                 if (harq_orig[j] > llr_max)
1881                                         harq_orig[j] = llr_max;
1882                                 if (harq_orig[j] < -llr_max)
1883                                         harq_orig[j] = -llr_max;
1884                         }
1885                         if (harq_orig[j] != harq_out[jj]) {
1886                                 error = (harq_orig[j] > harq_out[jj]) ?
1887                                                 harq_orig[j] - harq_out[jj] :
1888                                                 harq_out[jj] - harq_orig[j];
1889                                 abs_harq_origin = harq_orig[j] > 0 ?
1890                                                         harq_orig[j] :
1891                                                         -harq_orig[j];
1892                                 /* Residual quantization error */
1893                                 if ((error > 8 && (abs_harq_origin <
1894                                                 (llr_max - 16))) ||
1895                                                 (error > 16)) {
1896                                         printf("HARQ mismatch %d: exp %d act %d => %d\n",
1897                                                         j, harq_orig[j],
1898                                                         harq_out[jj], error);
1899                                         byte_error++;
1900                                         cum_error += error;
1901                                 }
1902                         }
1903                 }
1904                 m = m->next;
1905         }
1906
1907         if (byte_error)
1908                 TEST_ASSERT(byte_error <= 1,
1909                                 "HARQ output mismatch (%d) %d",
1910                                 byte_error, cum_error);
1911
1912         /* Validate total mbuf pkt length */
1913         uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1914         TEST_ASSERT(total_data_size < pkt_len + 64,
1915                         "Length of data differ in original (%u) and filled (%u) op",
1916                         total_data_size, pkt_len);
1917
1918         return TEST_SUCCESS;
1919 }
1920
1921 static int
1922 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1923                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1924 {
1925         unsigned int i;
1926         int ret;
1927         struct op_data_entries *hard_data_orig =
1928                         &test_vector.entries[DATA_HARD_OUTPUT];
1929         struct op_data_entries *soft_data_orig =
1930                         &test_vector.entries[DATA_SOFT_OUTPUT];
1931         struct rte_bbdev_op_turbo_dec *ops_td;
1932         struct rte_bbdev_op_data *hard_output;
1933         struct rte_bbdev_op_data *soft_output;
1934         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
1935
1936         for (i = 0; i < n; ++i) {
1937                 ops_td = &ops[i]->turbo_dec;
1938                 hard_output = &ops_td->hard_output;
1939                 soft_output = &ops_td->soft_output;
1940
1941                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1942                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1943                                         "Returned iter_count (%d) > expected iter_count (%d)",
1944                                         ops_td->iter_count, ref_td->iter_count);
1945                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1946                 TEST_ASSERT_SUCCESS(ret,
1947                                 "Checking status and ordering for decoder failed");
1948
1949                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1950                                 hard_data_orig),
1951                                 "Hard output buffers (CB=%u) are not equal",
1952                                 i);
1953
1954                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
1955                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1956                                         soft_data_orig),
1957                                         "Soft output buffers (CB=%u) are not equal",
1958                                         i);
1959         }
1960
1961         return TEST_SUCCESS;
1962 }
1963
1964 /* Check Number of code blocks errors */
1965 static int
1966 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
1967 {
1968         unsigned int i;
1969         struct op_data_entries *hard_data_orig =
1970                         &test_vector.entries[DATA_HARD_OUTPUT];
1971         struct rte_bbdev_op_ldpc_dec *ops_td;
1972         struct rte_bbdev_op_data *hard_output;
1973         int errors = 0;
1974         struct rte_mbuf *m;
1975
1976         for (i = 0; i < n; ++i) {
1977                 ops_td = &ops[i]->ldpc_dec;
1978                 hard_output = &ops_td->hard_output;
1979                 m = hard_output->data;
1980                 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
1981                                 hard_data_orig->segments[0].addr,
1982                                 hard_data_orig->segments[0].length))
1983                         errors++;
1984         }
1985         return errors;
1986 }
1987
1988 static int
1989 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1990                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1991 {
1992         unsigned int i;
1993         int ret;
1994         struct op_data_entries *hard_data_orig =
1995                         &test_vector.entries[DATA_HARD_OUTPUT];
1996         struct op_data_entries *soft_data_orig =
1997                         &test_vector.entries[DATA_SOFT_OUTPUT];
1998         struct op_data_entries *harq_data_orig =
1999                                 &test_vector.entries[DATA_HARQ_OUTPUT];
2000         struct rte_bbdev_op_ldpc_dec *ops_td;
2001         struct rte_bbdev_op_data *hard_output;
2002         struct rte_bbdev_op_data *harq_output;
2003         struct rte_bbdev_op_data *soft_output;
2004         struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2005
2006         for (i = 0; i < n; ++i) {
2007                 ops_td = &ops[i]->ldpc_dec;
2008                 hard_output = &ops_td->hard_output;
2009                 harq_output = &ops_td->harq_combined_output;
2010                 soft_output = &ops_td->soft_output;
2011
2012                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2013                 TEST_ASSERT_SUCCESS(ret,
2014                                 "Checking status and ordering for decoder failed");
2015                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2016                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2017                                         "Returned iter_count (%d) > expected iter_count (%d)",
2018                                         ops_td->iter_count, ref_td->iter_count);
2019                 /*
2020                  * We can ignore output data when the decoding failed to
2021                  * converge or for loop-back cases
2022                  */
2023                 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2024                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2025                                 ) && (
2026                                 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2027                                                 )) == 0)
2028                         TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2029                                         hard_data_orig),
2030                                         "Hard output buffers (CB=%u) are not equal",
2031                                         i);
2032
2033                 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2034                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2035                                         soft_data_orig),
2036                                         "Soft output buffers (CB=%u) are not equal",
2037                                         i);
2038                 if (ref_op->ldpc_dec.op_flags &
2039                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2040                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2041                                         harq_data_orig, ops_td),
2042                                         "HARQ output buffers (CB=%u) are not equal",
2043                                         i);
2044                 }
2045                 if (ref_op->ldpc_dec.op_flags &
2046                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2047                         TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2048                                         harq_data_orig, ops_td),
2049                                         "HARQ output buffers (CB=%u) are not equal",
2050                                         i);
2051
2052         }
2053
2054         return TEST_SUCCESS;
2055 }
2056
2057
2058 static int
2059 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2060                 struct rte_bbdev_enc_op *ref_op)
2061 {
2062         unsigned int i;
2063         int ret;
2064         struct op_data_entries *hard_data_orig =
2065                         &test_vector.entries[DATA_HARD_OUTPUT];
2066
2067         for (i = 0; i < n; ++i) {
2068                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2069                 TEST_ASSERT_SUCCESS(ret,
2070                                 "Checking status and ordering for encoder failed");
2071                 TEST_ASSERT_SUCCESS(validate_op_chain(
2072                                 &ops[i]->turbo_enc.output,
2073                                 hard_data_orig),
2074                                 "Output buffers (CB=%u) are not equal",
2075                                 i);
2076         }
2077
2078         return TEST_SUCCESS;
2079 }
2080
2081 static int
2082 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2083                 struct rte_bbdev_enc_op *ref_op)
2084 {
2085         unsigned int i;
2086         int ret;
2087         struct op_data_entries *hard_data_orig =
2088                         &test_vector.entries[DATA_HARD_OUTPUT];
2089
2090         for (i = 0; i < n; ++i) {
2091                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2092                 TEST_ASSERT_SUCCESS(ret,
2093                                 "Checking status and ordering for encoder failed");
2094                 TEST_ASSERT_SUCCESS(validate_op_chain(
2095                                 &ops[i]->ldpc_enc.output,
2096                                 hard_data_orig),
2097                                 "Output buffers (CB=%u) are not equal",
2098                                 i);
2099         }
2100
2101         return TEST_SUCCESS;
2102 }
2103
2104 static void
2105 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2106 {
2107         unsigned int i;
2108         struct op_data_entries *entry;
2109
2110         op->turbo_dec = test_vector.turbo_dec;
2111         entry = &test_vector.entries[DATA_INPUT];
2112         for (i = 0; i < entry->nb_segments; ++i)
2113                 op->turbo_dec.input.length +=
2114                                 entry->segments[i].length;
2115 }
2116
2117 static void
2118 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2119 {
2120         unsigned int i;
2121         struct op_data_entries *entry;
2122
2123         op->ldpc_dec = test_vector.ldpc_dec;
2124         entry = &test_vector.entries[DATA_INPUT];
2125         for (i = 0; i < entry->nb_segments; ++i)
2126                 op->ldpc_dec.input.length +=
2127                                 entry->segments[i].length;
2128         if (test_vector.ldpc_dec.op_flags &
2129                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2130                 entry = &test_vector.entries[DATA_HARQ_INPUT];
2131                 for (i = 0; i < entry->nb_segments; ++i)
2132                         op->ldpc_dec.harq_combined_input.length +=
2133                                 entry->segments[i].length;
2134         }
2135 }
2136
2137
2138 static void
2139 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2140 {
2141         unsigned int i;
2142         struct op_data_entries *entry;
2143
2144         op->turbo_enc = test_vector.turbo_enc;
2145         entry = &test_vector.entries[DATA_INPUT];
2146         for (i = 0; i < entry->nb_segments; ++i)
2147                 op->turbo_enc.input.length +=
2148                                 entry->segments[i].length;
2149 }
2150
2151 static void
2152 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2153 {
2154         unsigned int i;
2155         struct op_data_entries *entry;
2156
2157         op->ldpc_enc = test_vector.ldpc_enc;
2158         entry = &test_vector.entries[DATA_INPUT];
2159         for (i = 0; i < entry->nb_segments; ++i)
2160                 op->ldpc_enc.input.length +=
2161                                 entry->segments[i].length;
2162 }
2163
2164 static uint32_t
2165 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2166 {
2167         uint8_t i;
2168         uint32_t c, r, tb_size = 0;
2169
2170         if (op->turbo_dec.code_block_mode) {
2171                 tb_size = op->turbo_dec.tb_params.k_neg;
2172         } else {
2173                 c = op->turbo_dec.tb_params.c;
2174                 r = op->turbo_dec.tb_params.r;
2175                 for (i = 0; i < c-r; i++)
2176                         tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2177                                 op->turbo_dec.tb_params.k_neg :
2178                                 op->turbo_dec.tb_params.k_pos;
2179         }
2180         return tb_size;
2181 }
2182
2183 static uint32_t
2184 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2185 {
2186         uint8_t i;
2187         uint32_t c, r, tb_size = 0;
2188         uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2189
2190         if (op->ldpc_dec.code_block_mode) {
2191                 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2192         } else {
2193                 c = op->ldpc_dec.tb_params.c;
2194                 r = op->ldpc_dec.tb_params.r;
2195                 for (i = 0; i < c-r; i++)
2196                         tb_size += sys_cols * op->ldpc_dec.z_c
2197                                         - op->ldpc_dec.n_filler;
2198         }
2199         return tb_size;
2200 }
2201
2202 static uint32_t
2203 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2204 {
2205         uint8_t i;
2206         uint32_t c, r, tb_size = 0;
2207
2208         if (op->turbo_enc.code_block_mode) {
2209                 tb_size = op->turbo_enc.tb_params.k_neg;
2210         } else {
2211                 c = op->turbo_enc.tb_params.c;
2212                 r = op->turbo_enc.tb_params.r;
2213                 for (i = 0; i < c-r; i++)
2214                         tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2215                                 op->turbo_enc.tb_params.k_neg :
2216                                 op->turbo_enc.tb_params.k_pos;
2217         }
2218         return tb_size;
2219 }
2220
2221 static uint32_t
2222 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2223 {
2224         uint8_t i;
2225         uint32_t c, r, tb_size = 0;
2226         uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2227
2228         if (op->turbo_enc.code_block_mode) {
2229                 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2230         } else {
2231                 c = op->turbo_enc.tb_params.c;
2232                 r = op->turbo_enc.tb_params.r;
2233                 for (i = 0; i < c-r; i++)
2234                         tb_size += sys_cols * op->ldpc_enc.z_c
2235                                         - op->ldpc_enc.n_filler;
2236         }
2237         return tb_size;
2238 }
2239
2240
2241 static int
2242 init_test_op_params(struct test_op_params *op_params,
2243                 enum rte_bbdev_op_type op_type, const int expected_status,
2244                 const int vector_mask, struct rte_mempool *ops_mp,
2245                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2246 {
2247         int ret = 0;
2248         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2249                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2250                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2251                                 &op_params->ref_dec_op, 1);
2252         else
2253                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2254                                 &op_params->ref_enc_op, 1);
2255
2256         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2257
2258         op_params->mp = ops_mp;
2259         op_params->burst_sz = burst_sz;
2260         op_params->num_to_process = num_to_process;
2261         op_params->num_lcores = num_lcores;
2262         op_params->vector_mask = vector_mask;
2263         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2264                         op_type == RTE_BBDEV_OP_LDPC_DEC)
2265                 op_params->ref_dec_op->status = expected_status;
2266         else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2267                         || op_type == RTE_BBDEV_OP_LDPC_ENC)
2268                 op_params->ref_enc_op->status = expected_status;
2269         return 0;
2270 }
2271
2272 static int
2273 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2274                 struct test_op_params *op_params)
2275 {
2276         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2277         unsigned int i;
2278         struct active_device *ad;
2279         unsigned int burst_sz = get_burst_sz();
2280         enum rte_bbdev_op_type op_type = test_vector.op_type;
2281         const struct rte_bbdev_op_cap *capabilities = NULL;
2282
2283         ad = &active_devs[dev_id];
2284
2285         /* Check if device supports op_type */
2286         if (!is_avail_op(ad, test_vector.op_type))
2287                 return TEST_SUCCESS;
2288
2289         struct rte_bbdev_info info;
2290         rte_bbdev_info_get(ad->dev_id, &info);
2291         socket_id = GET_SOCKET(info.socket_id);
2292
2293         f_ret = create_mempools(ad, socket_id, op_type,
2294                         get_num_ops());
2295         if (f_ret != TEST_SUCCESS) {
2296                 printf("Couldn't create mempools");
2297                 goto fail;
2298         }
2299         if (op_type == RTE_BBDEV_OP_NONE)
2300                 op_type = RTE_BBDEV_OP_TURBO_ENC;
2301
2302         f_ret = init_test_op_params(op_params, test_vector.op_type,
2303                         test_vector.expected_status,
2304                         test_vector.mask,
2305                         ad->ops_mempool,
2306                         burst_sz,
2307                         get_num_ops(),
2308                         get_num_lcores());
2309         if (f_ret != TEST_SUCCESS) {
2310                 printf("Couldn't init test op params");
2311                 goto fail;
2312         }
2313
2314
2315         /* Find capabilities */
2316         const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2317         for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2318                 if (cap->type == test_vector.op_type) {
2319                         capabilities = cap;
2320                         break;
2321                 }
2322                 cap++;
2323         }
2324         TEST_ASSERT_NOT_NULL(capabilities,
2325                         "Couldn't find capabilities");
2326
2327         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2328                 create_reference_dec_op(op_params->ref_dec_op);
2329         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2330                 create_reference_enc_op(op_params->ref_enc_op);
2331         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2332                 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2333         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2334                 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2335
2336         for (i = 0; i < ad->nb_queues; ++i) {
2337                 f_ret = fill_queue_buffers(op_params,
2338                                 ad->in_mbuf_pool,
2339                                 ad->hard_out_mbuf_pool,
2340                                 ad->soft_out_mbuf_pool,
2341                                 ad->harq_in_mbuf_pool,
2342                                 ad->harq_out_mbuf_pool,
2343                                 ad->queue_ids[i],
2344                                 capabilities,
2345                                 info.drv.min_alignment,
2346                                 socket_id);
2347                 if (f_ret != TEST_SUCCESS) {
2348                         printf("Couldn't init queue buffers");
2349                         goto fail;
2350                 }
2351         }
2352
2353         /* Run test case function */
2354         t_ret = test_case_func(ad, op_params);
2355
2356         /* Free active device resources and return */
2357         free_buffers(ad, op_params);
2358         return t_ret;
2359
2360 fail:
2361         free_buffers(ad, op_params);
2362         return TEST_FAILED;
2363 }
2364
2365 /* Run given test function per active device per supported op type
2366  * per burst size.
2367  */
2368 static int
2369 run_test_case(test_case_function *test_case_func)
2370 {
2371         int ret = 0;
2372         uint8_t dev;
2373
2374         /* Alloc op_params */
2375         struct test_op_params *op_params = rte_zmalloc(NULL,
2376                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2377         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2378                         RTE_ALIGN(sizeof(struct test_op_params),
2379                                 RTE_CACHE_LINE_SIZE));
2380
2381         /* For each device run test case function */
2382         for (dev = 0; dev < nb_active_devs; ++dev)
2383                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2384
2385         rte_free(op_params);
2386
2387         return ret;
2388 }
2389
2390
2391 /* Push back the HARQ output from DDR to host */
2392 static void
2393 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2394                 struct rte_bbdev_dec_op **ops,
2395                 const uint16_t n)
2396 {
2397         uint16_t j;
2398         int save_status, ret;
2399         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2400         struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2401         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2402         bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2403         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2404         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2405         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2406         for (j = 0; j < n; ++j) {
2407                 if ((loopback && mem_out) || hc_out) {
2408                         save_status = ops[j]->status;
2409                         ops[j]->ldpc_dec.op_flags =
2410                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2411                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2412                         if (h_comp)
2413                                 ops[j]->ldpc_dec.op_flags +=
2414                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2415                         ops[j]->ldpc_dec.harq_combined_input.offset =
2416                                         harq_offset;
2417                         ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2418                         harq_offset += HARQ_INCR;
2419                         if (!loopback)
2420                                 ops[j]->ldpc_dec.harq_combined_input.length =
2421                                 ops[j]->ldpc_dec.harq_combined_output.length;
2422                         rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2423                                         &ops[j], 1);
2424                         ret = 0;
2425                         while (ret == 0)
2426                                 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2427                                                 dev_id, queue_id,
2428                                                 &ops_deq[j], 1);
2429                         ops[j]->ldpc_dec.op_flags = flags;
2430                         ops[j]->status = save_status;
2431                 }
2432         }
2433 }
2434
2435 /*
2436  * Push back the HARQ output from HW DDR to Host
2437  * Preload HARQ memory input and adjust HARQ offset
2438  */
2439 static void
2440 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2441                 struct rte_bbdev_dec_op **ops, const uint16_t n,
2442                 bool preload)
2443 {
2444         uint16_t j;
2445         int ret;
2446         uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2447         struct rte_bbdev_op_data save_hc_in, save_hc_out;
2448         struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2449         uint32_t flags = ops[0]->ldpc_dec.op_flags;
2450         bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2451         bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2452         bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2453         bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2454         bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2455         for (j = 0; j < n; ++j) {
2456                 if ((mem_in || hc_in) && preload) {
2457                         save_hc_in = ops[j]->ldpc_dec.harq_combined_input;
2458                         save_hc_out = ops[j]->ldpc_dec.harq_combined_output;
2459                         ops[j]->ldpc_dec.op_flags =
2460                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2461                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2462                         if (h_comp)
2463                                 ops[j]->ldpc_dec.op_flags +=
2464                                         RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2465                         ops[j]->ldpc_dec.harq_combined_output.offset =
2466                                         harq_offset;
2467                         ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2468                         rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2469                                         &ops[j], 1);
2470                         ret = 0;
2471                         while (ret == 0)
2472                                 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2473                                         dev_id, queue_id, &ops_deq[j], 1);
2474                         ops[j]->ldpc_dec.op_flags = flags;
2475                         ops[j]->ldpc_dec.harq_combined_input = save_hc_in;
2476                         ops[j]->ldpc_dec.harq_combined_output = save_hc_out;
2477                 }
2478                 /* Adjust HARQ offset when we reach external DDR */
2479                 if (mem_in || hc_in)
2480                         ops[j]->ldpc_dec.harq_combined_input.offset
2481                                 = harq_offset;
2482                 if (mem_out || hc_out)
2483                         ops[j]->ldpc_dec.harq_combined_output.offset
2484                                 = harq_offset;
2485                 harq_offset += HARQ_INCR;
2486         }
2487 }
2488
2489 static void
2490 dequeue_event_callback(uint16_t dev_id,
2491                 enum rte_bbdev_event_type event, void *cb_arg,
2492                 void *ret_param)
2493 {
2494         int ret;
2495         uint16_t i;
2496         uint64_t total_time;
2497         uint16_t deq, burst_sz, num_ops;
2498         uint16_t queue_id = *(uint16_t *) ret_param;
2499         struct rte_bbdev_info info;
2500         double tb_len_bits;
2501         struct thread_params *tp = cb_arg;
2502
2503         /* Find matching thread params using queue_id */
2504         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2505                 if (tp->queue_id == queue_id)
2506                         break;
2507
2508         if (i == MAX_QUEUES) {
2509                 printf("%s: Queue_id from interrupt details was not found!\n",
2510                                 __func__);
2511                 return;
2512         }
2513
2514         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2515                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2516                 printf(
2517                         "Dequeue interrupt handler called for incorrect event!\n");
2518                 return;
2519         }
2520
2521         burst_sz = rte_atomic16_read(&tp->burst_sz);
2522         num_ops = tp->op_params->num_to_process;
2523
2524         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2525                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2526                                 &tp->dec_ops[
2527                                         rte_atomic16_read(&tp->nb_dequeued)],
2528                                 burst_sz);
2529         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2530                 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2531                                 &tp->dec_ops[
2532                                         rte_atomic16_read(&tp->nb_dequeued)],
2533                                 burst_sz);
2534         else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2535                 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2536                                 &tp->enc_ops[
2537                                         rte_atomic16_read(&tp->nb_dequeued)],
2538                                 burst_sz);
2539         else /*RTE_BBDEV_OP_TURBO_ENC*/
2540                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2541                                 &tp->enc_ops[
2542                                         rte_atomic16_read(&tp->nb_dequeued)],
2543                                 burst_sz);
2544
2545         if (deq < burst_sz) {
2546                 printf(
2547                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2548                         burst_sz, deq);
2549                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2550                 return;
2551         }
2552
2553         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2554                 rte_atomic16_add(&tp->nb_dequeued, deq);
2555                 return;
2556         }
2557
2558         total_time = rte_rdtsc_precise() - tp->start_time;
2559
2560         rte_bbdev_info_get(dev_id, &info);
2561
2562         ret = TEST_SUCCESS;
2563
2564         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2565                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2566                 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2567                                 tp->op_params->vector_mask);
2568                 /* get the max of iter_count for all dequeued ops */
2569                 for (i = 0; i < num_ops; ++i)
2570                         tp->iter_count = RTE_MAX(
2571                                         tp->dec_ops[i]->turbo_dec.iter_count,
2572                                         tp->iter_count);
2573                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2574         } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2575                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2576                 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2577                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2578         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2579                 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2580                 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2581                 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2582         } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2583                 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2584                 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2585                                 tp->op_params->vector_mask);
2586                 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2587         }
2588
2589         if (ret) {
2590                 printf("Buffers validation failed\n");
2591                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2592         }
2593
2594         switch (test_vector.op_type) {
2595         case RTE_BBDEV_OP_TURBO_DEC:
2596                 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2597                 break;
2598         case RTE_BBDEV_OP_TURBO_ENC:
2599                 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2600                 break;
2601         case RTE_BBDEV_OP_LDPC_DEC:
2602                 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2603                 break;
2604         case RTE_BBDEV_OP_LDPC_ENC:
2605                 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2606                 break;
2607         case RTE_BBDEV_OP_NONE:
2608                 tb_len_bits = 0.0;
2609                 break;
2610         default:
2611                 printf("Unknown op type: %d\n", test_vector.op_type);
2612                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2613                 return;
2614         }
2615
2616         tp->ops_per_sec += ((double)num_ops) /
2617                         ((double)total_time / (double)rte_get_tsc_hz());
2618         tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2619                         ((double)total_time / (double)rte_get_tsc_hz());
2620
2621         rte_atomic16_add(&tp->nb_dequeued, deq);
2622 }
2623
2624 static int
2625 throughput_intr_lcore_ldpc_dec(void *arg)
2626 {
2627         struct thread_params *tp = arg;
2628         unsigned int enqueued;
2629         const uint16_t queue_id = tp->queue_id;
2630         const uint16_t burst_sz = tp->op_params->burst_sz;
2631         const uint16_t num_to_process = tp->op_params->num_to_process;
2632         struct rte_bbdev_dec_op *ops[num_to_process];
2633         struct test_buffers *bufs = NULL;
2634         struct rte_bbdev_info info;
2635         int ret, i, j;
2636         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2637         uint16_t num_to_enq, enq;
2638
2639         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2640                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2641         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2642                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2643
2644         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2645                         "BURST_SIZE should be <= %u", MAX_BURST);
2646
2647         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2648                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2649                         tp->dev_id, queue_id);
2650
2651         rte_bbdev_info_get(tp->dev_id, &info);
2652
2653         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2654                         "NUM_OPS cannot exceed %u for this device",
2655                         info.drv.queue_size_lim);
2656
2657         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2658
2659         rte_atomic16_clear(&tp->processing_status);
2660         rte_atomic16_clear(&tp->nb_dequeued);
2661
2662         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2663                 rte_pause();
2664
2665         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2666                                 num_to_process);
2667         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2668                         num_to_process);
2669         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2670                 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2671                                 bufs->hard_outputs, bufs->soft_outputs,
2672                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2673
2674         /* Set counter to validate the ordering */
2675         for (j = 0; j < num_to_process; ++j)
2676                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2677
2678         for (j = 0; j < TEST_REPETITIONS; ++j) {
2679                 for (i = 0; i < num_to_process; ++i) {
2680                         if (!loopback)
2681                                 rte_pktmbuf_reset(
2682                                         ops[i]->ldpc_dec.hard_output.data);
2683                         if (hc_out || loopback)
2684                                 mbuf_reset(
2685                                 ops[i]->ldpc_dec.harq_combined_output.data);
2686                 }
2687
2688                 tp->start_time = rte_rdtsc_precise();
2689                 for (enqueued = 0; enqueued < num_to_process;) {
2690                         num_to_enq = burst_sz;
2691
2692                         if (unlikely(num_to_process - enqueued < num_to_enq))
2693                                 num_to_enq = num_to_process - enqueued;
2694
2695                         enq = 0;
2696                         do {
2697                                 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2698                                                 tp->dev_id,
2699                                                 queue_id, &ops[enqueued],
2700                                                 num_to_enq);
2701                         } while (unlikely(num_to_enq != enq));
2702                         enqueued += enq;
2703
2704                         /* Write to thread burst_sz current number of enqueued
2705                          * descriptors. It ensures that proper number of
2706                          * descriptors will be dequeued in callback
2707                          * function - needed for last batch in case where
2708                          * the number of operations is not a multiple of
2709                          * burst size.
2710                          */
2711                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2712
2713                         /* Wait until processing of previous batch is
2714                          * completed
2715                          */
2716                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2717                                         (int16_t) enqueued)
2718                                 rte_pause();
2719                 }
2720                 if (j != TEST_REPETITIONS - 1)
2721                         rte_atomic16_clear(&tp->nb_dequeued);
2722         }
2723
2724         return TEST_SUCCESS;
2725 }
2726
2727 static int
2728 throughput_intr_lcore_dec(void *arg)
2729 {
2730         struct thread_params *tp = arg;
2731         unsigned int enqueued;
2732         const uint16_t queue_id = tp->queue_id;
2733         const uint16_t burst_sz = tp->op_params->burst_sz;
2734         const uint16_t num_to_process = tp->op_params->num_to_process;
2735         struct rte_bbdev_dec_op *ops[num_to_process];
2736         struct test_buffers *bufs = NULL;
2737         struct rte_bbdev_info info;
2738         int ret, i, j;
2739         uint16_t num_to_enq, enq;
2740
2741         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2742                         "BURST_SIZE should be <= %u", MAX_BURST);
2743
2744         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2745                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2746                         tp->dev_id, queue_id);
2747
2748         rte_bbdev_info_get(tp->dev_id, &info);
2749
2750         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2751                         "NUM_OPS cannot exceed %u for this device",
2752                         info.drv.queue_size_lim);
2753
2754         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2755
2756         rte_atomic16_clear(&tp->processing_status);
2757         rte_atomic16_clear(&tp->nb_dequeued);
2758
2759         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2760                 rte_pause();
2761
2762         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2763                                 num_to_process);
2764         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2765                         num_to_process);
2766         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2767                 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2768                                 bufs->hard_outputs, bufs->soft_outputs,
2769                                 tp->op_params->ref_dec_op);
2770
2771         /* Set counter to validate the ordering */
2772         for (j = 0; j < num_to_process; ++j)
2773                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2774
2775         for (j = 0; j < TEST_REPETITIONS; ++j) {
2776                 for (i = 0; i < num_to_process; ++i)
2777                         rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2778
2779                 tp->start_time = rte_rdtsc_precise();
2780                 for (enqueued = 0; enqueued < num_to_process;) {
2781                         num_to_enq = burst_sz;
2782
2783                         if (unlikely(num_to_process - enqueued < num_to_enq))
2784                                 num_to_enq = num_to_process - enqueued;
2785
2786                         enq = 0;
2787                         do {
2788                                 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2789                                                 queue_id, &ops[enqueued],
2790                                                 num_to_enq);
2791                         } while (unlikely(num_to_enq != enq));
2792                         enqueued += enq;
2793
2794                         /* Write to thread burst_sz current number of enqueued
2795                          * descriptors. It ensures that proper number of
2796                          * descriptors will be dequeued in callback
2797                          * function - needed for last batch in case where
2798                          * the number of operations is not a multiple of
2799                          * burst size.
2800                          */
2801                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2802
2803                         /* Wait until processing of previous batch is
2804                          * completed
2805                          */
2806                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2807                                         (int16_t) enqueued)
2808                                 rte_pause();
2809                 }
2810                 if (j != TEST_REPETITIONS - 1)
2811                         rte_atomic16_clear(&tp->nb_dequeued);
2812         }
2813
2814         return TEST_SUCCESS;
2815 }
2816
2817 static int
2818 throughput_intr_lcore_enc(void *arg)
2819 {
2820         struct thread_params *tp = arg;
2821         unsigned int enqueued;
2822         const uint16_t queue_id = tp->queue_id;
2823         const uint16_t burst_sz = tp->op_params->burst_sz;
2824         const uint16_t num_to_process = tp->op_params->num_to_process;
2825         struct rte_bbdev_enc_op *ops[num_to_process];
2826         struct test_buffers *bufs = NULL;
2827         struct rte_bbdev_info info;
2828         int ret, i, j;
2829         uint16_t num_to_enq, enq;
2830
2831         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2832                         "BURST_SIZE should be <= %u", MAX_BURST);
2833
2834         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2835                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2836                         tp->dev_id, queue_id);
2837
2838         rte_bbdev_info_get(tp->dev_id, &info);
2839
2840         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2841                         "NUM_OPS cannot exceed %u for this device",
2842                         info.drv.queue_size_lim);
2843
2844         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2845
2846         rte_atomic16_clear(&tp->processing_status);
2847         rte_atomic16_clear(&tp->nb_dequeued);
2848
2849         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2850                 rte_pause();
2851
2852         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2853                         num_to_process);
2854         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2855                         num_to_process);
2856         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2857                 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2858                                 bufs->hard_outputs, tp->op_params->ref_enc_op);
2859
2860         /* Set counter to validate the ordering */
2861         for (j = 0; j < num_to_process; ++j)
2862                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2863
2864         for (j = 0; j < TEST_REPETITIONS; ++j) {
2865                 for (i = 0; i < num_to_process; ++i)
2866                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2867
2868                 tp->start_time = rte_rdtsc_precise();
2869                 for (enqueued = 0; enqueued < num_to_process;) {
2870                         num_to_enq = burst_sz;
2871
2872                         if (unlikely(num_to_process - enqueued < num_to_enq))
2873                                 num_to_enq = num_to_process - enqueued;
2874
2875                         enq = 0;
2876                         do {
2877                                 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2878                                                 queue_id, &ops[enqueued],
2879                                                 num_to_enq);
2880                         } while (unlikely(enq != num_to_enq));
2881                         enqueued += enq;
2882
2883                         /* Write to thread burst_sz current number of enqueued
2884                          * descriptors. It ensures that proper number of
2885                          * descriptors will be dequeued in callback
2886                          * function - needed for last batch in case where
2887                          * the number of operations is not a multiple of
2888                          * burst size.
2889                          */
2890                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2891
2892                         /* Wait until processing of previous batch is
2893                          * completed
2894                          */
2895                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2896                                         (int16_t) enqueued)
2897                                 rte_pause();
2898                 }
2899                 if (j != TEST_REPETITIONS - 1)
2900                         rte_atomic16_clear(&tp->nb_dequeued);
2901         }
2902
2903         return TEST_SUCCESS;
2904 }
2905
2906
2907 static int
2908 throughput_intr_lcore_ldpc_enc(void *arg)
2909 {
2910         struct thread_params *tp = arg;
2911         unsigned int enqueued;
2912         const uint16_t queue_id = tp->queue_id;
2913         const uint16_t burst_sz = tp->op_params->burst_sz;
2914         const uint16_t num_to_process = tp->op_params->num_to_process;
2915         struct rte_bbdev_enc_op *ops[num_to_process];
2916         struct test_buffers *bufs = NULL;
2917         struct rte_bbdev_info info;
2918         int ret, i, j;
2919         uint16_t num_to_enq, enq;
2920
2921         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2922                         "BURST_SIZE should be <= %u", MAX_BURST);
2923
2924         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2925                         "Failed to enable interrupts for dev: %u, queue_id: %u",
2926                         tp->dev_id, queue_id);
2927
2928         rte_bbdev_info_get(tp->dev_id, &info);
2929
2930         TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2931                         "NUM_OPS cannot exceed %u for this device",
2932                         info.drv.queue_size_lim);
2933
2934         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2935
2936         rte_atomic16_clear(&tp->processing_status);
2937         rte_atomic16_clear(&tp->nb_dequeued);
2938
2939         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2940                 rte_pause();
2941
2942         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2943                         num_to_process);
2944         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2945                         num_to_process);
2946         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2947                 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
2948                                 bufs->inputs, bufs->hard_outputs,
2949                                 tp->op_params->ref_enc_op);
2950
2951         /* Set counter to validate the ordering */
2952         for (j = 0; j < num_to_process; ++j)
2953                 ops[j]->opaque_data = (void *)(uintptr_t)j;
2954
2955         for (j = 0; j < TEST_REPETITIONS; ++j) {
2956                 for (i = 0; i < num_to_process; ++i)
2957                         rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2958
2959                 tp->start_time = rte_rdtsc_precise();
2960                 for (enqueued = 0; enqueued < num_to_process;) {
2961                         num_to_enq = burst_sz;
2962
2963                         if (unlikely(num_to_process - enqueued < num_to_enq))
2964                                 num_to_enq = num_to_process - enqueued;
2965
2966                         enq = 0;
2967                         do {
2968                                 enq += rte_bbdev_enqueue_ldpc_enc_ops(
2969                                                 tp->dev_id,
2970                                                 queue_id, &ops[enqueued],
2971                                                 num_to_enq);
2972                         } while (unlikely(enq != num_to_enq));
2973                         enqueued += enq;
2974
2975                         /* Write to thread burst_sz current number of enqueued
2976                          * descriptors. It ensures that proper number of
2977                          * descriptors will be dequeued in callback
2978                          * function - needed for last batch in case where
2979                          * the number of operations is not a multiple of
2980                          * burst size.
2981                          */
2982                         rte_atomic16_set(&tp->burst_sz, num_to_enq);
2983
2984                         /* Wait until processing of previous batch is
2985                          * completed
2986                          */
2987                         while (rte_atomic16_read(&tp->nb_dequeued) !=
2988                                         (int16_t) enqueued)
2989                                 rte_pause();
2990                 }
2991                 if (j != TEST_REPETITIONS - 1)
2992                         rte_atomic16_clear(&tp->nb_dequeued);
2993         }
2994
2995         return TEST_SUCCESS;
2996 }
2997
2998 static int
2999 throughput_pmd_lcore_dec(void *arg)
3000 {
3001         struct thread_params *tp = arg;
3002         uint16_t enq, deq;
3003         uint64_t total_time = 0, start_time;
3004         const uint16_t queue_id = tp->queue_id;
3005         const uint16_t burst_sz = tp->op_params->burst_sz;
3006         const uint16_t num_ops = tp->op_params->num_to_process;
3007         struct rte_bbdev_dec_op *ops_enq[num_ops];
3008         struct rte_bbdev_dec_op *ops_deq[num_ops];
3009         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3010         struct test_buffers *bufs = NULL;
3011         int i, j, ret;
3012         struct rte_bbdev_info info;
3013         uint16_t num_to_enq;
3014
3015         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3016                         "BURST_SIZE should be <= %u", MAX_BURST);
3017
3018         rte_bbdev_info_get(tp->dev_id, &info);
3019
3020         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3021                         "NUM_OPS cannot exceed %u for this device",
3022                         info.drv.queue_size_lim);
3023
3024         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3025
3026         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3027                 rte_pause();
3028
3029         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3030         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3031
3032         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3033                 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3034                                 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3035
3036         /* Set counter to validate the ordering */
3037         for (j = 0; j < num_ops; ++j)
3038                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3039
3040         for (i = 0; i < TEST_REPETITIONS; ++i) {
3041
3042                 for (j = 0; j < num_ops; ++j)
3043                         mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3044
3045                 start_time = rte_rdtsc_precise();
3046
3047                 for (enq = 0, deq = 0; enq < num_ops;) {
3048                         num_to_enq = burst_sz;
3049
3050                         if (unlikely(num_ops - enq < num_to_enq))
3051                                 num_to_enq = num_ops - enq;
3052
3053                         enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3054                                         queue_id, &ops_enq[enq], num_to_enq);
3055
3056                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3057                                         queue_id, &ops_deq[deq], enq - deq);
3058                 }
3059
3060                 /* dequeue the remaining */
3061                 while (deq < enq) {
3062                         deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3063                                         queue_id, &ops_deq[deq], enq - deq);
3064                 }
3065
3066                 total_time += rte_rdtsc_precise() - start_time;
3067         }
3068
3069         tp->iter_count = 0;
3070         /* get the max of iter_count for all dequeued ops */
3071         for (i = 0; i < num_ops; ++i) {
3072                 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3073                                 tp->iter_count);
3074         }
3075
3076         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3077                 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3078                                 tp->op_params->vector_mask);
3079                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3080         }
3081
3082         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3083
3084         double tb_len_bits = calc_dec_TB_size(ref_op);
3085
3086         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3087                         ((double)total_time / (double)rte_get_tsc_hz());
3088         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3089                         1000000.0) / ((double)total_time /
3090                         (double)rte_get_tsc_hz());
3091
3092         return TEST_SUCCESS;
3093 }
3094
3095 static int
3096 bler_pmd_lcore_ldpc_dec(void *arg)
3097 {
3098         struct thread_params *tp = arg;
3099         uint16_t enq, deq;
3100         uint64_t total_time = 0, start_time;
3101         const uint16_t queue_id = tp->queue_id;
3102         const uint16_t burst_sz = tp->op_params->burst_sz;
3103         const uint16_t num_ops = tp->op_params->num_to_process;
3104         struct rte_bbdev_dec_op *ops_enq[num_ops];
3105         struct rte_bbdev_dec_op *ops_deq[num_ops];
3106         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3107         struct test_buffers *bufs = NULL;
3108         int i, j, ret;
3109         float parity_bler = 0;
3110         struct rte_bbdev_info info;
3111         uint16_t num_to_enq;
3112         bool extDdr = check_bit(ldpc_cap_flags,
3113                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3114         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3115                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3116         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3117                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3118
3119         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3120                         "BURST_SIZE should be <= %u", MAX_BURST);
3121
3122         rte_bbdev_info_get(tp->dev_id, &info);
3123
3124         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3125                         "NUM_OPS cannot exceed %u for this device",
3126                         info.drv.queue_size_lim);
3127
3128         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3129
3130         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3131                 rte_pause();
3132
3133         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3134         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3135
3136         /* For BLER tests we need to enable early termination */
3137         if (!check_bit(ref_op->ldpc_dec.op_flags,
3138                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3139                 ref_op->ldpc_dec.op_flags +=
3140                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3141         ref_op->ldpc_dec.iter_max = get_iter_max();
3142         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3143
3144         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3145                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3146                                 bufs->hard_outputs, bufs->soft_outputs,
3147                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3148         generate_llr_input(num_ops, bufs->inputs, ref_op);
3149
3150         /* Set counter to validate the ordering */
3151         for (j = 0; j < num_ops; ++j)
3152                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3153
3154         for (i = 0; i < 1; ++i) { /* Could add more iterations */
3155                 for (j = 0; j < num_ops; ++j) {
3156                         if (!loopback)
3157                                 mbuf_reset(
3158                                 ops_enq[j]->ldpc_dec.hard_output.data);
3159                         if (hc_out || loopback)
3160                                 mbuf_reset(
3161                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3162                 }
3163                 if (extDdr) {
3164                         bool preload = i == (TEST_REPETITIONS - 1);
3165                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3166                                         num_ops, preload);
3167                 }
3168                 start_time = rte_rdtsc_precise();
3169
3170                 for (enq = 0, deq = 0; enq < num_ops;) {
3171                         num_to_enq = burst_sz;
3172
3173                         if (unlikely(num_ops - enq < num_to_enq))
3174                                 num_to_enq = num_ops - enq;
3175
3176                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3177                                         queue_id, &ops_enq[enq], num_to_enq);
3178
3179                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3180                                         queue_id, &ops_deq[deq], enq - deq);
3181                 }
3182
3183                 /* dequeue the remaining */
3184                 while (deq < enq) {
3185                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3186                                         queue_id, &ops_deq[deq], enq - deq);
3187                 }
3188
3189                 total_time += rte_rdtsc_precise() - start_time;
3190         }
3191
3192         tp->iter_count = 0;
3193         tp->iter_average = 0;
3194         /* get the max of iter_count for all dequeued ops */
3195         for (i = 0; i < num_ops; ++i) {
3196                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3197                                 tp->iter_count);
3198                 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3199                 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3200                         parity_bler += 1.0;
3201         }
3202
3203         parity_bler /= num_ops; /* This one is based on SYND */
3204         tp->iter_average /= num_ops;
3205         tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3206
3207         if (test_vector.op_type != RTE_BBDEV_OP_NONE
3208                         && tp->bler == 0
3209                         && parity_bler == 0
3210                         && !hc_out) {
3211                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3212                                 tp->op_params->vector_mask);
3213                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3214         }
3215
3216         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3217
3218         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3219         tp->ops_per_sec = ((double)num_ops * 1) /
3220                         ((double)total_time / (double)rte_get_tsc_hz());
3221         tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3222                         1000000.0) / ((double)total_time /
3223                         (double)rte_get_tsc_hz());
3224
3225         return TEST_SUCCESS;
3226 }
3227
3228 static int
3229 throughput_pmd_lcore_ldpc_dec(void *arg)
3230 {
3231         struct thread_params *tp = arg;
3232         uint16_t enq, deq;
3233         uint64_t total_time = 0, start_time;
3234         const uint16_t queue_id = tp->queue_id;
3235         const uint16_t burst_sz = tp->op_params->burst_sz;
3236         const uint16_t num_ops = tp->op_params->num_to_process;
3237         struct rte_bbdev_dec_op *ops_enq[num_ops];
3238         struct rte_bbdev_dec_op *ops_deq[num_ops];
3239         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3240         struct test_buffers *bufs = NULL;
3241         int i, j, ret;
3242         struct rte_bbdev_info info;
3243         uint16_t num_to_enq;
3244         bool extDdr = check_bit(ldpc_cap_flags,
3245                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3246         bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3247                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3248         bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3249                         RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3250
3251         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3252                         "BURST_SIZE should be <= %u", MAX_BURST);
3253
3254         rte_bbdev_info_get(tp->dev_id, &info);
3255
3256         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3257                         "NUM_OPS cannot exceed %u for this device",
3258                         info.drv.queue_size_lim);
3259
3260         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3261
3262         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3263                 rte_pause();
3264
3265         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3266         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3267
3268         /* For throughput tests we need to disable early termination */
3269         if (check_bit(ref_op->ldpc_dec.op_flags,
3270                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3271                 ref_op->ldpc_dec.op_flags -=
3272                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3273         ref_op->ldpc_dec.iter_max = get_iter_max();
3274         ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3275
3276         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3277                 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3278                                 bufs->hard_outputs, bufs->soft_outputs,
3279                                 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3280
3281         /* Set counter to validate the ordering */
3282         for (j = 0; j < num_ops; ++j)
3283                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3284
3285         for (i = 0; i < TEST_REPETITIONS; ++i) {
3286                 for (j = 0; j < num_ops; ++j) {
3287                         if (!loopback)
3288                                 mbuf_reset(
3289                                 ops_enq[j]->ldpc_dec.hard_output.data);
3290                         if (hc_out || loopback)
3291                                 mbuf_reset(
3292                                 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3293                 }
3294                 if (extDdr) {
3295                         bool preload = i == (TEST_REPETITIONS - 1);
3296                         preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3297                                         num_ops, preload);
3298                 }
3299                 start_time = rte_rdtsc_precise();
3300
3301                 for (enq = 0, deq = 0; enq < num_ops;) {
3302                         num_to_enq = burst_sz;
3303
3304                         if (unlikely(num_ops - enq < num_to_enq))
3305                                 num_to_enq = num_ops - enq;
3306
3307                         enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3308                                         queue_id, &ops_enq[enq], num_to_enq);
3309
3310                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3311                                         queue_id, &ops_deq[deq], enq - deq);
3312                 }
3313
3314                 /* dequeue the remaining */
3315                 while (deq < enq) {
3316                         deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3317                                         queue_id, &ops_deq[deq], enq - deq);
3318                 }
3319
3320                 total_time += rte_rdtsc_precise() - start_time;
3321         }
3322
3323         tp->iter_count = 0;
3324         /* get the max of iter_count for all dequeued ops */
3325         for (i = 0; i < num_ops; ++i) {
3326                 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3327                                 tp->iter_count);
3328         }
3329         if (extDdr) {
3330                 /* Read loopback is not thread safe */
3331                 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3332         }
3333
3334         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3335                 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3336                                 tp->op_params->vector_mask);
3337                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3338         }
3339
3340         rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3341
3342         double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3343
3344         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3345                         ((double)total_time / (double)rte_get_tsc_hz());
3346         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3347                         1000000.0) / ((double)total_time /
3348                         (double)rte_get_tsc_hz());
3349
3350         return TEST_SUCCESS;
3351 }
3352
3353 static int
3354 throughput_pmd_lcore_enc(void *arg)
3355 {
3356         struct thread_params *tp = arg;
3357         uint16_t enq, deq;
3358         uint64_t total_time = 0, start_time;
3359         const uint16_t queue_id = tp->queue_id;
3360         const uint16_t burst_sz = tp->op_params->burst_sz;
3361         const uint16_t num_ops = tp->op_params->num_to_process;
3362         struct rte_bbdev_enc_op *ops_enq[num_ops];
3363         struct rte_bbdev_enc_op *ops_deq[num_ops];
3364         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3365         struct test_buffers *bufs = NULL;
3366         int i, j, ret;
3367         struct rte_bbdev_info info;
3368         uint16_t num_to_enq;
3369
3370         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3371                         "BURST_SIZE should be <= %u", MAX_BURST);
3372
3373         rte_bbdev_info_get(tp->dev_id, &info);
3374
3375         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3376                         "NUM_OPS cannot exceed %u for this device",
3377                         info.drv.queue_size_lim);
3378
3379         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3380
3381         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3382                 rte_pause();
3383
3384         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3385                         num_ops);
3386         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3387                         num_ops);
3388         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3389                 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3390                                 bufs->hard_outputs, ref_op);
3391
3392         /* Set counter to validate the ordering */
3393         for (j = 0; j < num_ops; ++j)
3394                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3395
3396         for (i = 0; i < TEST_REPETITIONS; ++i) {
3397
3398                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3399                         for (j = 0; j < num_ops; ++j)
3400                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3401
3402                 start_time = rte_rdtsc_precise();
3403
3404                 for (enq = 0, deq = 0; enq < num_ops;) {
3405                         num_to_enq = burst_sz;
3406
3407                         if (unlikely(num_ops - enq < num_to_enq))
3408                                 num_to_enq = num_ops - enq;
3409
3410                         enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3411                                         queue_id, &ops_enq[enq], num_to_enq);
3412
3413                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3414                                         queue_id, &ops_deq[deq], enq - deq);
3415                 }
3416
3417                 /* dequeue the remaining */
3418                 while (deq < enq) {
3419                         deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3420                                         queue_id, &ops_deq[deq], enq - deq);
3421                 }
3422
3423                 total_time += rte_rdtsc_precise() - start_time;
3424         }
3425
3426         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3427                 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3428                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3429         }
3430
3431         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3432
3433         double tb_len_bits = calc_enc_TB_size(ref_op);
3434
3435         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3436                         ((double)total_time / (double)rte_get_tsc_hz());
3437         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3438                         / 1000000.0) / ((double)total_time /
3439                         (double)rte_get_tsc_hz());
3440
3441         return TEST_SUCCESS;
3442 }
3443
3444 static int
3445 throughput_pmd_lcore_ldpc_enc(void *arg)
3446 {
3447         struct thread_params *tp = arg;
3448         uint16_t enq, deq;
3449         uint64_t total_time = 0, start_time;
3450         const uint16_t queue_id = tp->queue_id;
3451         const uint16_t burst_sz = tp->op_params->burst_sz;
3452         const uint16_t num_ops = tp->op_params->num_to_process;
3453         struct rte_bbdev_enc_op *ops_enq[num_ops];
3454         struct rte_bbdev_enc_op *ops_deq[num_ops];
3455         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3456         struct test_buffers *bufs = NULL;
3457         int i, j, ret;
3458         struct rte_bbdev_info info;
3459         uint16_t num_to_enq;
3460
3461         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3462                         "BURST_SIZE should be <= %u", MAX_BURST);
3463
3464         rte_bbdev_info_get(tp->dev_id, &info);
3465
3466         TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3467                         "NUM_OPS cannot exceed %u for this device",
3468                         info.drv.queue_size_lim);
3469
3470         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3471
3472         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3473                 rte_pause();
3474
3475         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3476                         num_ops);
3477         TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3478                         num_ops);
3479         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3480                 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3481                                 bufs->hard_outputs, ref_op);
3482
3483         /* Set counter to validate the ordering */
3484         for (j = 0; j < num_ops; ++j)
3485                 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3486
3487         for (i = 0; i < TEST_REPETITIONS; ++i) {
3488
3489                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3490                         for (j = 0; j < num_ops; ++j)
3491                                 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3492
3493                 start_time = rte_rdtsc_precise();
3494
3495                 for (enq = 0, deq = 0; enq < num_ops;) {
3496                         num_to_enq = burst_sz;
3497
3498                         if (unlikely(num_ops - enq < num_to_enq))
3499                                 num_to_enq = num_ops - enq;
3500
3501                         enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3502                                         queue_id, &ops_enq[enq], num_to_enq);
3503
3504                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3505                                         queue_id, &ops_deq[deq], enq - deq);
3506                 }
3507
3508                 /* dequeue the remaining */
3509                 while (deq < enq) {
3510                         deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3511                                         queue_id, &ops_deq[deq], enq - deq);
3512                 }
3513
3514                 total_time += rte_rdtsc_precise() - start_time;
3515         }
3516
3517         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3518                 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3519                 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3520         }
3521
3522         rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3523
3524         double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3525
3526         tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3527                         ((double)total_time / (double)rte_get_tsc_hz());
3528         tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3529                         / 1000000.0) / ((double)total_time /
3530                         (double)rte_get_tsc_hz());
3531
3532         return TEST_SUCCESS;
3533 }
3534
3535 static void
3536 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3537 {
3538         unsigned int iter = 0;
3539         double total_mops = 0, total_mbps = 0;
3540
3541         for (iter = 0; iter < used_cores; iter++) {
3542                 printf(
3543                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3544                         t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3545                         t_params[iter].mbps);
3546                 total_mops += t_params[iter].ops_per_sec;
3547                 total_mbps += t_params[iter].mbps;
3548         }
3549         printf(
3550                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3551                 used_cores, total_mops, total_mbps);
3552 }
3553
3554 /* Aggregate the performance results over the number of cores used */
3555 static void
3556 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3557 {
3558         unsigned int core_idx = 0;
3559         double total_mops = 0, total_mbps = 0;
3560         uint8_t iter_count = 0;
3561
3562         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3563                 printf(
3564                         "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3565                         t_params[core_idx].lcore_id,
3566                         t_params[core_idx].ops_per_sec,
3567                         t_params[core_idx].mbps,
3568                         t_params[core_idx].iter_count);
3569                 total_mops += t_params[core_idx].ops_per_sec;
3570                 total_mbps += t_params[core_idx].mbps;
3571                 iter_count = RTE_MAX(iter_count,
3572                                 t_params[core_idx].iter_count);
3573         }
3574         printf(
3575                 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3576                 used_cores, total_mops, total_mbps, iter_count);
3577 }
3578
3579 /* Aggregate the performance results over the number of cores used */
3580 static void
3581 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3582 {
3583         unsigned int core_idx = 0;
3584         double total_mbps = 0, total_bler = 0, total_iter = 0;
3585         double snr = get_snr();
3586
3587         for (core_idx = 0; core_idx < used_cores; core_idx++) {
3588                 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3589                                 t_params[core_idx].lcore_id,
3590                                 t_params[core_idx].bler * 100,
3591                                 t_params[core_idx].iter_average,
3592                                 t_params[core_idx].mbps,
3593                                 get_vector_filename());
3594                 total_mbps += t_params[core_idx].mbps;
3595                 total_bler += t_params[core_idx].bler;
3596                 total_iter += t_params[core_idx].iter_average;
3597         }
3598         total_bler /= used_cores;
3599         total_iter /= used_cores;
3600
3601         printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3602                         snr, total_bler * 100, total_iter, get_iter_max(),
3603                         total_mbps, get_vector_filename());
3604 }
3605
3606 /*
3607  * Test function that determines BLER wireless performance
3608  */
3609 static int
3610 bler_test(struct active_device *ad,
3611                 struct test_op_params *op_params)
3612 {
3613         int ret;
3614         unsigned int lcore_id, used_cores = 0;
3615         struct thread_params *t_params;
3616         struct rte_bbdev_info info;
3617         lcore_function_t *bler_function;
3618         uint16_t num_lcores;
3619         const char *op_type_str;
3620
3621         rte_bbdev_info_get(ad->dev_id, &info);
3622
3623         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3624         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3625                         test_vector.op_type);
3626
3627         printf("+ ------------------------------------------------------- +\n");
3628         printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3629                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3630                         op_params->num_to_process, op_params->num_lcores,
3631                         op_type_str,
3632                         intr_enabled ? "Interrupt mode" : "PMD mode",
3633                         (double)rte_get_tsc_hz() / 1000000000.0);
3634
3635         /* Set number of lcores */
3636         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3637                         ? ad->nb_queues
3638                         : op_params->num_lcores;
3639
3640         /* Allocate memory for thread parameters structure */
3641         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3642                         RTE_CACHE_LINE_SIZE);
3643         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3644                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3645                                 RTE_CACHE_LINE_SIZE));
3646
3647         if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3648                 bler_function = bler_pmd_lcore_ldpc_dec;
3649         else
3650                 return TEST_SKIPPED;
3651
3652         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3653
3654         /* Master core is set at first entry */
3655         t_params[0].dev_id = ad->dev_id;
3656         t_params[0].lcore_id = rte_lcore_id();
3657         t_params[0].op_params = op_params;
3658         t_params[0].queue_id = ad->queue_ids[used_cores++];
3659         t_params[0].iter_count = 0;
3660
3661         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3662                 if (used_cores >= num_lcores)
3663                         break;
3664
3665                 t_params[used_cores].dev_id = ad->dev_id;
3666                 t_params[used_cores].lcore_id = lcore_id;
3667                 t_params[used_cores].op_params = op_params;
3668                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3669                 t_params[used_cores].iter_count = 0;
3670
3671                 rte_eal_remote_launch(bler_function,
3672                                 &t_params[used_cores++], lcore_id);
3673         }
3674
3675         rte_atomic16_set(&op_params->sync, SYNC_START);
3676         ret = bler_function(&t_params[0]);
3677
3678         /* Master core is always used */
3679         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3680                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3681
3682         print_dec_bler(t_params, num_lcores);
3683
3684         /* Return if test failed */
3685         if (ret) {
3686                 rte_free(t_params);
3687                 return ret;
3688         }
3689
3690         /* Function to print something  here*/
3691         rte_free(t_params);
3692         return ret;
3693 }
3694
3695 /*
3696  * Test function that determines how long an enqueue + dequeue of a burst
3697  * takes on available lcores.
3698  */
3699 static int
3700 throughput_test(struct active_device *ad,
3701                 struct test_op_params *op_params)
3702 {
3703         int ret;
3704         unsigned int lcore_id, used_cores = 0;
3705         struct thread_params *t_params, *tp;
3706         struct rte_bbdev_info info;
3707         lcore_function_t *throughput_function;
3708         uint16_t num_lcores;
3709         const char *op_type_str;
3710
3711         rte_bbdev_info_get(ad->dev_id, &info);
3712
3713         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3714         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3715                         test_vector.op_type);
3716
3717         printf("+ ------------------------------------------------------- +\n");
3718         printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3719                         info.dev_name, ad->nb_queues, op_params->burst_sz,
3720                         op_params->num_to_process, op_params->num_lcores,
3721                         op_type_str,
3722                         intr_enabled ? "Interrupt mode" : "PMD mode",
3723                         (double)rte_get_tsc_hz() / 1000000000.0);
3724
3725         /* Set number of lcores */
3726         num_lcores = (ad->nb_queues < (op_params->num_lcores))
3727                         ? ad->nb_queues
3728                         : op_params->num_lcores;
3729
3730         /* Allocate memory for thread parameters structure */
3731         t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3732                         RTE_CACHE_LINE_SIZE);
3733         TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3734                         RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3735                                 RTE_CACHE_LINE_SIZE));
3736
3737         if (intr_enabled) {
3738                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3739                         throughput_function = throughput_intr_lcore_dec;
3740                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3741                         throughput_function = throughput_intr_lcore_ldpc_dec;
3742                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3743                         throughput_function = throughput_intr_lcore_enc;
3744                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3745                         throughput_function = throughput_intr_lcore_ldpc_enc;
3746                 else
3747                         throughput_function = throughput_intr_lcore_enc;
3748
3749                 /* Dequeue interrupt callback registration */
3750                 ret = rte_bbdev_callback_register(ad->dev_id,
3751                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3752                                 t_params);
3753                 if (ret < 0) {
3754                         rte_free(t_params);
3755                         return ret;
3756                 }
3757         } else {
3758                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3759                         throughput_function = throughput_pmd_lcore_dec;
3760                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3761                         throughput_function = throughput_pmd_lcore_ldpc_dec;
3762                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3763                         throughput_function = throughput_pmd_lcore_enc;
3764                 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3765                         throughput_function = throughput_pmd_lcore_ldpc_enc;
3766                 else
3767                         throughput_function = throughput_pmd_lcore_enc;
3768         }
3769
3770         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3771
3772         /* Master core is set at first entry */
3773         t_params[0].dev_id = ad->dev_id;
3774         t_params[0].lcore_id = rte_lcore_id();
3775         t_params[0].op_params = op_params;
3776         t_params[0].queue_id = ad->queue_ids[used_cores++];
3777         t_params[0].iter_count = 0;
3778
3779         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3780                 if (used_cores >= num_lcores)
3781                         break;
3782
3783                 t_params[used_cores].dev_id = ad->dev_id;
3784                 t_params[used_cores].lcore_id = lcore_id;
3785                 t_params[used_cores].op_params = op_params;
3786                 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3787                 t_params[used_cores].iter_count = 0;
3788
3789                 rte_eal_remote_launch(throughput_function,
3790                                 &t_params[used_cores++], lcore_id);
3791         }
3792
3793         rte_atomic16_set(&op_params->sync, SYNC_START);
3794         ret = throughput_function(&t_params[0]);
3795
3796         /* Master core is always used */
3797         for (used_cores = 1; used_cores < num_lcores; used_cores++)
3798                 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3799
3800         /* Return if test failed */
3801         if (ret) {
3802                 rte_free(t_params);
3803                 return ret;
3804         }
3805
3806         /* Print throughput if interrupts are disabled and test passed */
3807         if (!intr_enabled) {
3808                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3809                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3810                         print_dec_throughput(t_params, num_lcores);
3811                 else
3812                         print_enc_throughput(t_params, num_lcores);
3813                 rte_free(t_params);
3814                 return ret;
3815         }
3816
3817         /* In interrupt TC we need to wait for the interrupt callback to deqeue
3818          * all pending operations. Skip waiting for queues which reported an
3819          * error using processing_status variable.
3820          * Wait for master lcore operations.
3821          */
3822         tp = &t_params[0];
3823         while ((rte_atomic16_read(&tp->nb_dequeued) <
3824                         op_params->num_to_process) &&
3825                         (rte_atomic16_read(&tp->processing_status) !=
3826                         TEST_FAILED))
3827                 rte_pause();
3828
3829         tp->ops_per_sec /= TEST_REPETITIONS;
3830         tp->mbps /= TEST_REPETITIONS;
3831         ret |= (int)rte_atomic16_read(&tp->processing_status);
3832
3833         /* Wait for slave lcores operations */
3834         for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3835                 tp = &t_params[used_cores];
3836
3837                 while ((rte_atomic16_read(&tp->nb_dequeued) <
3838                                 op_params->num_to_process) &&
3839                                 (rte_atomic16_read(&tp->processing_status) !=
3840                                 TEST_FAILED))
3841                         rte_pause();
3842
3843                 tp->ops_per_sec /= TEST_REPETITIONS;
3844                 tp->mbps /= TEST_REPETITIONS;
3845                 ret |= (int)rte_atomic16_read(&tp->processing_status);
3846         }
3847
3848         /* Print throughput if test passed */
3849         if (!ret) {
3850                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3851                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3852                         print_dec_throughput(t_params, num_lcores);
3853                 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3854                                 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3855                         print_enc_throughput(t_params, num_lcores);
3856         }
3857
3858         rte_free(t_params);
3859         return ret;
3860 }
3861
3862 static int
3863 latency_test_dec(struct rte_mempool *mempool,
3864                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3865                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3866                 const uint16_t num_to_process, uint16_t burst_sz,
3867                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3868 {
3869         int ret = TEST_SUCCESS;
3870         uint16_t i, j, dequeued;
3871         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3872         uint64_t start_time = 0, last_time = 0;
3873
3874         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3875                 uint16_t enq = 0, deq = 0;
3876                 bool first_time = true;
3877                 last_time = 0;
3878
3879                 if (unlikely(num_to_process - dequeued < burst_sz))
3880                         burst_sz = num_to_process - dequeued;
3881
3882                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3883                 TEST_ASSERT_SUCCESS(ret,
3884                                 "rte_bbdev_dec_op_alloc_bulk() failed");
3885                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3886                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3887                                         bufs->inputs,
3888                                         bufs->hard_outputs,
3889                                         bufs->soft_outputs,
3890                                         ref_op);
3891
3892                 /* Set counter to validate the ordering */
3893                 for (j = 0; j < burst_sz; ++j)
3894                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3895
3896                 start_time = rte_rdtsc_precise();
3897
3898                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
3899                                 burst_sz);
3900                 TEST_ASSERT(enq == burst_sz,
3901                                 "Error enqueueing burst, expected %u, got %u",
3902                                 burst_sz, enq);
3903
3904                 /* Dequeue */
3905                 do {
3906                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3907                                         &ops_deq[deq], burst_sz - deq);
3908                         if (likely(first_time && (deq > 0))) {
3909                                 last_time = rte_rdtsc_precise() - start_time;
3910                                 first_time = false;
3911                         }
3912                 } while (unlikely(burst_sz != deq));
3913
3914                 *max_time = RTE_MAX(*max_time, last_time);
3915                 *min_time = RTE_MIN(*min_time, last_time);
3916                 *total_time += last_time;
3917
3918                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3919                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
3920                                         vector_mask);
3921                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3922                 }
3923
3924                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3925                 dequeued += deq;
3926         }
3927
3928         return i;
3929 }
3930
3931 static int
3932 latency_test_ldpc_dec(struct rte_mempool *mempool,
3933                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3934                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3935                 const uint16_t num_to_process, uint16_t burst_sz,
3936                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3937 {
3938         int ret = TEST_SUCCESS;
3939         uint16_t i, j, dequeued;
3940         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3941         uint64_t start_time = 0, last_time = 0;
3942         bool extDdr = ldpc_cap_flags &
3943                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
3944
3945         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3946                 uint16_t enq = 0, deq = 0;
3947                 bool first_time = true;
3948                 last_time = 0;
3949
3950                 if (unlikely(num_to_process - dequeued < burst_sz))
3951                         burst_sz = num_to_process - dequeued;
3952
3953                 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3954                 TEST_ASSERT_SUCCESS(ret,
3955                                 "rte_bbdev_dec_op_alloc_bulk() failed");
3956
3957                 /* For latency tests we need to disable early termination */
3958                 if (check_bit(ref_op->ldpc_dec.op_flags,
3959                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3960                         ref_op->ldpc_dec.op_flags -=
3961                                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3962                 ref_op->ldpc_dec.iter_max = get_iter_max();
3963                 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3964
3965                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3966                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
3967                                         bufs->inputs,
3968                                         bufs->hard_outputs,
3969                                         bufs->soft_outputs,
3970                                         bufs->harq_inputs,
3971                                         bufs->harq_outputs,
3972                                         ref_op);
3973
3974                 if (extDdr)
3975                         preload_harq_ddr(dev_id, queue_id, ops_enq,
3976                                         burst_sz, true);
3977
3978                 /* Set counter to validate the ordering */
3979                 for (j = 0; j < burst_sz; ++j)
3980                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3981
3982                 start_time = rte_rdtsc_precise();
3983
3984                 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
3985                                 &ops_enq[enq], burst_sz);
3986                 TEST_ASSERT(enq == burst_sz,
3987                                 "Error enqueueing burst, expected %u, got %u",
3988                                 burst_sz, enq);
3989
3990                 /* Dequeue */
3991                 do {
3992                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
3993                                         &ops_deq[deq], burst_sz - deq);
3994                         if (likely(first_time && (deq > 0))) {
3995                                 last_time = rte_rdtsc_precise() - start_time;
3996                                 first_time = false;
3997                         }
3998                 } while (unlikely(burst_sz != deq));
3999
4000                 *max_time = RTE_MAX(*max_time, last_time);
4001                 *min_time = RTE_MIN(*min_time, last_time);
4002                 *total_time += last_time;
4003
4004                 if (extDdr)
4005                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4006
4007                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4008                         ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4009                                         vector_mask);
4010                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4011                 }
4012
4013                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4014                 dequeued += deq;
4015         }
4016         return i;
4017 }
4018
4019 static int
4020 latency_test_enc(struct rte_mempool *mempool,
4021                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4022                 uint16_t dev_id, uint16_t queue_id,
4023                 const uint16_t num_to_process, uint16_t burst_sz,
4024                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4025 {
4026         int ret = TEST_SUCCESS;
4027         uint16_t i, j, dequeued;
4028         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4029         uint64_t start_time = 0, last_time = 0;
4030
4031         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4032                 uint16_t enq = 0, deq = 0;
4033                 bool first_time = true;
4034                 last_time = 0;
4035
4036                 if (unlikely(num_to_process - dequeued < burst_sz))
4037                         burst_sz = num_to_process - dequeued;
4038
4039                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4040                 TEST_ASSERT_SUCCESS(ret,
4041                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4042                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4043                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4044                                         bufs->inputs,
4045                                         bufs->hard_outputs,
4046                                         ref_op);
4047
4048                 /* Set counter to validate the ordering */
4049                 for (j = 0; j < burst_sz; ++j)
4050                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4051
4052                 start_time = rte_rdtsc_precise();
4053
4054                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4055                                 burst_sz);
4056                 TEST_ASSERT(enq == burst_sz,
4057                                 "Error enqueueing burst, expected %u, got %u",
4058                                 burst_sz, enq);
4059
4060                 /* Dequeue */
4061                 do {
4062                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4063                                         &ops_deq[deq], burst_sz - deq);
4064                         if (likely(first_time && (deq > 0))) {
4065                                 last_time += rte_rdtsc_precise() - start_time;
4066                                 first_time = false;
4067                         }
4068                 } while (unlikely(burst_sz != deq));
4069
4070                 *max_time = RTE_MAX(*max_time, last_time);
4071                 *min_time = RTE_MIN(*min_time, last_time);
4072                 *total_time += last_time;
4073
4074                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4075                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4076                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4077                 }
4078
4079                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4080                 dequeued += deq;
4081         }
4082
4083         return i;
4084 }
4085
4086 static int
4087 latency_test_ldpc_enc(struct rte_mempool *mempool,
4088                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4089                 uint16_t dev_id, uint16_t queue_id,
4090                 const uint16_t num_to_process, uint16_t burst_sz,
4091                 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4092 {
4093         int ret = TEST_SUCCESS;
4094         uint16_t i, j, dequeued;
4095         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4096         uint64_t start_time = 0, last_time = 0;
4097
4098         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4099                 uint16_t enq = 0, deq = 0;
4100                 bool first_time = true;
4101                 last_time = 0;
4102
4103                 if (unlikely(num_to_process - dequeued < burst_sz))
4104                         burst_sz = num_to_process - dequeued;
4105
4106                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4107                 TEST_ASSERT_SUCCESS(ret,
4108                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4109                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4110                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4111                                         bufs->inputs,
4112                                         bufs->hard_outputs,
4113                                         ref_op);
4114
4115                 /* Set counter to validate the ordering */
4116                 for (j = 0; j < burst_sz; ++j)
4117                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4118
4119                 start_time = rte_rdtsc_precise();
4120
4121                 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4122                                 &ops_enq[enq], burst_sz);
4123                 TEST_ASSERT(enq == burst_sz,
4124                                 "Error enqueueing burst, expected %u, got %u",
4125                                 burst_sz, enq);
4126
4127                 /* Dequeue */
4128                 do {
4129                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4130                                         &ops_deq[deq], burst_sz - deq);
4131                         if (likely(first_time && (deq > 0))) {
4132                                 last_time += rte_rdtsc_precise() - start_time;
4133                                 first_time = false;
4134                         }
4135                 } while (unlikely(burst_sz != deq));
4136
4137                 *max_time = RTE_MAX(*max_time, last_time);
4138                 *min_time = RTE_MIN(*min_time, last_time);
4139                 *total_time += last_time;
4140
4141                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4142                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4143                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4144                 }
4145
4146                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4147                 dequeued += deq;
4148         }
4149
4150         return i;
4151 }
4152
4153 static int
4154 latency_test(struct active_device *ad,
4155                 struct test_op_params *op_params)
4156 {
4157         int iter;
4158         uint16_t burst_sz = op_params->burst_sz;
4159         const uint16_t num_to_process = op_params->num_to_process;
4160         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4161         const uint16_t queue_id = ad->queue_ids[0];
4162         struct test_buffers *bufs = NULL;
4163         struct rte_bbdev_info info;
4164         uint64_t total_time, min_time, max_time;
4165         const char *op_type_str;
4166
4167         total_time = max_time = 0;
4168         min_time = UINT64_MAX;
4169
4170         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4171                         "BURST_SIZE should be <= %u", MAX_BURST);
4172
4173         rte_bbdev_info_get(ad->dev_id, &info);
4174         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4175
4176         op_type_str = rte_bbdev_op_type_str(op_type);
4177         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4178
4179         printf("+ ------------------------------------------------------- +\n");
4180         printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4181                         info.dev_name, burst_sz, num_to_process, op_type_str);
4182
4183         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4184                 iter = latency_test_dec(op_params->mp, bufs,
4185                                 op_params->ref_dec_op, op_params->vector_mask,
4186                                 ad->dev_id, queue_id, num_to_process,
4187                                 burst_sz, &total_time, &min_time, &max_time);
4188         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4189                 iter = latency_test_enc(op_params->mp, bufs,
4190                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4191                                 num_to_process, burst_sz, &total_time,
4192                                 &min_time, &max_time);
4193         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4194                 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4195                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4196                                 num_to_process, burst_sz, &total_time,
4197                                 &min_time, &max_time);
4198         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4199                 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4200                                 op_params->ref_dec_op, op_params->vector_mask,
4201                                 ad->dev_id, queue_id, num_to_process,
4202                                 burst_sz, &total_time, &min_time, &max_time);
4203         else
4204                 iter = latency_test_enc(op_params->mp, bufs,
4205                                         op_params->ref_enc_op,
4206                                         ad->dev_id, queue_id,
4207                                         num_to_process, burst_sz, &total_time,
4208                                         &min_time, &max_time);
4209
4210         if (iter <= 0)
4211                 return TEST_FAILED;
4212
4213         printf("Operation latency:\n"
4214                         "\tavg: %lg cycles, %lg us\n"
4215                         "\tmin: %lg cycles, %lg us\n"
4216                         "\tmax: %lg cycles, %lg us\n",
4217                         (double)total_time / (double)iter,
4218                         (double)(total_time * 1000000) / (double)iter /
4219                         (double)rte_get_tsc_hz(), (double)min_time,
4220                         (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4221                         (double)max_time, (double)(max_time * 1000000) /
4222                         (double)rte_get_tsc_hz());
4223
4224         return TEST_SUCCESS;
4225 }
4226
4227 #ifdef RTE_BBDEV_OFFLOAD_COST
4228 static int
4229 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4230                 struct rte_bbdev_stats *stats)
4231 {
4232         struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4233         struct rte_bbdev_stats *q_stats;
4234
4235         if (queue_id >= dev->data->num_queues)
4236                 return -1;
4237
4238         q_stats = &dev->data->queues[queue_id].queue_stats;
4239
4240         stats->enqueued_count = q_stats->enqueued_count;
4241         stats->dequeued_count = q_stats->dequeued_count;
4242         stats->enqueue_err_count = q_stats->enqueue_err_count;
4243         stats->dequeue_err_count = q_stats->dequeue_err_count;
4244         stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4245
4246         return 0;
4247 }
4248
4249 static int
4250 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4251                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4252                 uint16_t queue_id, const uint16_t num_to_process,
4253                 uint16_t burst_sz, struct test_time_stats *time_st)
4254 {
4255         int i, dequeued, ret;
4256         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4257         uint64_t enq_start_time, deq_start_time;
4258         uint64_t enq_sw_last_time, deq_last_time;
4259         struct rte_bbdev_stats stats;
4260
4261         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4262                 uint16_t enq = 0, deq = 0;
4263
4264                 if (unlikely(num_to_process - dequeued < burst_sz))
4265                         burst_sz = num_to_process - dequeued;
4266
4267                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4268                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4269                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4270                                         bufs->inputs,
4271                                         bufs->hard_outputs,
4272                                         bufs->soft_outputs,
4273                                         ref_op);
4274
4275                 /* Start time meas for enqueue function offload latency */
4276                 enq_start_time = rte_rdtsc_precise();
4277                 do {
4278                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4279                                         &ops_enq[enq], burst_sz - enq);
4280                 } while (unlikely(burst_sz != enq));
4281
4282                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4283                 TEST_ASSERT_SUCCESS(ret,
4284                                 "Failed to get stats for queue (%u) of device (%u)",
4285                                 queue_id, dev_id);
4286
4287                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4288                                 stats.acc_offload_cycles;
4289                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4290                                 enq_sw_last_time);
4291                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4292                                 enq_sw_last_time);
4293                 time_st->enq_sw_total_time += enq_sw_last_time;
4294
4295                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4296                                 stats.acc_offload_cycles);
4297                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4298                                 stats.acc_offload_cycles);
4299                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4300
4301                 /* give time for device to process ops */
4302                 rte_delay_us(200);
4303
4304                 /* Start time meas for dequeue function offload latency */
4305                 deq_start_time = rte_rdtsc_precise();
4306                 /* Dequeue one operation */
4307                 do {
4308                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4309                                         &ops_deq[deq], 1);
4310                 } while (unlikely(deq != 1));
4311
4312                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4313                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4314                                 deq_last_time);
4315                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4316                                 deq_last_time);
4317                 time_st->deq_total_time += deq_last_time;
4318
4319                 /* Dequeue remaining operations if needed*/
4320                 while (burst_sz != deq)
4321                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4322                                         &ops_deq[deq], burst_sz - deq);
4323
4324                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4325                 dequeued += deq;
4326         }
4327
4328         return i;
4329 }
4330
4331 static int
4332 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4333                 struct test_buffers *bufs,
4334                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4335                 uint16_t queue_id, const uint16_t num_to_process,
4336                 uint16_t burst_sz, struct test_time_stats *time_st)
4337 {
4338         int i, dequeued, ret;
4339         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4340         uint64_t enq_start_time, deq_start_time;
4341         uint64_t enq_sw_last_time, deq_last_time;
4342         struct rte_bbdev_stats stats;
4343         bool extDdr = ldpc_cap_flags &
4344                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4345
4346         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4347                 uint16_t enq = 0, deq = 0;
4348
4349                 if (unlikely(num_to_process - dequeued < burst_sz))
4350                         burst_sz = num_to_process - dequeued;
4351
4352                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4353                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4354                         copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4355                                         bufs->inputs,
4356                                         bufs->hard_outputs,
4357                                         bufs->soft_outputs,
4358                                         bufs->harq_inputs,
4359                                         bufs->harq_outputs,
4360                                         ref_op);
4361
4362                 if (extDdr)
4363                         preload_harq_ddr(dev_id, queue_id, ops_enq,
4364                                         burst_sz, true);
4365
4366                 /* Start time meas for enqueue function offload latency */
4367                 enq_start_time = rte_rdtsc_precise();
4368                 do {
4369                         enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4370                                         &ops_enq[enq], burst_sz - enq);
4371                 } while (unlikely(burst_sz != enq));
4372
4373                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4374                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4375                 TEST_ASSERT_SUCCESS(ret,
4376                                 "Failed to get stats for queue (%u) of device (%u)",
4377                                 queue_id, dev_id);
4378
4379                 enq_sw_last_time -= stats.acc_offload_cycles;
4380                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4381                                 enq_sw_last_time);
4382                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4383                                 enq_sw_last_time);
4384                 time_st->enq_sw_total_time += enq_sw_last_time;
4385
4386                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4387                                 stats.acc_offload_cycles);
4388                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4389                                 stats.acc_offload_cycles);
4390                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4391
4392                 /* give time for device to process ops */
4393                 rte_delay_us(200);
4394
4395                 /* Start time meas for dequeue function offload latency */
4396                 deq_start_time = rte_rdtsc_precise();
4397                 /* Dequeue one operation */
4398                 do {
4399                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4400                                         &ops_deq[deq], 1);
4401                 } while (unlikely(deq != 1));
4402
4403                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4404                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4405                                 deq_last_time);
4406                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4407                                 deq_last_time);
4408                 time_st->deq_total_time += deq_last_time;
4409
4410                 /* Dequeue remaining operations if needed*/
4411                 while (burst_sz != deq)
4412                         deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4413                                         &ops_deq[deq], burst_sz - deq);
4414
4415                 if (extDdr) {
4416                         /* Read loopback is not thread safe */
4417                         retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4418                 }
4419
4420                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4421                 dequeued += deq;
4422         }
4423
4424         return i;
4425 }
4426
4427 static int
4428 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4429                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4430                 uint16_t queue_id, const uint16_t num_to_process,
4431                 uint16_t burst_sz, struct test_time_stats *time_st)
4432 {
4433         int i, dequeued, ret;
4434         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4435         uint64_t enq_start_time, deq_start_time;
4436         uint64_t enq_sw_last_time, deq_last_time;
4437         struct rte_bbdev_stats stats;
4438
4439         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4440                 uint16_t enq = 0, deq = 0;
4441
4442                 if (unlikely(num_to_process - dequeued < burst_sz))
4443                         burst_sz = num_to_process - dequeued;
4444
4445                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4446                 TEST_ASSERT_SUCCESS(ret,
4447                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4448                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4449                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4450                                         bufs->inputs,
4451                                         bufs->hard_outputs,
4452                                         ref_op);
4453
4454                 /* Start time meas for enqueue function offload latency */
4455                 enq_start_time = rte_rdtsc_precise();
4456                 do {
4457                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4458                                         &ops_enq[enq], burst_sz - enq);
4459                 } while (unlikely(burst_sz != enq));
4460
4461                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4462
4463                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4464                 TEST_ASSERT_SUCCESS(ret,
4465                                 "Failed to get stats for queue (%u) of device (%u)",
4466                                 queue_id, dev_id);
4467                 enq_sw_last_time -= stats.acc_offload_cycles;
4468                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4469                                 enq_sw_last_time);
4470                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4471                                 enq_sw_last_time);
4472                 time_st->enq_sw_total_time += enq_sw_last_time;
4473
4474                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4475                                 stats.acc_offload_cycles);
4476                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4477                                 stats.acc_offload_cycles);
4478                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4479
4480                 /* give time for device to process ops */
4481                 rte_delay_us(200);
4482
4483                 /* Start time meas for dequeue function offload latency */
4484                 deq_start_time = rte_rdtsc_precise();
4485                 /* Dequeue one operation */
4486                 do {
4487                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4488                                         &ops_deq[deq], 1);
4489                 } while (unlikely(deq != 1));
4490
4491                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4492                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4493                                 deq_last_time);
4494                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4495                                 deq_last_time);
4496                 time_st->deq_total_time += deq_last_time;
4497
4498                 while (burst_sz != deq)
4499                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4500                                         &ops_deq[deq], burst_sz - deq);
4501
4502                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4503                 dequeued += deq;
4504         }
4505
4506         return i;
4507 }
4508
4509 static int
4510 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4511                 struct test_buffers *bufs,
4512                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4513                 uint16_t queue_id, const uint16_t num_to_process,
4514                 uint16_t burst_sz, struct test_time_stats *time_st)
4515 {
4516         int i, dequeued, ret;
4517         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4518         uint64_t enq_start_time, deq_start_time;
4519         uint64_t enq_sw_last_time, deq_last_time;
4520         struct rte_bbdev_stats stats;
4521
4522         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4523                 uint16_t enq = 0, deq = 0;
4524
4525                 if (unlikely(num_to_process - dequeued < burst_sz))
4526                         burst_sz = num_to_process - dequeued;
4527
4528                 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4529                 TEST_ASSERT_SUCCESS(ret,
4530                                 "rte_bbdev_enc_op_alloc_bulk() failed");
4531                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4532                         copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4533                                         bufs->inputs,
4534                                         bufs->hard_outputs,
4535                                         ref_op);
4536
4537                 /* Start time meas for enqueue function offload latency */
4538                 enq_start_time = rte_rdtsc_precise();
4539                 do {
4540                         enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4541                                         &ops_enq[enq], burst_sz - enq);
4542                 } while (unlikely(burst_sz != enq));
4543
4544                 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4545                 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4546                 TEST_ASSERT_SUCCESS(ret,
4547                                 "Failed to get stats for queue (%u) of device (%u)",
4548                                 queue_id, dev_id);
4549
4550                 enq_sw_last_time -= stats.acc_offload_cycles;
4551                 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4552                                 enq_sw_last_time);
4553                 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4554                                 enq_sw_last_time);
4555                 time_st->enq_sw_total_time += enq_sw_last_time;
4556
4557                 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4558                                 stats.acc_offload_cycles);
4559                 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4560                                 stats.acc_offload_cycles);
4561                 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4562
4563                 /* give time for device to process ops */
4564                 rte_delay_us(200);
4565
4566                 /* Start time meas for dequeue function offload latency */
4567                 deq_start_time = rte_rdtsc_precise();
4568                 /* Dequeue one operation */
4569                 do {
4570                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4571                                         &ops_deq[deq], 1);
4572                 } while (unlikely(deq != 1));
4573
4574                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4575                 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4576                                 deq_last_time);
4577                 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4578                                 deq_last_time);
4579                 time_st->deq_total_time += deq_last_time;
4580
4581                 while (burst_sz != deq)
4582                         deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4583                                         &ops_deq[deq], burst_sz - deq);
4584
4585                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4586                 dequeued += deq;
4587         }
4588
4589         return i;
4590 }
4591 #endif
4592
4593 static int
4594 offload_cost_test(struct active_device *ad,
4595                 struct test_op_params *op_params)
4596 {
4597 #ifndef RTE_BBDEV_OFFLOAD_COST
4598         RTE_SET_USED(ad);
4599         RTE_SET_USED(op_params);
4600         printf("Offload latency test is disabled.\n");
4601         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4602         return TEST_SKIPPED;
4603 #else
4604         int iter;
4605         uint16_t burst_sz = op_params->burst_sz;
4606         const uint16_t num_to_process = op_params->num_to_process;
4607         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4608         const uint16_t queue_id = ad->queue_ids[0];
4609         struct test_buffers *bufs = NULL;
4610         struct rte_bbdev_info info;
4611         const char *op_type_str;
4612         struct test_time_stats time_st;
4613
4614         memset(&time_st, 0, sizeof(struct test_time_stats));
4615         time_st.enq_sw_min_time = UINT64_MAX;
4616         time_st.enq_acc_min_time = UINT64_MAX;
4617         time_st.deq_min_time = UINT64_MAX;
4618
4619         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4620                         "BURST_SIZE should be <= %u", MAX_BURST);
4621
4622         rte_bbdev_info_get(ad->dev_id, &info);
4623         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4624
4625         op_type_str = rte_bbdev_op_type_str(op_type);
4626         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4627
4628         printf("+ ------------------------------------------------------- +\n");
4629         printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4630                         info.dev_name, burst_sz, num_to_process, op_type_str);
4631
4632         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4633                 iter = offload_latency_test_dec(op_params->mp, bufs,
4634                                 op_params->ref_dec_op, ad->dev_id, queue_id,
4635                                 num_to_process, burst_sz, &time_st);
4636         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4637                 iter = offload_latency_test_enc(op_params->mp, bufs,
4638                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4639                                 num_to_process, burst_sz, &time_st);
4640         else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4641                 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4642                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4643                                 num_to_process, burst_sz, &time_st);
4644         else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4645                 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4646                         op_params->ref_dec_op, ad->dev_id, queue_id,
4647                         num_to_process, burst_sz, &time_st);
4648         else
4649                 iter = offload_latency_test_enc(op_params->mp, bufs,
4650                                 op_params->ref_enc_op, ad->dev_id, queue_id,
4651                                 num_to_process, burst_sz, &time_st);
4652
4653         if (iter <= 0)
4654                 return TEST_FAILED;
4655
4656         printf("Enqueue driver offload cost latency:\n"
4657                         "\tavg: %lg cycles, %lg us\n"
4658                         "\tmin: %lg cycles, %lg us\n"
4659                         "\tmax: %lg cycles, %lg us\n"
4660                         "Enqueue accelerator offload cost latency:\n"
4661                         "\tavg: %lg cycles, %lg us\n"
4662                         "\tmin: %lg cycles, %lg us\n"
4663                         "\tmax: %lg cycles, %lg us\n",
4664                         (double)time_st.enq_sw_total_time / (double)iter,
4665                         (double)(time_st.enq_sw_total_time * 1000000) /
4666                         (double)iter / (double)rte_get_tsc_hz(),
4667                         (double)time_st.enq_sw_min_time,
4668                         (double)(time_st.enq_sw_min_time * 1000000) /
4669                         rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4670                         (double)(time_st.enq_sw_max_time * 1000000) /
4671                         rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4672                         (double)iter,
4673                         (double)(time_st.enq_acc_total_time * 1000000) /
4674                         (double)iter / (double)rte_get_tsc_hz(),
4675                         (double)time_st.enq_acc_min_time,
4676                         (double)(time_st.enq_acc_min_time * 1000000) /
4677                         rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4678                         (double)(time_st.enq_acc_max_time * 1000000) /
4679                         rte_get_tsc_hz());
4680
4681         printf("Dequeue offload cost latency - one op:\n"
4682                         "\tavg: %lg cycles, %lg us\n"
4683                         "\tmin: %lg cycles, %lg us\n"
4684                         "\tmax: %lg cycles, %lg us\n",
4685                         (double)time_st.deq_total_time / (double)iter,
4686                         (double)(time_st.deq_total_time * 1000000) /
4687                         (double)iter / (double)rte_get_tsc_hz(),
4688                         (double)time_st.deq_min_time,
4689                         (double)(time_st.deq_min_time * 1000000) /
4690                         rte_get_tsc_hz(), (double)time_st.deq_max_time,
4691                         (double)(time_st.deq_max_time * 1000000) /
4692                         rte_get_tsc_hz());
4693
4694         return TEST_SUCCESS;
4695 #endif
4696 }
4697
4698 #ifdef RTE_BBDEV_OFFLOAD_COST
4699 static int
4700 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4701                 const uint16_t num_to_process, uint16_t burst_sz,
4702                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4703                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4704 {
4705         int i, deq_total;
4706         struct rte_bbdev_dec_op *ops[MAX_BURST];
4707         uint64_t deq_start_time, deq_last_time;
4708
4709         /* Test deq offload latency from an empty queue */
4710
4711         for (i = 0, deq_total = 0; deq_total < num_to_process;
4712                         ++i, deq_total += burst_sz) {
4713                 deq_start_time = rte_rdtsc_precise();
4714
4715                 if (unlikely(num_to_process - deq_total < burst_sz))
4716                         burst_sz = num_to_process - deq_total;
4717                 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4718                         rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4719                                         burst_sz);
4720                 else
4721                         rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4722                                         burst_sz);
4723
4724                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4725                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4726                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4727                 *deq_total_time += deq_last_time;
4728         }
4729
4730         return i;
4731 }
4732
4733 static int
4734 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4735                 const uint16_t num_to_process, uint16_t burst_sz,
4736                 uint64_t *deq_total_time, uint64_t *deq_min_time,
4737                 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4738 {
4739         int i, deq_total;
4740         struct rte_bbdev_enc_op *ops[MAX_BURST];
4741         uint64_t deq_start_time, deq_last_time;
4742
4743         /* Test deq offload latency from an empty queue */
4744         for (i = 0, deq_total = 0; deq_total < num_to_process;
4745                         ++i, deq_total += burst_sz) {
4746                 deq_start_time = rte_rdtsc_precise();
4747
4748                 if (unlikely(num_to_process - deq_total < burst_sz))
4749                         burst_sz = num_to_process - deq_total;
4750                 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4751                         rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4752                                         burst_sz);
4753                 else
4754                         rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4755                                         burst_sz);
4756
4757                 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4758                 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4759                 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4760                 *deq_total_time += deq_last_time;
4761         }
4762
4763         return i;
4764 }
4765
4766 #endif
4767
4768 static int
4769 offload_latency_empty_q_test(struct active_device *ad,
4770                 struct test_op_params *op_params)
4771 {
4772 #ifndef RTE_BBDEV_OFFLOAD_COST
4773         RTE_SET_USED(ad);
4774         RTE_SET_USED(op_params);
4775         printf("Offload latency empty dequeue test is disabled.\n");
4776         printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4777         return TEST_SKIPPED;
4778 #else
4779         int iter;
4780         uint64_t deq_total_time, deq_min_time, deq_max_time;
4781         uint16_t burst_sz = op_params->burst_sz;
4782         const uint16_t num_to_process = op_params->num_to_process;
4783         const enum rte_bbdev_op_type op_type = test_vector.op_type;
4784         const uint16_t queue_id = ad->queue_ids[0];
4785         struct rte_bbdev_info info;
4786         const char *op_type_str;
4787
4788         deq_total_time = deq_max_time = 0;
4789         deq_min_time = UINT64_MAX;
4790
4791         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4792                         "BURST_SIZE should be <= %u", MAX_BURST);
4793
4794         rte_bbdev_info_get(ad->dev_id, &info);
4795
4796         op_type_str = rte_bbdev_op_type_str(op_type);
4797         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4798
4799         printf("+ ------------------------------------------------------- +\n");
4800         printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4801                         info.dev_name, burst_sz, num_to_process, op_type_str);
4802
4803         if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4804                         op_type == RTE_BBDEV_OP_LDPC_DEC)
4805                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4806                                 num_to_process, burst_sz, &deq_total_time,
4807                                 &deq_min_time, &deq_max_time, op_type);
4808         else
4809                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4810                                 num_to_process, burst_sz, &deq_total_time,
4811                                 &deq_min_time, &deq_max_time, op_type);
4812
4813         if (iter <= 0)
4814                 return TEST_FAILED;
4815
4816         printf("Empty dequeue offload:\n"
4817                         "\tavg: %lg cycles, %lg us\n"
4818                         "\tmin: %lg cycles, %lg us\n"
4819                         "\tmax: %lg cycles, %lg us\n",
4820                         (double)deq_total_time / (double)iter,
4821                         (double)(deq_total_time * 1000000) / (double)iter /
4822                         (double)rte_get_tsc_hz(), (double)deq_min_time,
4823                         (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4824                         (double)deq_max_time, (double)(deq_max_time * 1000000) /
4825                         rte_get_tsc_hz());
4826
4827         return TEST_SUCCESS;
4828 #endif
4829 }
4830
4831 static int
4832 bler_tc(void)
4833 {
4834         return run_test_case(bler_test);
4835 }
4836
4837 static int
4838 throughput_tc(void)
4839 {
4840         return run_test_case(throughput_test);
4841 }
4842
4843 static int
4844 offload_cost_tc(void)
4845 {
4846         return run_test_case(offload_cost_test);
4847 }
4848
4849 static int
4850 offload_latency_empty_q_tc(void)
4851 {
4852         return run_test_case(offload_latency_empty_q_test);
4853 }
4854
4855 static int
4856 latency_tc(void)
4857 {
4858         return run_test_case(latency_test);
4859 }
4860
4861 static int
4862 interrupt_tc(void)
4863 {
4864         return run_test_case(throughput_test);
4865 }
4866
4867 static struct unit_test_suite bbdev_bler_testsuite = {
4868         .suite_name = "BBdev BLER Tests",
4869         .setup = testsuite_setup,
4870         .teardown = testsuite_teardown,
4871         .unit_test_cases = {
4872                 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
4873                 TEST_CASES_END() /**< NULL terminate unit test array */
4874         }
4875 };
4876
4877 static struct unit_test_suite bbdev_throughput_testsuite = {
4878         .suite_name = "BBdev Throughput Tests",
4879         .setup = testsuite_setup,
4880         .teardown = testsuite_teardown,
4881         .unit_test_cases = {
4882                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
4883                 TEST_CASES_END() /**< NULL terminate unit test array */
4884         }
4885 };
4886
4887 static struct unit_test_suite bbdev_validation_testsuite = {
4888         .suite_name = "BBdev Validation Tests",
4889         .setup = testsuite_setup,
4890         .teardown = testsuite_teardown,
4891         .unit_test_cases = {
4892                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4893                 TEST_CASES_END() /**< NULL terminate unit test array */
4894         }
4895 };
4896
4897 static struct unit_test_suite bbdev_latency_testsuite = {
4898         .suite_name = "BBdev Latency Tests",
4899         .setup = testsuite_setup,
4900         .teardown = testsuite_teardown,
4901         .unit_test_cases = {
4902                 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4903                 TEST_CASES_END() /**< NULL terminate unit test array */
4904         }
4905 };
4906
4907 static struct unit_test_suite bbdev_offload_cost_testsuite = {
4908         .suite_name = "BBdev Offload Cost Tests",
4909         .setup = testsuite_setup,
4910         .teardown = testsuite_teardown,
4911         .unit_test_cases = {
4912                 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
4913                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
4914                 TEST_CASES_END() /**< NULL terminate unit test array */
4915         }
4916 };
4917
4918 static struct unit_test_suite bbdev_interrupt_testsuite = {
4919         .suite_name = "BBdev Interrupt Tests",
4920         .setup = interrupt_testsuite_setup,
4921         .teardown = testsuite_teardown,
4922         .unit_test_cases = {
4923                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
4924                 TEST_CASES_END() /**< NULL terminate unit test array */
4925         }
4926 };
4927
4928 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
4929 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
4930 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
4931 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
4932 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
4933 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);