1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
19 #include <rte_interrupts.h>
22 #include "test_bbdev_vector.h"
24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
26 #define MAX_QUEUES RTE_MAX_LCORE
27 #define TEST_REPETITIONS 1000
29 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
30 #include <fpga_lte_fec.h>
31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
33 #define VF_UL_4G_QUEUE_VALUE 4
34 #define VF_DL_4G_QUEUE_VALUE 4
35 #define UL_4G_BANDWIDTH 3
36 #define DL_4G_BANDWIDTH 3
37 #define UL_4G_LOAD_BALANCE 128
38 #define DL_4G_LOAD_BALANCE 128
39 #define FLR_4G_TIMEOUT 610
42 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
43 #include <rte_pmd_fpga_5gnr_fec.h>
44 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
45 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
46 #define VF_UL_5G_QUEUE_VALUE 4
47 #define VF_DL_5G_QUEUE_VALUE 4
48 #define UL_5G_BANDWIDTH 3
49 #define DL_5G_BANDWIDTH 3
50 #define UL_5G_LOAD_BALANCE 128
51 #define DL_5G_LOAD_BALANCE 128
52 #define FLR_5G_TIMEOUT 610
55 #ifdef RTE_BASEBAND_ACC100
56 #include <rte_acc100_cfg.h>
57 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf")
58 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf")
59 #define ACC100_QMGR_NUM_AQS 16
60 #define ACC100_QMGR_NUM_QGS 2
61 #define ACC100_QMGR_AQ_DEPTH 5
62 #define ACC100_QMGR_INVALID_IDX -1
63 #define ACC100_QMGR_RR 1
64 #define ACC100_QOS_GBR 0
67 #define OPS_CACHE_SIZE 256U
68 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
72 #define INVALID_OPAQUE -1
74 #define INVALID_QUEUE_ID -1
75 /* Increment for next code block in external HARQ memory */
76 #define HARQ_INCR 32768
77 /* Headroom for filler LLRs insertion in HARQ buffer */
78 #define FILLER_HEADROOM 1024
79 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
80 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
81 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
82 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
83 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
84 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
85 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
86 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
87 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
89 static struct test_bbdev_vector test_vector;
91 /* Switch between PMD and Interrupt for throughput TC */
92 static bool intr_enabled;
94 /* LLR arithmetic representation for numerical conversion */
95 static int ldpc_llr_decimals;
96 static int ldpc_llr_size;
97 /* Keep track of the LDPC decoder device capability flag */
98 static uint32_t ldpc_cap_flags;
100 /* Represents tested active devices */
101 static struct active_device {
102 const char *driver_name;
104 uint16_t supported_ops;
105 uint16_t queue_ids[MAX_QUEUES];
107 struct rte_mempool *ops_mempool;
108 struct rte_mempool *in_mbuf_pool;
109 struct rte_mempool *hard_out_mbuf_pool;
110 struct rte_mempool *soft_out_mbuf_pool;
111 struct rte_mempool *harq_in_mbuf_pool;
112 struct rte_mempool *harq_out_mbuf_pool;
113 } active_devs[RTE_BBDEV_MAX_DEVS];
115 static uint8_t nb_active_devs;
117 /* Data buffers used by BBDEV ops */
118 struct test_buffers {
119 struct rte_bbdev_op_data *inputs;
120 struct rte_bbdev_op_data *hard_outputs;
121 struct rte_bbdev_op_data *soft_outputs;
122 struct rte_bbdev_op_data *harq_inputs;
123 struct rte_bbdev_op_data *harq_outputs;
126 /* Operation parameters specific for given test case */
127 struct test_op_params {
128 struct rte_mempool *mp;
129 struct rte_bbdev_dec_op *ref_dec_op;
130 struct rte_bbdev_enc_op *ref_enc_op;
132 uint16_t num_to_process;
136 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
139 /* Contains per lcore params */
140 struct thread_params {
150 rte_atomic16_t nb_dequeued;
151 rte_atomic16_t processing_status;
152 rte_atomic16_t burst_sz;
153 struct test_op_params *op_params;
154 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
155 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
158 #ifdef RTE_BBDEV_OFFLOAD_COST
159 /* Stores time statistics */
160 struct test_time_stats {
161 /* Stores software enqueue total working time */
162 uint64_t enq_sw_total_time;
163 /* Stores minimum value of software enqueue working time */
164 uint64_t enq_sw_min_time;
165 /* Stores maximum value of software enqueue working time */
166 uint64_t enq_sw_max_time;
167 /* Stores turbo enqueue total working time */
168 uint64_t enq_acc_total_time;
169 /* Stores minimum value of accelerator enqueue working time */
170 uint64_t enq_acc_min_time;
171 /* Stores maximum value of accelerator enqueue working time */
172 uint64_t enq_acc_max_time;
173 /* Stores dequeue total working time */
174 uint64_t deq_total_time;
175 /* Stores minimum value of dequeue working time */
176 uint64_t deq_min_time;
177 /* Stores maximum value of dequeue working time */
178 uint64_t deq_max_time;
182 typedef int (test_case_function)(struct active_device *ad,
183 struct test_op_params *op_params);
186 mbuf_reset(struct rte_mbuf *m)
196 /* Read flag value 0/1 from bitmap */
198 check_bit(uint32_t bitmap, uint32_t bitmask)
200 return bitmap & bitmask;
204 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
206 ad->supported_ops |= (1 << op_type);
210 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
212 return ad->supported_ops & (1 << op_type);
216 flags_match(uint32_t flags_req, uint32_t flags_present)
218 return (flags_req & flags_present) == flags_req;
222 clear_soft_out_cap(uint32_t *op_flags)
224 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
225 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
226 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
230 check_dev_cap(const struct rte_bbdev_info *dev_info)
233 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
234 nb_harq_inputs, nb_harq_outputs;
235 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
237 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
238 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
239 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
240 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
241 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
243 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
244 if (op_cap->type != test_vector.op_type)
247 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
248 const struct rte_bbdev_op_cap_turbo_dec *cap =
249 &op_cap->cap.turbo_dec;
250 /* Ignore lack of soft output capability, just skip
251 * checking if soft output is valid.
253 if ((test_vector.turbo_dec.op_flags &
254 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
255 !(cap->capability_flags &
256 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
258 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
261 &test_vector.turbo_dec.op_flags);
264 if (!flags_match(test_vector.turbo_dec.op_flags,
265 cap->capability_flags))
267 if (nb_inputs > cap->num_buffers_src) {
268 printf("Too many inputs defined: %u, max: %u\n",
269 nb_inputs, cap->num_buffers_src);
272 if (nb_soft_outputs > cap->num_buffers_soft_out &&
273 (test_vector.turbo_dec.op_flags &
274 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
276 "Too many soft outputs defined: %u, max: %u\n",
278 cap->num_buffers_soft_out);
281 if (nb_hard_outputs > cap->num_buffers_hard_out) {
283 "Too many hard outputs defined: %u, max: %u\n",
285 cap->num_buffers_hard_out);
288 if (intr_enabled && !(cap->capability_flags &
289 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
291 "Dequeue interrupts are not supported!\n");
296 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
297 const struct rte_bbdev_op_cap_turbo_enc *cap =
298 &op_cap->cap.turbo_enc;
300 if (!flags_match(test_vector.turbo_enc.op_flags,
301 cap->capability_flags))
303 if (nb_inputs > cap->num_buffers_src) {
304 printf("Too many inputs defined: %u, max: %u\n",
305 nb_inputs, cap->num_buffers_src);
308 if (nb_hard_outputs > cap->num_buffers_dst) {
310 "Too many hard outputs defined: %u, max: %u\n",
311 nb_hard_outputs, cap->num_buffers_dst);
314 if (intr_enabled && !(cap->capability_flags &
315 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
317 "Dequeue interrupts are not supported!\n");
322 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
323 const struct rte_bbdev_op_cap_ldpc_enc *cap =
324 &op_cap->cap.ldpc_enc;
326 if (!flags_match(test_vector.ldpc_enc.op_flags,
327 cap->capability_flags)){
328 printf("Flag Mismatch\n");
331 if (nb_inputs > cap->num_buffers_src) {
332 printf("Too many inputs defined: %u, max: %u\n",
333 nb_inputs, cap->num_buffers_src);
336 if (nb_hard_outputs > cap->num_buffers_dst) {
338 "Too many hard outputs defined: %u, max: %u\n",
339 nb_hard_outputs, cap->num_buffers_dst);
342 if (intr_enabled && !(cap->capability_flags &
343 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
345 "Dequeue interrupts are not supported!\n");
350 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
351 const struct rte_bbdev_op_cap_ldpc_dec *cap =
352 &op_cap->cap.ldpc_dec;
354 if (!flags_match(test_vector.ldpc_dec.op_flags,
355 cap->capability_flags)){
356 printf("Flag Mismatch\n");
359 if (nb_inputs > cap->num_buffers_src) {
360 printf("Too many inputs defined: %u, max: %u\n",
361 nb_inputs, cap->num_buffers_src);
364 if (nb_hard_outputs > cap->num_buffers_hard_out) {
366 "Too many hard outputs defined: %u, max: %u\n",
368 cap->num_buffers_hard_out);
371 if (nb_harq_inputs > cap->num_buffers_hard_out) {
373 "Too many HARQ inputs defined: %u, max: %u\n",
375 cap->num_buffers_hard_out);
378 if (nb_harq_outputs > cap->num_buffers_hard_out) {
380 "Too many HARQ outputs defined: %u, max: %u\n",
382 cap->num_buffers_hard_out);
385 if (intr_enabled && !(cap->capability_flags &
386 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
388 "Dequeue interrupts are not supported!\n");
391 if (intr_enabled && (test_vector.ldpc_dec.op_flags &
392 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
393 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
394 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
396 printf("Skip loop-back with interrupt\n");
403 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
404 return TEST_SUCCESS; /* Special case for NULL device */
409 /* calculates optimal mempool size not smaller than the val */
411 optimal_mempool_size(unsigned int val)
413 return rte_align32pow2(val + 1) - 1;
416 /* allocates mbuf mempool for inputs and outputs */
417 static struct rte_mempool *
418 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
419 int socket_id, unsigned int mbuf_pool_size,
420 const char *op_type_str)
423 uint32_t max_seg_sz = 0;
424 char pool_name[RTE_MEMPOOL_NAMESIZE];
426 /* find max input segment size */
427 for (i = 0; i < entries->nb_segments; ++i)
428 if (entries->segments[i].length > max_seg_sz)
429 max_seg_sz = entries->segments[i].length;
431 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
433 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
434 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
436 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
440 create_mempools(struct active_device *ad, int socket_id,
441 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
443 struct rte_mempool *mp;
444 unsigned int ops_pool_size, mbuf_pool_size = 0;
445 char pool_name[RTE_MEMPOOL_NAMESIZE];
446 const char *op_type_str;
447 enum rte_bbdev_op_type op_type = org_op_type;
449 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
450 struct op_data_entries *hard_out =
451 &test_vector.entries[DATA_HARD_OUTPUT];
452 struct op_data_entries *soft_out =
453 &test_vector.entries[DATA_SOFT_OUTPUT];
454 struct op_data_entries *harq_in =
455 &test_vector.entries[DATA_HARQ_INPUT];
456 struct op_data_entries *harq_out =
457 &test_vector.entries[DATA_HARQ_OUTPUT];
459 /* allocate ops mempool */
460 ops_pool_size = optimal_mempool_size(RTE_MAX(
461 /* Ops used plus 1 reference op */
462 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
463 /* Minimal cache size plus 1 reference op */
464 (unsigned int)(1.5 * rte_lcore_count() *
465 OPS_CACHE_SIZE + 1)),
468 if (org_op_type == RTE_BBDEV_OP_NONE)
469 op_type = RTE_BBDEV_OP_TURBO_ENC;
471 op_type_str = rte_bbdev_op_type_str(op_type);
472 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
474 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
476 mp = rte_bbdev_op_pool_create(pool_name, op_type,
477 ops_pool_size, OPS_CACHE_SIZE, socket_id);
478 TEST_ASSERT_NOT_NULL(mp,
479 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
483 ad->ops_mempool = mp;
485 /* Do not create inputs and outputs mbufs for BaseBand Null Device */
486 if (org_op_type == RTE_BBDEV_OP_NONE)
490 if (in->nb_segments > 0) {
491 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
493 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
494 mbuf_pool_size, "in");
495 TEST_ASSERT_NOT_NULL(mp,
496 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
500 ad->in_mbuf_pool = mp;
504 if (hard_out->nb_segments > 0) {
505 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
506 hard_out->nb_segments);
507 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
510 TEST_ASSERT_NOT_NULL(mp,
511 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
515 ad->hard_out_mbuf_pool = mp;
519 if (soft_out->nb_segments > 0) {
520 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
521 soft_out->nb_segments);
522 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
525 TEST_ASSERT_NOT_NULL(mp,
526 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
530 ad->soft_out_mbuf_pool = mp;
534 if (harq_in->nb_segments > 0) {
535 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
536 harq_in->nb_segments);
537 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
540 TEST_ASSERT_NOT_NULL(mp,
541 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
545 ad->harq_in_mbuf_pool = mp;
549 if (harq_out->nb_segments > 0) {
550 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
551 harq_out->nb_segments);
552 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
555 TEST_ASSERT_NOT_NULL(mp,
556 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
560 ad->harq_out_mbuf_pool = mp;
567 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
568 struct test_bbdev_vector *vector)
571 unsigned int queue_id;
572 struct rte_bbdev_queue_conf qconf;
573 struct active_device *ad = &active_devs[nb_active_devs];
574 unsigned int nb_queues;
575 enum rte_bbdev_op_type op_type = vector->op_type;
577 /* Configure fpga lte fec with PF & VF values
578 * if '-i' flag is set and using fpga device
580 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
581 if ((get_init_device() == true) &&
582 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
583 struct rte_fpga_lte_fec_conf conf;
586 printf("Configure FPGA LTE FEC Driver %s with default values\n",
587 info->drv.driver_name);
589 /* clear default configuration before initialization */
590 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf));
593 * true if PF is used for data plane
596 conf.pf_mode_en = true;
598 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
599 /* Number of UL queues per VF (fpga supports 8 VFs) */
600 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
601 /* Number of DL queues per VF (fpga supports 8 VFs) */
602 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
605 /* UL bandwidth. Needed for schedule algorithm */
606 conf.ul_bandwidth = UL_4G_BANDWIDTH;
608 conf.dl_bandwidth = DL_4G_BANDWIDTH;
610 /* UL & DL load Balance Factor to 64 */
611 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
612 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
614 /**< FLR timeout value */
615 conf.flr_time_out = FLR_4G_TIMEOUT;
617 /* setup FPGA PF with configuration information */
618 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf);
619 TEST_ASSERT_SUCCESS(ret,
620 "Failed to configure 4G FPGA PF for bbdev %s",
624 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
625 if ((get_init_device() == true) &&
626 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
627 struct rte_fpga_5gnr_fec_conf conf;
630 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
631 info->drv.driver_name);
633 /* clear default configuration before initialization */
634 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
637 * true if PF is used for data plane
640 conf.pf_mode_en = true;
642 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
643 /* Number of UL queues per VF (fpga supports 8 VFs) */
644 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
645 /* Number of DL queues per VF (fpga supports 8 VFs) */
646 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
649 /* UL bandwidth. Needed for schedule algorithm */
650 conf.ul_bandwidth = UL_5G_BANDWIDTH;
652 conf.dl_bandwidth = DL_5G_BANDWIDTH;
654 /* UL & DL load Balance Factor to 64 */
655 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
656 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
658 /**< FLR timeout value */
659 conf.flr_time_out = FLR_5G_TIMEOUT;
661 /* setup FPGA PF with configuration information */
662 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf);
663 TEST_ASSERT_SUCCESS(ret,
664 "Failed to configure 5G FPGA PF for bbdev %s",
668 #ifdef RTE_BASEBAND_ACC100
669 if ((get_init_device() == true) &&
670 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) {
671 struct rte_acc100_conf conf;
674 printf("Configure ACC100 FEC Driver %s with default values\n",
675 info->drv.driver_name);
677 /* clear default configuration before initialization */
678 memset(&conf, 0, sizeof(struct rte_acc100_conf));
680 /* Always set in PF mode for built-in configuration */
681 conf.pf_mode_en = true;
682 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) {
683 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
684 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
685 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR;
686 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
687 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
688 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR;
689 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
690 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
691 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR;
692 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
693 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
694 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR;
697 conf.input_pos_llr_1_bit = true;
698 conf.output_pos_llr_1_bit = true;
699 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */
701 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
702 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
703 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
704 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
705 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
706 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
707 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
708 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
709 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
710 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
711 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
712 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
713 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
714 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
715 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
716 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
718 /* setup PF with configuration information */
719 ret = rte_acc100_configure(info->dev_name, &conf);
720 TEST_ASSERT_SUCCESS(ret,
721 "Failed to configure ACC100 PF for bbdev %s",
725 /* Let's refresh this now this is configured */
726 rte_bbdev_info_get(dev_id, info);
727 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
728 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
731 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
733 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
734 dev_id, nb_queues, info->socket_id, ret);
738 /* configure interrupts if needed */
740 ret = rte_bbdev_intr_enable(dev_id);
742 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
748 /* setup device queues */
749 qconf.socket = info->socket_id;
750 qconf.queue_size = info->drv.default_queue_conf.queue_size;
752 qconf.deferred_start = 0;
753 qconf.op_type = op_type;
755 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
756 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
759 "Allocated all queues (id=%u) at prio%u on dev%u\n",
760 queue_id, qconf.priority, dev_id);
762 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
766 printf("All queues on dev %u allocated: %u\n",
770 ad->queue_ids[queue_id] = queue_id;
772 TEST_ASSERT(queue_id != 0,
773 "ERROR Failed to configure any queues on dev %u",
775 ad->nb_queues = queue_id;
777 set_avail_op(ad, op_type);
783 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
784 struct test_bbdev_vector *vector)
788 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
789 active_devs[nb_active_devs].dev_id = dev_id;
791 ret = add_bbdev_dev(dev_id, info, vector);
792 if (ret == TEST_SUCCESS)
798 populate_active_devices(void)
802 uint8_t nb_devs_added = 0;
803 struct rte_bbdev_info info;
805 RTE_BBDEV_FOREACH(dev_id) {
806 rte_bbdev_info_get(dev_id, &info);
808 if (check_dev_cap(&info)) {
810 "Device %d (%s) does not support specified capabilities\n",
811 dev_id, info.dev_name);
815 ret = add_active_device(dev_id, &info, &test_vector);
817 printf("Adding active bbdev %s skipped\n",
824 return nb_devs_added;
828 read_test_vector(void)
832 memset(&test_vector, 0, sizeof(test_vector));
833 printf("Test vector file = %s\n", get_vector_filename());
834 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
835 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
836 get_vector_filename());
842 testsuite_setup(void)
844 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
846 if (populate_active_devices() == 0) {
847 printf("No suitable devices found!\n");
855 interrupt_testsuite_setup(void)
857 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
859 /* Enable interrupts */
862 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
863 if (populate_active_devices() == 0 ||
864 test_vector.op_type == RTE_BBDEV_OP_NONE) {
865 intr_enabled = false;
866 printf("No suitable devices found!\n");
874 testsuite_teardown(void)
878 /* Unconfigure devices */
879 RTE_BBDEV_FOREACH(dev_id)
880 rte_bbdev_close(dev_id);
882 /* Clear active devices structs. */
883 memset(active_devs, 0, sizeof(active_devs));
886 /* Disable interrupts */
887 intr_enabled = false;
895 for (i = 0; i < nb_active_devs; i++) {
896 dev_id = active_devs[i].dev_id;
897 /* reset bbdev stats */
898 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
899 "Failed to reset stats of bbdev %u", dev_id);
900 /* start the device */
901 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
902 "Failed to start bbdev %u", dev_id);
912 struct rte_bbdev_stats stats;
914 for (i = 0; i < nb_active_devs; i++) {
915 dev_id = active_devs[i].dev_id;
916 /* read stats and print */
917 rte_bbdev_stats_get(dev_id, &stats);
918 /* Stop the device */
919 rte_bbdev_stop(dev_id);
924 init_op_data_objs(struct rte_bbdev_op_data *bufs,
925 struct op_data_entries *ref_entries,
926 struct rte_mempool *mbuf_pool, const uint16_t n,
927 enum op_data_type op_type, uint16_t min_alignment)
931 bool large_input = false;
933 for (i = 0; i < n; ++i) {
935 struct op_data_buf *seg = &ref_entries->segments[0];
936 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
937 TEST_ASSERT_NOT_NULL(m_head,
938 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
939 op_type, n * ref_entries->nb_segments,
942 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
944 * Special case when DPDK mbuf cannot handle
945 * the required input size
947 printf("Warning: Larger input size than DPDK mbuf %d\n",
951 bufs[i].data = m_head;
955 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
956 if ((op_type == DATA_INPUT) && large_input) {
957 /* Allocate a fake overused mbuf */
958 data = rte_malloc(NULL, seg->length, 0);
959 memcpy(data, seg->addr, seg->length);
960 m_head->buf_addr = data;
961 m_head->buf_iova = rte_malloc_virt2iova(data);
962 m_head->data_off = 0;
963 m_head->data_len = seg->length;
965 data = rte_pktmbuf_append(m_head, seg->length);
966 TEST_ASSERT_NOT_NULL(data,
967 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
968 seg->length, op_type);
970 TEST_ASSERT(data == RTE_PTR_ALIGN(
971 data, min_alignment),
972 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
973 data, min_alignment);
974 rte_memcpy(data, seg->addr, seg->length);
977 bufs[i].length += seg->length;
979 for (j = 1; j < ref_entries->nb_segments; ++j) {
980 struct rte_mbuf *m_tail =
981 rte_pktmbuf_alloc(mbuf_pool);
982 TEST_ASSERT_NOT_NULL(m_tail,
983 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
985 n * ref_entries->nb_segments,
989 data = rte_pktmbuf_append(m_tail, seg->length);
990 TEST_ASSERT_NOT_NULL(data,
991 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
992 seg->length, op_type);
994 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
996 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
997 data, min_alignment);
998 rte_memcpy(data, seg->addr, seg->length);
999 bufs[i].length += seg->length;
1001 ret = rte_pktmbuf_chain(m_head, m_tail);
1002 TEST_ASSERT_SUCCESS(ret,
1003 "Couldn't chain mbufs from %d data type mbuf pool",
1008 /* allocate chained-mbuf for output buffer */
1009 for (j = 1; j < ref_entries->nb_segments; ++j) {
1010 struct rte_mbuf *m_tail =
1011 rte_pktmbuf_alloc(mbuf_pool);
1012 TEST_ASSERT_NOT_NULL(m_tail,
1013 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1015 n * ref_entries->nb_segments,
1018 ret = rte_pktmbuf_chain(m_head, m_tail);
1019 TEST_ASSERT_SUCCESS(ret,
1020 "Couldn't chain mbufs from %d data type mbuf pool",
1030 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
1035 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
1036 if (*buffers == NULL) {
1037 printf("WARNING: Failed to allocate op_data on socket %d\n",
1039 /* try to allocate memory on other detected sockets */
1040 for (i = 0; i < socket; i++) {
1041 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
1042 if (*buffers != NULL)
1047 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
1051 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
1052 const uint16_t n, const int8_t max_llr_modulus)
1054 uint16_t i, byte_idx;
1056 for (i = 0; i < n; ++i) {
1057 struct rte_mbuf *m = input_ops[i].data;
1059 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1060 input_ops[i].offset);
1061 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1063 llr[byte_idx] = round((double)max_llr_modulus *
1064 llr[byte_idx] / INT8_MAX);
1072 * We may have to insert filler bits
1073 * when they are required by the HARQ assumption
1076 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1077 const uint16_t n, struct test_op_params *op_params)
1079 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1081 if (input_ops == NULL)
1083 /* No need to add filler if not required by device */
1084 if (!(ldpc_cap_flags &
1085 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1087 /* No need to add filler for loopback operation */
1088 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1091 uint16_t i, j, parity_offset;
1092 for (i = 0; i < n; ++i) {
1093 struct rte_mbuf *m = input_ops[i].data;
1094 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1095 input_ops[i].offset);
1096 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1097 * dec.z_c - dec.n_filler;
1098 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1099 m->data_len = new_hin_size;
1100 input_ops[i].length = new_hin_size;
1101 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1103 llr[j] = llr[j - dec.n_filler];
1104 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1105 for (j = 0; j < dec.n_filler; j++)
1106 llr[parity_offset + j] = llr_max_pre_scaling;
1111 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1112 const uint16_t n, const int8_t llr_size,
1113 const int8_t llr_decimals)
1115 if (input_ops == NULL)
1118 uint16_t i, byte_idx;
1120 int16_t llr_max, llr_min, llr_tmp;
1121 llr_max = (1 << (llr_size - 1)) - 1;
1123 for (i = 0; i < n; ++i) {
1124 struct rte_mbuf *m = input_ops[i].data;
1126 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1127 input_ops[i].offset);
1128 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1131 llr_tmp = llr[byte_idx];
1132 if (llr_decimals == 4)
1134 else if (llr_decimals == 2)
1136 else if (llr_decimals == 0)
1138 llr_tmp = RTE_MIN(llr_max,
1139 RTE_MAX(llr_min, llr_tmp));
1140 llr[byte_idx] = (int8_t) llr_tmp;
1151 fill_queue_buffers(struct test_op_params *op_params,
1152 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1153 struct rte_mempool *soft_out_mp,
1154 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1156 const struct rte_bbdev_op_cap *capabilities,
1157 uint16_t min_alignment, const int socket_id)
1160 enum op_data_type type;
1161 const uint16_t n = op_params->num_to_process;
1163 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1171 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1172 &op_params->q_bufs[socket_id][queue_id].inputs,
1173 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1174 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1175 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1176 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1179 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1180 struct op_data_entries *ref_entries =
1181 &test_vector.entries[type];
1182 if (ref_entries->nb_segments == 0)
1185 ret = allocate_buffers_on_socket(queue_ops[type],
1186 n * sizeof(struct rte_bbdev_op_data),
1188 TEST_ASSERT_SUCCESS(ret,
1189 "Couldn't allocate memory for rte_bbdev_op_data structs");
1191 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1192 mbuf_pools[type], n, type, min_alignment);
1193 TEST_ASSERT_SUCCESS(ret,
1194 "Couldn't init rte_bbdev_op_data structs");
1197 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1198 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1199 capabilities->cap.turbo_dec.max_llr_modulus);
1201 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1202 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1203 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1204 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1205 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1206 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1207 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1208 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1209 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1210 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1211 if (!loopback && !llr_comp)
1212 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1213 ldpc_llr_size, ldpc_llr_decimals);
1214 if (!loopback && !harq_comp)
1215 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1216 ldpc_llr_size, ldpc_llr_decimals);
1218 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1226 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1230 rte_mempool_free(ad->ops_mempool);
1231 rte_mempool_free(ad->in_mbuf_pool);
1232 rte_mempool_free(ad->hard_out_mbuf_pool);
1233 rte_mempool_free(ad->soft_out_mbuf_pool);
1234 rte_mempool_free(ad->harq_in_mbuf_pool);
1235 rte_mempool_free(ad->harq_out_mbuf_pool);
1237 for (i = 0; i < rte_lcore_count(); ++i) {
1238 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1239 rte_free(op_params->q_bufs[j][i].inputs);
1240 rte_free(op_params->q_bufs[j][i].hard_outputs);
1241 rte_free(op_params->q_bufs[j][i].soft_outputs);
1242 rte_free(op_params->q_bufs[j][i].harq_inputs);
1243 rte_free(op_params->q_bufs[j][i].harq_outputs);
1249 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1250 unsigned int start_idx,
1251 struct rte_bbdev_op_data *inputs,
1252 struct rte_bbdev_op_data *hard_outputs,
1253 struct rte_bbdev_op_data *soft_outputs,
1254 struct rte_bbdev_dec_op *ref_op)
1257 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1259 for (i = 0; i < n; ++i) {
1260 if (turbo_dec->code_block_mode == 0) {
1261 ops[i]->turbo_dec.tb_params.ea =
1262 turbo_dec->tb_params.ea;
1263 ops[i]->turbo_dec.tb_params.eb =
1264 turbo_dec->tb_params.eb;
1265 ops[i]->turbo_dec.tb_params.k_pos =
1266 turbo_dec->tb_params.k_pos;
1267 ops[i]->turbo_dec.tb_params.k_neg =
1268 turbo_dec->tb_params.k_neg;
1269 ops[i]->turbo_dec.tb_params.c =
1270 turbo_dec->tb_params.c;
1271 ops[i]->turbo_dec.tb_params.c_neg =
1272 turbo_dec->tb_params.c_neg;
1273 ops[i]->turbo_dec.tb_params.cab =
1274 turbo_dec->tb_params.cab;
1275 ops[i]->turbo_dec.tb_params.r =
1276 turbo_dec->tb_params.r;
1278 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1279 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1282 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1283 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1284 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1285 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1286 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1287 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1288 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1290 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1291 ops[i]->turbo_dec.input = inputs[start_idx + i];
1292 if (soft_outputs != NULL)
1293 ops[i]->turbo_dec.soft_output =
1294 soft_outputs[start_idx + i];
1299 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1300 unsigned int start_idx,
1301 struct rte_bbdev_op_data *inputs,
1302 struct rte_bbdev_op_data *outputs,
1303 struct rte_bbdev_enc_op *ref_op)
1306 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1307 for (i = 0; i < n; ++i) {
1308 if (turbo_enc->code_block_mode == 0) {
1309 ops[i]->turbo_enc.tb_params.ea =
1310 turbo_enc->tb_params.ea;
1311 ops[i]->turbo_enc.tb_params.eb =
1312 turbo_enc->tb_params.eb;
1313 ops[i]->turbo_enc.tb_params.k_pos =
1314 turbo_enc->tb_params.k_pos;
1315 ops[i]->turbo_enc.tb_params.k_neg =
1316 turbo_enc->tb_params.k_neg;
1317 ops[i]->turbo_enc.tb_params.c =
1318 turbo_enc->tb_params.c;
1319 ops[i]->turbo_enc.tb_params.c_neg =
1320 turbo_enc->tb_params.c_neg;
1321 ops[i]->turbo_enc.tb_params.cab =
1322 turbo_enc->tb_params.cab;
1323 ops[i]->turbo_enc.tb_params.ncb_pos =
1324 turbo_enc->tb_params.ncb_pos;
1325 ops[i]->turbo_enc.tb_params.ncb_neg =
1326 turbo_enc->tb_params.ncb_neg;
1327 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1329 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1330 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1331 ops[i]->turbo_enc.cb_params.ncb =
1332 turbo_enc->cb_params.ncb;
1334 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1335 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1336 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1338 ops[i]->turbo_enc.output = outputs[start_idx + i];
1339 ops[i]->turbo_enc.input = inputs[start_idx + i];
1344 /* Returns a random number drawn from a normal distribution
1345 * with mean of 0 and variance of 1
1346 * Marsaglia algorithm
1351 double S, Z, U1, U2, u, v, fac;
1354 U1 = (double)rand() / RAND_MAX;
1355 U2 = (double)rand() / RAND_MAX;
1359 } while (S >= 1 || S == 0);
1360 fac = sqrt(-2. * log(S) / S);
1361 Z = (n % 2) ? u * fac : v * fac;
1365 static inline double
1366 maxstar(double A, double B)
1368 if (fabs(A - B) > 5)
1369 return RTE_MAX(A, B);
1371 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1375 * Generate Qm LLRS for Qm==8
1376 * Modulation, AWGN and LLR estimation from max log development
1379 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1384 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1385 /* 5.1.4 of TS38.211 */
1386 const double symbols_I[256] = {
1387 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1388 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1389 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1390 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1391 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1392 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1393 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1394 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1395 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1396 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1397 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1398 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1399 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1400 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1401 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1402 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1403 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1404 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1405 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1406 -13, -13, -15, -15, -13, -13, -15, -15};
1407 const double symbols_Q[256] = {
1408 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1409 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1410 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1411 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1412 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1413 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1414 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1415 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1416 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1417 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1418 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1419 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1420 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1421 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1422 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1423 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1424 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1425 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1426 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1427 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1428 /* Average constellation point energy */
1430 for (k = 0; k < qm; k++)
1431 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1432 /* 5.1.4 of TS38.211 */
1433 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1434 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1435 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1436 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1438 I += sqrt(N0 / 2) * randn(0);
1439 Q += sqrt(N0 / 2) * randn(1);
1441 * Calculate the log of the probability that each of
1442 * the constellation points was transmitted
1444 for (m = 0; m < qam; m++)
1445 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1446 + pow(Q - symbols_Q[m], 2.0)) / N0;
1447 /* Calculate an LLR for each of the k_64QAM bits in the set */
1448 for (k = 0; k < qm; k++) {
1451 /* For each constellation point */
1452 for (m = 0; m < qam; m++) {
1453 if ((m >> (qm - k - 1)) & 1)
1454 p1 = maxstar(p1, log_syml_prob[m]);
1456 p0 = maxstar(p0, log_syml_prob[m]);
1458 /* Calculate the LLR */
1460 llr_ *= (1 << ldpc_llr_decimals);
1464 if (llr_ < -llr_max)
1466 llrs[qm * i + k] = (int8_t) llr_;
1472 * Generate Qm LLRS for Qm==6
1473 * Modulation, AWGN and LLR estimation from max log development
1476 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1481 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1482 /* 5.1.4 of TS38.211 */
1483 const double symbols_I[64] = {
1484 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1485 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1486 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1487 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1488 -5, -5, -7, -7, -5, -5, -7, -7};
1489 const double symbols_Q[64] = {
1490 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1491 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1492 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1493 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1494 -3, -1, -3, -1, -5, -7, -5, -7};
1495 /* Average constellation point energy */
1497 for (k = 0; k < qm; k++)
1498 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1499 /* 5.1.4 of TS38.211 */
1500 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1501 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1503 I += sqrt(N0 / 2) * randn(0);
1504 Q += sqrt(N0 / 2) * randn(1);
1506 * Calculate the log of the probability that each of
1507 * the constellation points was transmitted
1509 for (m = 0; m < qam; m++)
1510 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1511 + pow(Q - symbols_Q[m], 2.0)) / N0;
1512 /* Calculate an LLR for each of the k_64QAM bits in the set */
1513 for (k = 0; k < qm; k++) {
1516 /* For each constellation point */
1517 for (m = 0; m < qam; m++) {
1518 if ((m >> (qm - k - 1)) & 1)
1519 p1 = maxstar(p1, log_syml_prob[m]);
1521 p0 = maxstar(p0, log_syml_prob[m]);
1523 /* Calculate the LLR */
1525 llr_ *= (1 << ldpc_llr_decimals);
1529 if (llr_ < -llr_max)
1531 llrs[qm * i + k] = (int8_t) llr_;
1536 * Generate Qm LLRS for Qm==4
1537 * Modulation, AWGN and LLR estimation from max log development
1540 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1545 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1546 /* 5.1.4 of TS38.211 */
1547 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1548 -1, -1, -3, -3, -1, -1, -3, -3};
1549 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1550 1, 3, 1, 3, -1, -3, -1, -3};
1551 /* Average constellation point energy */
1553 for (k = 0; k < qm; k++)
1554 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1555 /* 5.1.4 of TS38.211 */
1556 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1557 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1559 I += sqrt(N0 / 2) * randn(0);
1560 Q += sqrt(N0 / 2) * randn(1);
1562 * Calculate the log of the probability that each of
1563 * the constellation points was transmitted
1565 for (m = 0; m < qam; m++)
1566 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1567 + pow(Q - symbols_Q[m], 2.0)) / N0;
1568 /* Calculate an LLR for each of the k_64QAM bits in the set */
1569 for (k = 0; k < qm; k++) {
1572 /* For each constellation point */
1573 for (m = 0; m < qam; m++) {
1574 if ((m >> (qm - k - 1)) & 1)
1575 p1 = maxstar(p1, log_syml_prob[m]);
1577 p0 = maxstar(p0, log_syml_prob[m]);
1579 /* Calculate the LLR */
1581 llr_ *= (1 << ldpc_llr_decimals);
1585 if (llr_ < -llr_max)
1587 llrs[qm * i + k] = (int8_t) llr_;
1592 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1595 double coeff = 2.0 * sqrt(N0);
1597 /* Ignore in vectors rare quasi null LLRs not to be saturated */
1598 if (llrs[j] < 8 && llrs[j] > -8)
1601 /* Note don't change sign here */
1603 b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1605 b = b1 * (1 << ldpc_llr_decimals);
1611 llrs[j] = (int8_t) b;
1614 /* Generate LLR for a given SNR */
1616 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1617 struct rte_bbdev_dec_op *ref_op)
1621 uint32_t i, j, e, range;
1624 e = ref_op->ldpc_dec.cb_params.e;
1625 qm = ref_op->ldpc_dec.q_m;
1626 llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1628 N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1630 for (i = 0; i < n; ++i) {
1632 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1634 for (j = 0; j < range; ++j)
1635 gen_qm8_llr(llrs, j, N0, llr_max);
1636 } else if (qm == 6) {
1637 for (j = 0; j < range; ++j)
1638 gen_qm6_llr(llrs, j, N0, llr_max);
1639 } else if (qm == 4) {
1640 for (j = 0; j < range; ++j)
1641 gen_qm4_llr(llrs, j, N0, llr_max);
1643 for (j = 0; j < e; ++j)
1644 gen_qm2_llr(llrs, j, N0, llr_max);
1650 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1651 unsigned int start_idx,
1652 struct rte_bbdev_op_data *inputs,
1653 struct rte_bbdev_op_data *hard_outputs,
1654 struct rte_bbdev_op_data *soft_outputs,
1655 struct rte_bbdev_op_data *harq_inputs,
1656 struct rte_bbdev_op_data *harq_outputs,
1657 struct rte_bbdev_dec_op *ref_op)
1660 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1662 for (i = 0; i < n; ++i) {
1663 if (ldpc_dec->code_block_mode == 0) {
1664 ops[i]->ldpc_dec.tb_params.ea =
1665 ldpc_dec->tb_params.ea;
1666 ops[i]->ldpc_dec.tb_params.eb =
1667 ldpc_dec->tb_params.eb;
1668 ops[i]->ldpc_dec.tb_params.c =
1669 ldpc_dec->tb_params.c;
1670 ops[i]->ldpc_dec.tb_params.cab =
1671 ldpc_dec->tb_params.cab;
1672 ops[i]->ldpc_dec.tb_params.r =
1673 ldpc_dec->tb_params.r;
1675 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1678 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1679 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1680 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1681 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1682 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1683 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1684 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1685 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1686 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1688 if (hard_outputs != NULL)
1689 ops[i]->ldpc_dec.hard_output =
1690 hard_outputs[start_idx + i];
1692 ops[i]->ldpc_dec.input =
1693 inputs[start_idx + i];
1694 if (soft_outputs != NULL)
1695 ops[i]->ldpc_dec.soft_output =
1696 soft_outputs[start_idx + i];
1697 if (harq_inputs != NULL)
1698 ops[i]->ldpc_dec.harq_combined_input =
1699 harq_inputs[start_idx + i];
1700 if (harq_outputs != NULL)
1701 ops[i]->ldpc_dec.harq_combined_output =
1702 harq_outputs[start_idx + i];
1708 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1709 unsigned int start_idx,
1710 struct rte_bbdev_op_data *inputs,
1711 struct rte_bbdev_op_data *outputs,
1712 struct rte_bbdev_enc_op *ref_op)
1715 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1716 for (i = 0; i < n; ++i) {
1717 if (ldpc_enc->code_block_mode == 0) {
1718 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1719 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1720 ops[i]->ldpc_enc.tb_params.cab =
1721 ldpc_enc->tb_params.cab;
1722 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1723 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1725 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1727 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1728 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1729 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1730 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1731 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1732 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1733 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1734 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1735 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1736 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1741 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1742 unsigned int order_idx, const int expected_status)
1744 int status = op->status;
1745 /* ignore parity mismatch false alarms for long iterations */
1746 if (get_iter_max() >= 10) {
1747 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1748 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1749 printf("WARNING: Ignore Syndrome Check mismatch\n");
1750 status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1752 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1753 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1754 printf("WARNING: Ignore Syndrome Check mismatch\n");
1755 status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1759 TEST_ASSERT(status == expected_status,
1760 "op_status (%d) != expected_status (%d)",
1761 op->status, expected_status);
1763 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1764 "Ordering error, expected %p, got %p",
1765 (void *)(uintptr_t)order_idx, op->opaque_data);
1767 return TEST_SUCCESS;
1771 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1772 unsigned int order_idx, const int expected_status)
1774 TEST_ASSERT(op->status == expected_status,
1775 "op_status (%d) != expected_status (%d)",
1776 op->status, expected_status);
1778 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1779 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1780 "Ordering error, expected %p, got %p",
1781 (void *)(uintptr_t)order_idx, op->opaque_data);
1783 return TEST_SUCCESS;
1787 validate_op_chain(struct rte_bbdev_op_data *op,
1788 struct op_data_entries *orig_op)
1791 struct rte_mbuf *m = op->data;
1792 uint8_t nb_dst_segments = orig_op->nb_segments;
1793 uint32_t total_data_size = 0;
1795 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1796 "Number of segments differ in original (%u) and filled (%u) op",
1797 nb_dst_segments, m->nb_segs);
1799 /* Validate each mbuf segment length */
1800 for (i = 0; i < nb_dst_segments; ++i) {
1801 /* Apply offset to the first mbuf segment */
1802 uint16_t offset = (i == 0) ? op->offset : 0;
1803 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1804 total_data_size += orig_op->segments[i].length;
1806 TEST_ASSERT(orig_op->segments[i].length == data_len,
1807 "Length of segment differ in original (%u) and filled (%u) op",
1808 orig_op->segments[i].length, data_len);
1809 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1810 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1812 "Output buffers (CB=%u) are not equal", i);
1816 /* Validate total mbuf pkt length */
1817 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1818 TEST_ASSERT(total_data_size == pkt_len,
1819 "Length of data differ in original (%u) and filled (%u) op",
1820 total_data_size, pkt_len);
1822 return TEST_SUCCESS;
1826 * Compute K0 for a given configuration for HARQ output length computation
1827 * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1829 static inline uint16_t
1830 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1834 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1837 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1838 else if (rv_index == 2)
1839 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1841 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1843 /* LBRM case - includes a division by N */
1845 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1847 else if (rv_index == 2)
1848 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1851 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1855 /* HARQ output length including the Filler bits */
1856 static inline uint16_t
1857 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1860 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1861 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1862 /* Compute RM out size and number of rows */
1863 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1864 * ops_ld->z_c - ops_ld->n_filler;
1865 uint16_t deRmOutSize = RTE_MIN(
1866 k0 + ops_ld->cb_params.e +
1867 ((k0 > parity_offset) ?
1868 0 : ops_ld->n_filler),
1870 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1872 uint16_t harq_output_len = numRows * ops_ld->z_c;
1873 return harq_output_len;
1877 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1878 struct op_data_entries *orig_op,
1879 struct rte_bbdev_op_ldpc_dec *ops_ld)
1883 struct rte_mbuf *m = op->data;
1884 uint8_t nb_dst_segments = orig_op->nb_segments;
1885 uint32_t total_data_size = 0;
1886 int8_t *harq_orig, *harq_out, abs_harq_origin;
1887 uint32_t byte_error = 0, cum_error = 0, error;
1888 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1889 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1890 uint16_t parity_offset;
1892 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1893 "Number of segments differ in original (%u) and filled (%u) op",
1894 nb_dst_segments, m->nb_segs);
1896 /* Validate each mbuf segment length */
1897 for (i = 0; i < nb_dst_segments; ++i) {
1898 /* Apply offset to the first mbuf segment */
1899 uint16_t offset = (i == 0) ? op->offset : 0;
1900 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1901 total_data_size += orig_op->segments[i].length;
1903 TEST_ASSERT(orig_op->segments[i].length <
1904 (uint32_t)(data_len + 64),
1905 "Length of segment differ in original (%u) and filled (%u) op",
1906 orig_op->segments[i].length, data_len);
1907 harq_orig = (int8_t *) orig_op->segments[i].addr;
1908 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1910 if (!(ldpc_cap_flags &
1911 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1912 ) || (ops_ld->op_flags &
1913 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1914 data_len -= ops_ld->z_c;
1915 parity_offset = data_len;
1917 /* Compute RM out size and number of rows */
1918 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1919 * ops_ld->z_c - ops_ld->n_filler;
1920 uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1922 if (data_len > deRmOutSize)
1923 data_len = deRmOutSize;
1924 if (data_len > orig_op->segments[i].length)
1925 data_len = orig_op->segments[i].length;
1928 * HARQ output can have minor differences
1929 * due to integer representation and related scaling
1931 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1932 if (j == parity_offset) {
1933 /* Special Handling of the filler bits */
1934 for (k = 0; k < ops_ld->n_filler; k++) {
1936 llr_max_pre_scaling) {
1937 printf("HARQ Filler issue %d: %d %d\n",
1945 if (!(ops_ld->op_flags &
1946 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1947 if (ldpc_llr_decimals > 1)
1948 harq_out[jj] = (harq_out[jj] + 1)
1949 >> (ldpc_llr_decimals - 1);
1950 /* Saturated to S7 */
1951 if (harq_orig[j] > llr_max)
1952 harq_orig[j] = llr_max;
1953 if (harq_orig[j] < -llr_max)
1954 harq_orig[j] = -llr_max;
1956 if (harq_orig[j] != harq_out[jj]) {
1957 error = (harq_orig[j] > harq_out[jj]) ?
1958 harq_orig[j] - harq_out[jj] :
1959 harq_out[jj] - harq_orig[j];
1960 abs_harq_origin = harq_orig[j] > 0 ?
1963 /* Residual quantization error */
1964 if ((error > 8 && (abs_harq_origin <
1967 printf("HARQ mismatch %d: exp %d act %d => %d\n",
1969 harq_out[jj], error);
1979 TEST_ASSERT(byte_error <= 1,
1980 "HARQ output mismatch (%d) %d",
1981 byte_error, cum_error);
1983 /* Validate total mbuf pkt length */
1984 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1985 TEST_ASSERT(total_data_size < pkt_len + 64,
1986 "Length of data differ in original (%u) and filled (%u) op",
1987 total_data_size, pkt_len);
1989 return TEST_SUCCESS;
1993 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1994 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1998 struct op_data_entries *hard_data_orig =
1999 &test_vector.entries[DATA_HARD_OUTPUT];
2000 struct op_data_entries *soft_data_orig =
2001 &test_vector.entries[DATA_SOFT_OUTPUT];
2002 struct rte_bbdev_op_turbo_dec *ops_td;
2003 struct rte_bbdev_op_data *hard_output;
2004 struct rte_bbdev_op_data *soft_output;
2005 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
2007 for (i = 0; i < n; ++i) {
2008 ops_td = &ops[i]->turbo_dec;
2009 hard_output = &ops_td->hard_output;
2010 soft_output = &ops_td->soft_output;
2012 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2013 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2014 "Returned iter_count (%d) > expected iter_count (%d)",
2015 ops_td->iter_count, ref_td->iter_count);
2016 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2017 TEST_ASSERT_SUCCESS(ret,
2018 "Checking status and ordering for decoder failed");
2020 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2022 "Hard output buffers (CB=%u) are not equal",
2025 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
2026 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2028 "Soft output buffers (CB=%u) are not equal",
2032 return TEST_SUCCESS;
2035 /* Check Number of code blocks errors */
2037 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
2040 struct op_data_entries *hard_data_orig =
2041 &test_vector.entries[DATA_HARD_OUTPUT];
2042 struct rte_bbdev_op_ldpc_dec *ops_td;
2043 struct rte_bbdev_op_data *hard_output;
2047 for (i = 0; i < n; ++i) {
2048 ops_td = &ops[i]->ldpc_dec;
2049 hard_output = &ops_td->hard_output;
2050 m = hard_output->data;
2051 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
2052 hard_data_orig->segments[0].addr,
2053 hard_data_orig->segments[0].length))
2060 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2061 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2065 struct op_data_entries *hard_data_orig =
2066 &test_vector.entries[DATA_HARD_OUTPUT];
2067 struct op_data_entries *soft_data_orig =
2068 &test_vector.entries[DATA_SOFT_OUTPUT];
2069 struct op_data_entries *harq_data_orig =
2070 &test_vector.entries[DATA_HARQ_OUTPUT];
2071 struct rte_bbdev_op_ldpc_dec *ops_td;
2072 struct rte_bbdev_op_data *hard_output;
2073 struct rte_bbdev_op_data *harq_output;
2074 struct rte_bbdev_op_data *soft_output;
2075 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2077 for (i = 0; i < n; ++i) {
2078 ops_td = &ops[i]->ldpc_dec;
2079 hard_output = &ops_td->hard_output;
2080 harq_output = &ops_td->harq_combined_output;
2081 soft_output = &ops_td->soft_output;
2083 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2084 TEST_ASSERT_SUCCESS(ret,
2085 "Checking status and ordering for decoder failed");
2086 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2087 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2088 "Returned iter_count (%d) > expected iter_count (%d)",
2089 ops_td->iter_count, ref_td->iter_count);
2091 * We can ignore output data when the decoding failed to
2092 * converge or for loop-back cases
2094 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2095 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2097 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2099 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2101 "Hard output buffers (CB=%u) are not equal",
2104 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2105 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2107 "Soft output buffers (CB=%u) are not equal",
2109 if (ref_op->ldpc_dec.op_flags &
2110 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2111 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2112 harq_data_orig, ops_td),
2113 "HARQ output buffers (CB=%u) are not equal",
2116 if (ref_op->ldpc_dec.op_flags &
2117 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2118 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2119 harq_data_orig, ops_td),
2120 "HARQ output buffers (CB=%u) are not equal",
2125 return TEST_SUCCESS;
2130 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2131 struct rte_bbdev_enc_op *ref_op)
2135 struct op_data_entries *hard_data_orig =
2136 &test_vector.entries[DATA_HARD_OUTPUT];
2138 for (i = 0; i < n; ++i) {
2139 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2140 TEST_ASSERT_SUCCESS(ret,
2141 "Checking status and ordering for encoder failed");
2142 TEST_ASSERT_SUCCESS(validate_op_chain(
2143 &ops[i]->turbo_enc.output,
2145 "Output buffers (CB=%u) are not equal",
2149 return TEST_SUCCESS;
2153 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2154 struct rte_bbdev_enc_op *ref_op)
2158 struct op_data_entries *hard_data_orig =
2159 &test_vector.entries[DATA_HARD_OUTPUT];
2161 for (i = 0; i < n; ++i) {
2162 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2163 TEST_ASSERT_SUCCESS(ret,
2164 "Checking status and ordering for encoder failed");
2165 TEST_ASSERT_SUCCESS(validate_op_chain(
2166 &ops[i]->ldpc_enc.output,
2168 "Output buffers (CB=%u) are not equal",
2172 return TEST_SUCCESS;
2176 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2179 struct op_data_entries *entry;
2181 op->turbo_dec = test_vector.turbo_dec;
2182 entry = &test_vector.entries[DATA_INPUT];
2183 for (i = 0; i < entry->nb_segments; ++i)
2184 op->turbo_dec.input.length +=
2185 entry->segments[i].length;
2189 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2192 struct op_data_entries *entry;
2194 op->ldpc_dec = test_vector.ldpc_dec;
2195 entry = &test_vector.entries[DATA_INPUT];
2196 for (i = 0; i < entry->nb_segments; ++i)
2197 op->ldpc_dec.input.length +=
2198 entry->segments[i].length;
2199 if (test_vector.ldpc_dec.op_flags &
2200 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2201 entry = &test_vector.entries[DATA_HARQ_INPUT];
2202 for (i = 0; i < entry->nb_segments; ++i)
2203 op->ldpc_dec.harq_combined_input.length +=
2204 entry->segments[i].length;
2210 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2213 struct op_data_entries *entry;
2215 op->turbo_enc = test_vector.turbo_enc;
2216 entry = &test_vector.entries[DATA_INPUT];
2217 for (i = 0; i < entry->nb_segments; ++i)
2218 op->turbo_enc.input.length +=
2219 entry->segments[i].length;
2223 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2226 struct op_data_entries *entry;
2228 op->ldpc_enc = test_vector.ldpc_enc;
2229 entry = &test_vector.entries[DATA_INPUT];
2230 for (i = 0; i < entry->nb_segments; ++i)
2231 op->ldpc_enc.input.length +=
2232 entry->segments[i].length;
2236 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2239 uint32_t c, r, tb_size = 0;
2241 if (op->turbo_dec.code_block_mode) {
2242 tb_size = op->turbo_dec.tb_params.k_neg;
2244 c = op->turbo_dec.tb_params.c;
2245 r = op->turbo_dec.tb_params.r;
2246 for (i = 0; i < c-r; i++)
2247 tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2248 op->turbo_dec.tb_params.k_neg :
2249 op->turbo_dec.tb_params.k_pos;
2255 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2258 uint32_t c, r, tb_size = 0;
2259 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2261 if (op->ldpc_dec.code_block_mode) {
2262 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2264 c = op->ldpc_dec.tb_params.c;
2265 r = op->ldpc_dec.tb_params.r;
2266 for (i = 0; i < c-r; i++)
2267 tb_size += sys_cols * op->ldpc_dec.z_c
2268 - op->ldpc_dec.n_filler;
2274 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2277 uint32_t c, r, tb_size = 0;
2279 if (op->turbo_enc.code_block_mode) {
2280 tb_size = op->turbo_enc.tb_params.k_neg;
2282 c = op->turbo_enc.tb_params.c;
2283 r = op->turbo_enc.tb_params.r;
2284 for (i = 0; i < c-r; i++)
2285 tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2286 op->turbo_enc.tb_params.k_neg :
2287 op->turbo_enc.tb_params.k_pos;
2293 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2296 uint32_t c, r, tb_size = 0;
2297 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2299 if (op->turbo_enc.code_block_mode) {
2300 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2302 c = op->turbo_enc.tb_params.c;
2303 r = op->turbo_enc.tb_params.r;
2304 for (i = 0; i < c-r; i++)
2305 tb_size += sys_cols * op->ldpc_enc.z_c
2306 - op->ldpc_enc.n_filler;
2313 init_test_op_params(struct test_op_params *op_params,
2314 enum rte_bbdev_op_type op_type, const int expected_status,
2315 const int vector_mask, struct rte_mempool *ops_mp,
2316 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2319 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2320 op_type == RTE_BBDEV_OP_LDPC_DEC)
2321 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2322 &op_params->ref_dec_op, 1);
2324 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2325 &op_params->ref_enc_op, 1);
2327 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2329 op_params->mp = ops_mp;
2330 op_params->burst_sz = burst_sz;
2331 op_params->num_to_process = num_to_process;
2332 op_params->num_lcores = num_lcores;
2333 op_params->vector_mask = vector_mask;
2334 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2335 op_type == RTE_BBDEV_OP_LDPC_DEC)
2336 op_params->ref_dec_op->status = expected_status;
2337 else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2338 || op_type == RTE_BBDEV_OP_LDPC_ENC)
2339 op_params->ref_enc_op->status = expected_status;
2344 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2345 struct test_op_params *op_params)
2347 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2349 struct active_device *ad;
2350 unsigned int burst_sz = get_burst_sz();
2351 enum rte_bbdev_op_type op_type = test_vector.op_type;
2352 const struct rte_bbdev_op_cap *capabilities = NULL;
2354 ad = &active_devs[dev_id];
2356 /* Check if device supports op_type */
2357 if (!is_avail_op(ad, test_vector.op_type))
2358 return TEST_SUCCESS;
2360 struct rte_bbdev_info info;
2361 rte_bbdev_info_get(ad->dev_id, &info);
2362 socket_id = GET_SOCKET(info.socket_id);
2364 f_ret = create_mempools(ad, socket_id, op_type,
2366 if (f_ret != TEST_SUCCESS) {
2367 printf("Couldn't create mempools");
2370 if (op_type == RTE_BBDEV_OP_NONE)
2371 op_type = RTE_BBDEV_OP_TURBO_ENC;
2373 f_ret = init_test_op_params(op_params, test_vector.op_type,
2374 test_vector.expected_status,
2380 if (f_ret != TEST_SUCCESS) {
2381 printf("Couldn't init test op params");
2386 /* Find capabilities */
2387 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2388 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2389 if (cap->type == test_vector.op_type) {
2395 TEST_ASSERT_NOT_NULL(capabilities,
2396 "Couldn't find capabilities");
2398 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2399 create_reference_dec_op(op_params->ref_dec_op);
2400 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2401 create_reference_enc_op(op_params->ref_enc_op);
2402 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2403 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2404 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2405 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2407 for (i = 0; i < ad->nb_queues; ++i) {
2408 f_ret = fill_queue_buffers(op_params,
2410 ad->hard_out_mbuf_pool,
2411 ad->soft_out_mbuf_pool,
2412 ad->harq_in_mbuf_pool,
2413 ad->harq_out_mbuf_pool,
2416 info.drv.min_alignment,
2418 if (f_ret != TEST_SUCCESS) {
2419 printf("Couldn't init queue buffers");
2424 /* Run test case function */
2425 t_ret = test_case_func(ad, op_params);
2427 /* Free active device resources and return */
2428 free_buffers(ad, op_params);
2432 free_buffers(ad, op_params);
2436 /* Run given test function per active device per supported op type
2440 run_test_case(test_case_function *test_case_func)
2445 /* Alloc op_params */
2446 struct test_op_params *op_params = rte_zmalloc(NULL,
2447 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2448 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2449 RTE_ALIGN(sizeof(struct test_op_params),
2450 RTE_CACHE_LINE_SIZE));
2452 /* For each device run test case function */
2453 for (dev = 0; dev < nb_active_devs; ++dev)
2454 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2456 rte_free(op_params);
2462 /* Push back the HARQ output from DDR to host */
2464 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2465 struct rte_bbdev_dec_op **ops,
2469 int save_status, ret;
2470 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2471 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2472 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2473 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2474 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2475 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2476 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2477 for (j = 0; j < n; ++j) {
2478 if ((loopback && mem_out) || hc_out) {
2479 save_status = ops[j]->status;
2480 ops[j]->ldpc_dec.op_flags =
2481 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2482 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2484 ops[j]->ldpc_dec.op_flags +=
2485 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2486 ops[j]->ldpc_dec.harq_combined_input.offset =
2488 ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2489 harq_offset += HARQ_INCR;
2491 ops[j]->ldpc_dec.harq_combined_input.length =
2492 ops[j]->ldpc_dec.harq_combined_output.length;
2493 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2497 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2500 ops[j]->ldpc_dec.op_flags = flags;
2501 ops[j]->status = save_status;
2507 * Push back the HARQ output from HW DDR to Host
2508 * Preload HARQ memory input and adjust HARQ offset
2511 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2512 struct rte_bbdev_dec_op **ops, const uint16_t n,
2517 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2518 struct rte_bbdev_op_data save_hc_in, save_hc_out;
2519 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2520 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2521 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2522 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2523 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2524 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2525 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2526 for (j = 0; j < n; ++j) {
2527 if ((mem_in || hc_in) && preload) {
2528 save_hc_in = ops[j]->ldpc_dec.harq_combined_input;
2529 save_hc_out = ops[j]->ldpc_dec.harq_combined_output;
2530 ops[j]->ldpc_dec.op_flags =
2531 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2532 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2534 ops[j]->ldpc_dec.op_flags +=
2535 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2536 ops[j]->ldpc_dec.harq_combined_output.offset =
2538 ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2539 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2543 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2544 dev_id, queue_id, &ops_deq[j], 1);
2545 ops[j]->ldpc_dec.op_flags = flags;
2546 ops[j]->ldpc_dec.harq_combined_input = save_hc_in;
2547 ops[j]->ldpc_dec.harq_combined_output = save_hc_out;
2549 /* Adjust HARQ offset when we reach external DDR */
2550 if (mem_in || hc_in)
2551 ops[j]->ldpc_dec.harq_combined_input.offset
2553 if (mem_out || hc_out)
2554 ops[j]->ldpc_dec.harq_combined_output.offset
2556 harq_offset += HARQ_INCR;
2561 dequeue_event_callback(uint16_t dev_id,
2562 enum rte_bbdev_event_type event, void *cb_arg,
2567 uint64_t total_time;
2568 uint16_t deq, burst_sz, num_ops;
2569 uint16_t queue_id = *(uint16_t *) ret_param;
2570 struct rte_bbdev_info info;
2572 struct thread_params *tp = cb_arg;
2574 /* Find matching thread params using queue_id */
2575 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2576 if (tp->queue_id == queue_id)
2579 if (i == MAX_QUEUES) {
2580 printf("%s: Queue_id from interrupt details was not found!\n",
2585 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2586 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2588 "Dequeue interrupt handler called for incorrect event!\n");
2592 burst_sz = rte_atomic16_read(&tp->burst_sz);
2593 num_ops = tp->op_params->num_to_process;
2595 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2596 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2598 rte_atomic16_read(&tp->nb_dequeued)],
2600 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2601 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2603 rte_atomic16_read(&tp->nb_dequeued)],
2605 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2606 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2608 rte_atomic16_read(&tp->nb_dequeued)],
2610 else /*RTE_BBDEV_OP_TURBO_ENC*/
2611 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2613 rte_atomic16_read(&tp->nb_dequeued)],
2616 if (deq < burst_sz) {
2618 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2620 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2624 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2625 rte_atomic16_add(&tp->nb_dequeued, deq);
2629 total_time = rte_rdtsc_precise() - tp->start_time;
2631 rte_bbdev_info_get(dev_id, &info);
2635 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2636 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2637 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2638 tp->op_params->vector_mask);
2639 /* get the max of iter_count for all dequeued ops */
2640 for (i = 0; i < num_ops; ++i)
2641 tp->iter_count = RTE_MAX(
2642 tp->dec_ops[i]->turbo_dec.iter_count,
2644 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2645 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2646 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2647 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2648 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2649 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2650 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2651 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2652 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2653 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2654 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2655 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2656 tp->op_params->vector_mask);
2657 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2661 printf("Buffers validation failed\n");
2662 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2665 switch (test_vector.op_type) {
2666 case RTE_BBDEV_OP_TURBO_DEC:
2667 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2669 case RTE_BBDEV_OP_TURBO_ENC:
2670 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2672 case RTE_BBDEV_OP_LDPC_DEC:
2673 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2675 case RTE_BBDEV_OP_LDPC_ENC:
2676 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2678 case RTE_BBDEV_OP_NONE:
2682 printf("Unknown op type: %d\n", test_vector.op_type);
2683 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2687 tp->ops_per_sec += ((double)num_ops) /
2688 ((double)total_time / (double)rte_get_tsc_hz());
2689 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2690 ((double)total_time / (double)rte_get_tsc_hz());
2692 rte_atomic16_add(&tp->nb_dequeued, deq);
2696 throughput_intr_lcore_ldpc_dec(void *arg)
2698 struct thread_params *tp = arg;
2699 unsigned int enqueued;
2700 const uint16_t queue_id = tp->queue_id;
2701 const uint16_t burst_sz = tp->op_params->burst_sz;
2702 const uint16_t num_to_process = tp->op_params->num_to_process;
2703 struct rte_bbdev_dec_op *ops[num_to_process];
2704 struct test_buffers *bufs = NULL;
2705 struct rte_bbdev_info info;
2707 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2708 uint16_t num_to_enq, enq;
2710 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2711 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2712 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2713 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2715 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2716 "BURST_SIZE should be <= %u", MAX_BURST);
2718 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2719 "Failed to enable interrupts for dev: %u, queue_id: %u",
2720 tp->dev_id, queue_id);
2722 rte_bbdev_info_get(tp->dev_id, &info);
2724 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2725 "NUM_OPS cannot exceed %u for this device",
2726 info.drv.queue_size_lim);
2728 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2730 rte_atomic16_clear(&tp->processing_status);
2731 rte_atomic16_clear(&tp->nb_dequeued);
2733 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2736 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2738 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2740 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2741 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2742 bufs->hard_outputs, bufs->soft_outputs,
2743 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2745 /* Set counter to validate the ordering */
2746 for (j = 0; j < num_to_process; ++j)
2747 ops[j]->opaque_data = (void *)(uintptr_t)j;
2749 for (j = 0; j < TEST_REPETITIONS; ++j) {
2750 for (i = 0; i < num_to_process; ++i) {
2753 ops[i]->ldpc_dec.hard_output.data);
2754 if (hc_out || loopback)
2756 ops[i]->ldpc_dec.harq_combined_output.data);
2759 tp->start_time = rte_rdtsc_precise();
2760 for (enqueued = 0; enqueued < num_to_process;) {
2761 num_to_enq = burst_sz;
2763 if (unlikely(num_to_process - enqueued < num_to_enq))
2764 num_to_enq = num_to_process - enqueued;
2768 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2770 queue_id, &ops[enqueued],
2772 } while (unlikely(num_to_enq != enq));
2775 /* Write to thread burst_sz current number of enqueued
2776 * descriptors. It ensures that proper number of
2777 * descriptors will be dequeued in callback
2778 * function - needed for last batch in case where
2779 * the number of operations is not a multiple of
2782 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2784 /* Wait until processing of previous batch is
2787 while (rte_atomic16_read(&tp->nb_dequeued) !=
2791 if (j != TEST_REPETITIONS - 1)
2792 rte_atomic16_clear(&tp->nb_dequeued);
2795 return TEST_SUCCESS;
2799 throughput_intr_lcore_dec(void *arg)
2801 struct thread_params *tp = arg;
2802 unsigned int enqueued;
2803 const uint16_t queue_id = tp->queue_id;
2804 const uint16_t burst_sz = tp->op_params->burst_sz;
2805 const uint16_t num_to_process = tp->op_params->num_to_process;
2806 struct rte_bbdev_dec_op *ops[num_to_process];
2807 struct test_buffers *bufs = NULL;
2808 struct rte_bbdev_info info;
2810 uint16_t num_to_enq, enq;
2812 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2813 "BURST_SIZE should be <= %u", MAX_BURST);
2815 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2816 "Failed to enable interrupts for dev: %u, queue_id: %u",
2817 tp->dev_id, queue_id);
2819 rte_bbdev_info_get(tp->dev_id, &info);
2821 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2822 "NUM_OPS cannot exceed %u for this device",
2823 info.drv.queue_size_lim);
2825 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2827 rte_atomic16_clear(&tp->processing_status);
2828 rte_atomic16_clear(&tp->nb_dequeued);
2830 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2833 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2835 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2837 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2838 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2839 bufs->hard_outputs, bufs->soft_outputs,
2840 tp->op_params->ref_dec_op);
2842 /* Set counter to validate the ordering */
2843 for (j = 0; j < num_to_process; ++j)
2844 ops[j]->opaque_data = (void *)(uintptr_t)j;
2846 for (j = 0; j < TEST_REPETITIONS; ++j) {
2847 for (i = 0; i < num_to_process; ++i)
2848 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2850 tp->start_time = rte_rdtsc_precise();
2851 for (enqueued = 0; enqueued < num_to_process;) {
2852 num_to_enq = burst_sz;
2854 if (unlikely(num_to_process - enqueued < num_to_enq))
2855 num_to_enq = num_to_process - enqueued;
2859 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2860 queue_id, &ops[enqueued],
2862 } while (unlikely(num_to_enq != enq));
2865 /* Write to thread burst_sz current number of enqueued
2866 * descriptors. It ensures that proper number of
2867 * descriptors will be dequeued in callback
2868 * function - needed for last batch in case where
2869 * the number of operations is not a multiple of
2872 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2874 /* Wait until processing of previous batch is
2877 while (rte_atomic16_read(&tp->nb_dequeued) !=
2881 if (j != TEST_REPETITIONS - 1)
2882 rte_atomic16_clear(&tp->nb_dequeued);
2885 return TEST_SUCCESS;
2889 throughput_intr_lcore_enc(void *arg)
2891 struct thread_params *tp = arg;
2892 unsigned int enqueued;
2893 const uint16_t queue_id = tp->queue_id;
2894 const uint16_t burst_sz = tp->op_params->burst_sz;
2895 const uint16_t num_to_process = tp->op_params->num_to_process;
2896 struct rte_bbdev_enc_op *ops[num_to_process];
2897 struct test_buffers *bufs = NULL;
2898 struct rte_bbdev_info info;
2900 uint16_t num_to_enq, enq;
2902 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2903 "BURST_SIZE should be <= %u", MAX_BURST);
2905 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2906 "Failed to enable interrupts for dev: %u, queue_id: %u",
2907 tp->dev_id, queue_id);
2909 rte_bbdev_info_get(tp->dev_id, &info);
2911 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2912 "NUM_OPS cannot exceed %u for this device",
2913 info.drv.queue_size_lim);
2915 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2917 rte_atomic16_clear(&tp->processing_status);
2918 rte_atomic16_clear(&tp->nb_dequeued);
2920 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2923 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2925 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2927 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2928 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2929 bufs->hard_outputs, tp->op_params->ref_enc_op);
2931 /* Set counter to validate the ordering */
2932 for (j = 0; j < num_to_process; ++j)
2933 ops[j]->opaque_data = (void *)(uintptr_t)j;
2935 for (j = 0; j < TEST_REPETITIONS; ++j) {
2936 for (i = 0; i < num_to_process; ++i)
2937 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2939 tp->start_time = rte_rdtsc_precise();
2940 for (enqueued = 0; enqueued < num_to_process;) {
2941 num_to_enq = burst_sz;
2943 if (unlikely(num_to_process - enqueued < num_to_enq))
2944 num_to_enq = num_to_process - enqueued;
2948 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2949 queue_id, &ops[enqueued],
2951 } while (unlikely(enq != num_to_enq));
2954 /* Write to thread burst_sz current number of enqueued
2955 * descriptors. It ensures that proper number of
2956 * descriptors will be dequeued in callback
2957 * function - needed for last batch in case where
2958 * the number of operations is not a multiple of
2961 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2963 /* Wait until processing of previous batch is
2966 while (rte_atomic16_read(&tp->nb_dequeued) !=
2970 if (j != TEST_REPETITIONS - 1)
2971 rte_atomic16_clear(&tp->nb_dequeued);
2974 return TEST_SUCCESS;
2979 throughput_intr_lcore_ldpc_enc(void *arg)
2981 struct thread_params *tp = arg;
2982 unsigned int enqueued;
2983 const uint16_t queue_id = tp->queue_id;
2984 const uint16_t burst_sz = tp->op_params->burst_sz;
2985 const uint16_t num_to_process = tp->op_params->num_to_process;
2986 struct rte_bbdev_enc_op *ops[num_to_process];
2987 struct test_buffers *bufs = NULL;
2988 struct rte_bbdev_info info;
2990 uint16_t num_to_enq, enq;
2992 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2993 "BURST_SIZE should be <= %u", MAX_BURST);
2995 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2996 "Failed to enable interrupts for dev: %u, queue_id: %u",
2997 tp->dev_id, queue_id);
2999 rte_bbdev_info_get(tp->dev_id, &info);
3001 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
3002 "NUM_OPS cannot exceed %u for this device",
3003 info.drv.queue_size_lim);
3005 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3007 rte_atomic16_clear(&tp->processing_status);
3008 rte_atomic16_clear(&tp->nb_dequeued);
3010 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3013 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
3015 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3017 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3018 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
3019 bufs->inputs, bufs->hard_outputs,
3020 tp->op_params->ref_enc_op);
3022 /* Set counter to validate the ordering */
3023 for (j = 0; j < num_to_process; ++j)
3024 ops[j]->opaque_data = (void *)(uintptr_t)j;
3026 for (j = 0; j < TEST_REPETITIONS; ++j) {
3027 for (i = 0; i < num_to_process; ++i)
3028 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
3030 tp->start_time = rte_rdtsc_precise();
3031 for (enqueued = 0; enqueued < num_to_process;) {
3032 num_to_enq = burst_sz;
3034 if (unlikely(num_to_process - enqueued < num_to_enq))
3035 num_to_enq = num_to_process - enqueued;
3039 enq += rte_bbdev_enqueue_ldpc_enc_ops(
3041 queue_id, &ops[enqueued],
3043 } while (unlikely(enq != num_to_enq));
3046 /* Write to thread burst_sz current number of enqueued
3047 * descriptors. It ensures that proper number of
3048 * descriptors will be dequeued in callback
3049 * function - needed for last batch in case where
3050 * the number of operations is not a multiple of
3053 rte_atomic16_set(&tp->burst_sz, num_to_enq);
3055 /* Wait until processing of previous batch is
3058 while (rte_atomic16_read(&tp->nb_dequeued) !=
3062 if (j != TEST_REPETITIONS - 1)
3063 rte_atomic16_clear(&tp->nb_dequeued);
3066 return TEST_SUCCESS;
3070 throughput_pmd_lcore_dec(void *arg)
3072 struct thread_params *tp = arg;
3074 uint64_t total_time = 0, start_time;
3075 const uint16_t queue_id = tp->queue_id;
3076 const uint16_t burst_sz = tp->op_params->burst_sz;
3077 const uint16_t num_ops = tp->op_params->num_to_process;
3078 struct rte_bbdev_dec_op *ops_enq[num_ops];
3079 struct rte_bbdev_dec_op *ops_deq[num_ops];
3080 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3081 struct test_buffers *bufs = NULL;
3083 struct rte_bbdev_info info;
3084 uint16_t num_to_enq;
3086 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3087 "BURST_SIZE should be <= %u", MAX_BURST);
3089 rte_bbdev_info_get(tp->dev_id, &info);
3091 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3092 "NUM_OPS cannot exceed %u for this device",
3093 info.drv.queue_size_lim);
3095 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3097 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3100 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3101 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3103 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3104 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3105 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3107 /* Set counter to validate the ordering */
3108 for (j = 0; j < num_ops; ++j)
3109 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3111 for (i = 0; i < TEST_REPETITIONS; ++i) {
3113 for (j = 0; j < num_ops; ++j)
3114 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3116 start_time = rte_rdtsc_precise();
3118 for (enq = 0, deq = 0; enq < num_ops;) {
3119 num_to_enq = burst_sz;
3121 if (unlikely(num_ops - enq < num_to_enq))
3122 num_to_enq = num_ops - enq;
3124 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3125 queue_id, &ops_enq[enq], num_to_enq);
3127 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3128 queue_id, &ops_deq[deq], enq - deq);
3131 /* dequeue the remaining */
3133 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3134 queue_id, &ops_deq[deq], enq - deq);
3137 total_time += rte_rdtsc_precise() - start_time;
3141 /* get the max of iter_count for all dequeued ops */
3142 for (i = 0; i < num_ops; ++i) {
3143 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3147 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3148 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3149 tp->op_params->vector_mask);
3150 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3153 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3155 double tb_len_bits = calc_dec_TB_size(ref_op);
3157 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3158 ((double)total_time / (double)rte_get_tsc_hz());
3159 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3160 1000000.0) / ((double)total_time /
3161 (double)rte_get_tsc_hz());
3163 return TEST_SUCCESS;
3167 bler_pmd_lcore_ldpc_dec(void *arg)
3169 struct thread_params *tp = arg;
3171 uint64_t total_time = 0, start_time;
3172 const uint16_t queue_id = tp->queue_id;
3173 const uint16_t burst_sz = tp->op_params->burst_sz;
3174 const uint16_t num_ops = tp->op_params->num_to_process;
3175 struct rte_bbdev_dec_op *ops_enq[num_ops];
3176 struct rte_bbdev_dec_op *ops_deq[num_ops];
3177 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3178 struct test_buffers *bufs = NULL;
3180 float parity_bler = 0;
3181 struct rte_bbdev_info info;
3182 uint16_t num_to_enq;
3183 bool extDdr = check_bit(ldpc_cap_flags,
3184 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3185 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3186 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3187 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3188 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3190 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3191 "BURST_SIZE should be <= %u", MAX_BURST);
3193 rte_bbdev_info_get(tp->dev_id, &info);
3195 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3196 "NUM_OPS cannot exceed %u for this device",
3197 info.drv.queue_size_lim);
3199 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3201 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3204 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3205 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3207 /* For BLER tests we need to enable early termination */
3208 if (!check_bit(ref_op->ldpc_dec.op_flags,
3209 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3210 ref_op->ldpc_dec.op_flags +=
3211 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3212 ref_op->ldpc_dec.iter_max = get_iter_max();
3213 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3215 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3216 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3217 bufs->hard_outputs, bufs->soft_outputs,
3218 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3219 generate_llr_input(num_ops, bufs->inputs, ref_op);
3221 /* Set counter to validate the ordering */
3222 for (j = 0; j < num_ops; ++j)
3223 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3225 for (i = 0; i < 1; ++i) { /* Could add more iterations */
3226 for (j = 0; j < num_ops; ++j) {
3229 ops_enq[j]->ldpc_dec.hard_output.data);
3230 if (hc_out || loopback)
3232 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3235 bool preload = i == (TEST_REPETITIONS - 1);
3236 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3239 start_time = rte_rdtsc_precise();
3241 for (enq = 0, deq = 0; enq < num_ops;) {
3242 num_to_enq = burst_sz;
3244 if (unlikely(num_ops - enq < num_to_enq))
3245 num_to_enq = num_ops - enq;
3247 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3248 queue_id, &ops_enq[enq], num_to_enq);
3250 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3251 queue_id, &ops_deq[deq], enq - deq);
3254 /* dequeue the remaining */
3256 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3257 queue_id, &ops_deq[deq], enq - deq);
3260 total_time += rte_rdtsc_precise() - start_time;
3264 tp->iter_average = 0;
3265 /* get the max of iter_count for all dequeued ops */
3266 for (i = 0; i < num_ops; ++i) {
3267 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3269 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3270 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3274 parity_bler /= num_ops; /* This one is based on SYND */
3275 tp->iter_average /= num_ops;
3276 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3278 if (test_vector.op_type != RTE_BBDEV_OP_NONE
3282 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3283 tp->op_params->vector_mask);
3284 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3287 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3289 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3290 tp->ops_per_sec = ((double)num_ops * 1) /
3291 ((double)total_time / (double)rte_get_tsc_hz());
3292 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3293 1000000.0) / ((double)total_time /
3294 (double)rte_get_tsc_hz());
3296 return TEST_SUCCESS;
3300 throughput_pmd_lcore_ldpc_dec(void *arg)
3302 struct thread_params *tp = arg;
3304 uint64_t total_time = 0, start_time;
3305 const uint16_t queue_id = tp->queue_id;
3306 const uint16_t burst_sz = tp->op_params->burst_sz;
3307 const uint16_t num_ops = tp->op_params->num_to_process;
3308 struct rte_bbdev_dec_op *ops_enq[num_ops];
3309 struct rte_bbdev_dec_op *ops_deq[num_ops];
3310 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3311 struct test_buffers *bufs = NULL;
3313 struct rte_bbdev_info info;
3314 uint16_t num_to_enq;
3315 bool extDdr = check_bit(ldpc_cap_flags,
3316 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3317 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3318 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3319 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3320 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3322 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3323 "BURST_SIZE should be <= %u", MAX_BURST);
3325 rte_bbdev_info_get(tp->dev_id, &info);
3327 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3328 "NUM_OPS cannot exceed %u for this device",
3329 info.drv.queue_size_lim);
3331 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3333 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3336 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3337 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3339 /* For throughput tests we need to disable early termination */
3340 if (check_bit(ref_op->ldpc_dec.op_flags,
3341 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3342 ref_op->ldpc_dec.op_flags -=
3343 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3344 ref_op->ldpc_dec.iter_max = get_iter_max();
3345 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3347 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3348 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3349 bufs->hard_outputs, bufs->soft_outputs,
3350 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3352 /* Set counter to validate the ordering */
3353 for (j = 0; j < num_ops; ++j)
3354 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3356 for (i = 0; i < TEST_REPETITIONS; ++i) {
3357 for (j = 0; j < num_ops; ++j) {
3360 ops_enq[j]->ldpc_dec.hard_output.data);
3361 if (hc_out || loopback)
3363 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3366 bool preload = i == (TEST_REPETITIONS - 1);
3367 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3370 start_time = rte_rdtsc_precise();
3372 for (enq = 0, deq = 0; enq < num_ops;) {
3373 num_to_enq = burst_sz;
3375 if (unlikely(num_ops - enq < num_to_enq))
3376 num_to_enq = num_ops - enq;
3378 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3379 queue_id, &ops_enq[enq], num_to_enq);
3381 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3382 queue_id, &ops_deq[deq], enq - deq);
3385 /* dequeue the remaining */
3387 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3388 queue_id, &ops_deq[deq], enq - deq);
3391 total_time += rte_rdtsc_precise() - start_time;
3395 /* get the max of iter_count for all dequeued ops */
3396 for (i = 0; i < num_ops; ++i) {
3397 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3401 /* Read loopback is not thread safe */
3402 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3405 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3406 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3407 tp->op_params->vector_mask);
3408 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3411 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3413 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3415 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3416 ((double)total_time / (double)rte_get_tsc_hz());
3417 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3418 1000000.0) / ((double)total_time /
3419 (double)rte_get_tsc_hz());
3421 return TEST_SUCCESS;
3425 throughput_pmd_lcore_enc(void *arg)
3427 struct thread_params *tp = arg;
3429 uint64_t total_time = 0, start_time;
3430 const uint16_t queue_id = tp->queue_id;
3431 const uint16_t burst_sz = tp->op_params->burst_sz;
3432 const uint16_t num_ops = tp->op_params->num_to_process;
3433 struct rte_bbdev_enc_op *ops_enq[num_ops];
3434 struct rte_bbdev_enc_op *ops_deq[num_ops];
3435 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3436 struct test_buffers *bufs = NULL;
3438 struct rte_bbdev_info info;
3439 uint16_t num_to_enq;
3441 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3442 "BURST_SIZE should be <= %u", MAX_BURST);
3444 rte_bbdev_info_get(tp->dev_id, &info);
3446 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3447 "NUM_OPS cannot exceed %u for this device",
3448 info.drv.queue_size_lim);
3450 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3452 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3455 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3457 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3459 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3460 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3461 bufs->hard_outputs, ref_op);
3463 /* Set counter to validate the ordering */
3464 for (j = 0; j < num_ops; ++j)
3465 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3467 for (i = 0; i < TEST_REPETITIONS; ++i) {
3469 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3470 for (j = 0; j < num_ops; ++j)
3471 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3473 start_time = rte_rdtsc_precise();
3475 for (enq = 0, deq = 0; enq < num_ops;) {
3476 num_to_enq = burst_sz;
3478 if (unlikely(num_ops - enq < num_to_enq))
3479 num_to_enq = num_ops - enq;
3481 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3482 queue_id, &ops_enq[enq], num_to_enq);
3484 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3485 queue_id, &ops_deq[deq], enq - deq);
3488 /* dequeue the remaining */
3490 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3491 queue_id, &ops_deq[deq], enq - deq);
3494 total_time += rte_rdtsc_precise() - start_time;
3497 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3498 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3499 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3502 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3504 double tb_len_bits = calc_enc_TB_size(ref_op);
3506 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3507 ((double)total_time / (double)rte_get_tsc_hz());
3508 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3509 / 1000000.0) / ((double)total_time /
3510 (double)rte_get_tsc_hz());
3512 return TEST_SUCCESS;
3516 throughput_pmd_lcore_ldpc_enc(void *arg)
3518 struct thread_params *tp = arg;
3520 uint64_t total_time = 0, start_time;
3521 const uint16_t queue_id = tp->queue_id;
3522 const uint16_t burst_sz = tp->op_params->burst_sz;
3523 const uint16_t num_ops = tp->op_params->num_to_process;
3524 struct rte_bbdev_enc_op *ops_enq[num_ops];
3525 struct rte_bbdev_enc_op *ops_deq[num_ops];
3526 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3527 struct test_buffers *bufs = NULL;
3529 struct rte_bbdev_info info;
3530 uint16_t num_to_enq;
3532 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3533 "BURST_SIZE should be <= %u", MAX_BURST);
3535 rte_bbdev_info_get(tp->dev_id, &info);
3537 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3538 "NUM_OPS cannot exceed %u for this device",
3539 info.drv.queue_size_lim);
3541 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3543 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3546 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3548 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3550 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3551 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3552 bufs->hard_outputs, ref_op);
3554 /* Set counter to validate the ordering */
3555 for (j = 0; j < num_ops; ++j)
3556 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3558 for (i = 0; i < TEST_REPETITIONS; ++i) {
3560 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3561 for (j = 0; j < num_ops; ++j)
3562 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3564 start_time = rte_rdtsc_precise();
3566 for (enq = 0, deq = 0; enq < num_ops;) {
3567 num_to_enq = burst_sz;
3569 if (unlikely(num_ops - enq < num_to_enq))
3570 num_to_enq = num_ops - enq;
3572 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3573 queue_id, &ops_enq[enq], num_to_enq);
3575 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3576 queue_id, &ops_deq[deq], enq - deq);
3579 /* dequeue the remaining */
3581 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3582 queue_id, &ops_deq[deq], enq - deq);
3585 total_time += rte_rdtsc_precise() - start_time;
3588 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3589 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3590 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3593 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3595 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3597 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3598 ((double)total_time / (double)rte_get_tsc_hz());
3599 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3600 / 1000000.0) / ((double)total_time /
3601 (double)rte_get_tsc_hz());
3603 return TEST_SUCCESS;
3607 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3609 unsigned int iter = 0;
3610 double total_mops = 0, total_mbps = 0;
3612 for (iter = 0; iter < used_cores; iter++) {
3614 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3615 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3616 t_params[iter].mbps);
3617 total_mops += t_params[iter].ops_per_sec;
3618 total_mbps += t_params[iter].mbps;
3621 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3622 used_cores, total_mops, total_mbps);
3625 /* Aggregate the performance results over the number of cores used */
3627 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3629 unsigned int core_idx = 0;
3630 double total_mops = 0, total_mbps = 0;
3631 uint8_t iter_count = 0;
3633 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3635 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3636 t_params[core_idx].lcore_id,
3637 t_params[core_idx].ops_per_sec,
3638 t_params[core_idx].mbps,
3639 t_params[core_idx].iter_count);
3640 total_mops += t_params[core_idx].ops_per_sec;
3641 total_mbps += t_params[core_idx].mbps;
3642 iter_count = RTE_MAX(iter_count,
3643 t_params[core_idx].iter_count);
3646 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3647 used_cores, total_mops, total_mbps, iter_count);
3650 /* Aggregate the performance results over the number of cores used */
3652 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3654 unsigned int core_idx = 0;
3655 double total_mbps = 0, total_bler = 0, total_iter = 0;
3656 double snr = get_snr();
3658 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3659 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3660 t_params[core_idx].lcore_id,
3661 t_params[core_idx].bler * 100,
3662 t_params[core_idx].iter_average,
3663 t_params[core_idx].mbps,
3664 get_vector_filename());
3665 total_mbps += t_params[core_idx].mbps;
3666 total_bler += t_params[core_idx].bler;
3667 total_iter += t_params[core_idx].iter_average;
3669 total_bler /= used_cores;
3670 total_iter /= used_cores;
3672 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3673 snr, total_bler * 100, total_iter, get_iter_max(),
3674 total_mbps, get_vector_filename());
3678 * Test function that determines BLER wireless performance
3681 bler_test(struct active_device *ad,
3682 struct test_op_params *op_params)
3685 unsigned int lcore_id, used_cores = 0;
3686 struct thread_params *t_params;
3687 struct rte_bbdev_info info;
3688 lcore_function_t *bler_function;
3689 uint16_t num_lcores;
3690 const char *op_type_str;
3692 rte_bbdev_info_get(ad->dev_id, &info);
3694 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3695 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3696 test_vector.op_type);
3698 printf("+ ------------------------------------------------------- +\n");
3699 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3700 info.dev_name, ad->nb_queues, op_params->burst_sz,
3701 op_params->num_to_process, op_params->num_lcores,
3703 intr_enabled ? "Interrupt mode" : "PMD mode",
3704 (double)rte_get_tsc_hz() / 1000000000.0);
3706 /* Set number of lcores */
3707 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3709 : op_params->num_lcores;
3711 /* Allocate memory for thread parameters structure */
3712 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3713 RTE_CACHE_LINE_SIZE);
3714 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3715 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3716 RTE_CACHE_LINE_SIZE));
3718 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3719 bler_function = bler_pmd_lcore_ldpc_dec;
3721 return TEST_SKIPPED;
3723 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3725 /* Main core is set at first entry */
3726 t_params[0].dev_id = ad->dev_id;
3727 t_params[0].lcore_id = rte_lcore_id();
3728 t_params[0].op_params = op_params;
3729 t_params[0].queue_id = ad->queue_ids[used_cores++];
3730 t_params[0].iter_count = 0;
3732 RTE_LCORE_FOREACH_WORKER(lcore_id) {
3733 if (used_cores >= num_lcores)
3736 t_params[used_cores].dev_id = ad->dev_id;
3737 t_params[used_cores].lcore_id = lcore_id;
3738 t_params[used_cores].op_params = op_params;
3739 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3740 t_params[used_cores].iter_count = 0;
3742 rte_eal_remote_launch(bler_function,
3743 &t_params[used_cores++], lcore_id);
3746 rte_atomic16_set(&op_params->sync, SYNC_START);
3747 ret = bler_function(&t_params[0]);
3749 /* Main core is always used */
3750 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3751 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3753 print_dec_bler(t_params, num_lcores);
3755 /* Return if test failed */
3761 /* Function to print something here*/
3767 * Test function that determines how long an enqueue + dequeue of a burst
3768 * takes on available lcores.
3771 throughput_test(struct active_device *ad,
3772 struct test_op_params *op_params)
3775 unsigned int lcore_id, used_cores = 0;
3776 struct thread_params *t_params, *tp;
3777 struct rte_bbdev_info info;
3778 lcore_function_t *throughput_function;
3779 uint16_t num_lcores;
3780 const char *op_type_str;
3782 rte_bbdev_info_get(ad->dev_id, &info);
3784 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3785 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3786 test_vector.op_type);
3788 printf("+ ------------------------------------------------------- +\n");
3789 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3790 info.dev_name, ad->nb_queues, op_params->burst_sz,
3791 op_params->num_to_process, op_params->num_lcores,
3793 intr_enabled ? "Interrupt mode" : "PMD mode",
3794 (double)rte_get_tsc_hz() / 1000000000.0);
3796 /* Set number of lcores */
3797 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3799 : op_params->num_lcores;
3801 /* Allocate memory for thread parameters structure */
3802 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3803 RTE_CACHE_LINE_SIZE);
3804 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3805 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3806 RTE_CACHE_LINE_SIZE));
3809 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3810 throughput_function = throughput_intr_lcore_dec;
3811 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3812 throughput_function = throughput_intr_lcore_ldpc_dec;
3813 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3814 throughput_function = throughput_intr_lcore_enc;
3815 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3816 throughput_function = throughput_intr_lcore_ldpc_enc;
3818 throughput_function = throughput_intr_lcore_enc;
3820 /* Dequeue interrupt callback registration */
3821 ret = rte_bbdev_callback_register(ad->dev_id,
3822 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3829 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3830 throughput_function = throughput_pmd_lcore_dec;
3831 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3832 throughput_function = throughput_pmd_lcore_ldpc_dec;
3833 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3834 throughput_function = throughput_pmd_lcore_enc;
3835 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3836 throughput_function = throughput_pmd_lcore_ldpc_enc;
3838 throughput_function = throughput_pmd_lcore_enc;
3841 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3843 /* Main core is set at first entry */
3844 t_params[0].dev_id = ad->dev_id;
3845 t_params[0].lcore_id = rte_lcore_id();
3846 t_params[0].op_params = op_params;
3847 t_params[0].queue_id = ad->queue_ids[used_cores++];
3848 t_params[0].iter_count = 0;
3850 RTE_LCORE_FOREACH_WORKER(lcore_id) {
3851 if (used_cores >= num_lcores)
3854 t_params[used_cores].dev_id = ad->dev_id;
3855 t_params[used_cores].lcore_id = lcore_id;
3856 t_params[used_cores].op_params = op_params;
3857 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3858 t_params[used_cores].iter_count = 0;
3860 rte_eal_remote_launch(throughput_function,
3861 &t_params[used_cores++], lcore_id);
3864 rte_atomic16_set(&op_params->sync, SYNC_START);
3865 ret = throughput_function(&t_params[0]);
3867 /* Main core is always used */
3868 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3869 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3871 /* Return if test failed */
3877 /* Print throughput if interrupts are disabled and test passed */
3878 if (!intr_enabled) {
3879 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3880 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3881 print_dec_throughput(t_params, num_lcores);
3883 print_enc_throughput(t_params, num_lcores);
3888 /* In interrupt TC we need to wait for the interrupt callback to deqeue
3889 * all pending operations. Skip waiting for queues which reported an
3890 * error using processing_status variable.
3891 * Wait for main lcore operations.
3894 while ((rte_atomic16_read(&tp->nb_dequeued) <
3895 op_params->num_to_process) &&
3896 (rte_atomic16_read(&tp->processing_status) !=
3900 tp->ops_per_sec /= TEST_REPETITIONS;
3901 tp->mbps /= TEST_REPETITIONS;
3902 ret |= (int)rte_atomic16_read(&tp->processing_status);
3904 /* Wait for worker lcores operations */
3905 for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3906 tp = &t_params[used_cores];
3908 while ((rte_atomic16_read(&tp->nb_dequeued) <
3909 op_params->num_to_process) &&
3910 (rte_atomic16_read(&tp->processing_status) !=
3914 tp->ops_per_sec /= TEST_REPETITIONS;
3915 tp->mbps /= TEST_REPETITIONS;
3916 ret |= (int)rte_atomic16_read(&tp->processing_status);
3919 /* Print throughput if test passed */
3921 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3922 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3923 print_dec_throughput(t_params, num_lcores);
3924 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3925 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3926 print_enc_throughput(t_params, num_lcores);
3934 latency_test_dec(struct rte_mempool *mempool,
3935 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3936 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3937 const uint16_t num_to_process, uint16_t burst_sz,
3938 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3940 int ret = TEST_SUCCESS;
3941 uint16_t i, j, dequeued;
3942 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3943 uint64_t start_time = 0, last_time = 0;
3945 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3946 uint16_t enq = 0, deq = 0;
3947 bool first_time = true;
3950 if (unlikely(num_to_process - dequeued < burst_sz))
3951 burst_sz = num_to_process - dequeued;
3953 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3954 TEST_ASSERT_SUCCESS(ret,
3955 "rte_bbdev_dec_op_alloc_bulk() failed");
3956 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3957 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3963 /* Set counter to validate the ordering */
3964 for (j = 0; j < burst_sz; ++j)
3965 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3967 start_time = rte_rdtsc_precise();
3969 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
3971 TEST_ASSERT(enq == burst_sz,
3972 "Error enqueueing burst, expected %u, got %u",
3977 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3978 &ops_deq[deq], burst_sz - deq);
3979 if (likely(first_time && (deq > 0))) {
3980 last_time = rte_rdtsc_precise() - start_time;
3983 } while (unlikely(burst_sz != deq));
3985 *max_time = RTE_MAX(*max_time, last_time);
3986 *min_time = RTE_MIN(*min_time, last_time);
3987 *total_time += last_time;
3989 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3990 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
3992 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3995 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4002 /* Test case for latency/validation for LDPC Decoder */
4004 latency_test_ldpc_dec(struct rte_mempool *mempool,
4005 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
4006 int vector_mask, uint16_t dev_id, uint16_t queue_id,
4007 const uint16_t num_to_process, uint16_t burst_sz,
4008 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time,
4011 int ret = TEST_SUCCESS;
4012 uint16_t i, j, dequeued;
4013 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4014 uint64_t start_time = 0, last_time = 0;
4015 bool extDdr = ldpc_cap_flags &
4016 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4018 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4019 uint16_t enq = 0, deq = 0;
4020 bool first_time = true;
4023 if (unlikely(num_to_process - dequeued < burst_sz))
4024 burst_sz = num_to_process - dequeued;
4026 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4027 TEST_ASSERT_SUCCESS(ret,
4028 "rte_bbdev_dec_op_alloc_bulk() failed");
4030 /* For latency tests we need to disable early termination */
4031 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags,
4032 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
4033 ref_op->ldpc_dec.op_flags -=
4034 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
4035 ref_op->ldpc_dec.iter_max = get_iter_max();
4036 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
4038 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4039 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4048 preload_harq_ddr(dev_id, queue_id, ops_enq,
4051 /* Set counter to validate the ordering */
4052 for (j = 0; j < burst_sz; ++j)
4053 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4055 start_time = rte_rdtsc_precise();
4057 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4058 &ops_enq[enq], burst_sz);
4059 TEST_ASSERT(enq == burst_sz,
4060 "Error enqueueing burst, expected %u, got %u",
4065 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4066 &ops_deq[deq], burst_sz - deq);
4067 if (likely(first_time && (deq > 0))) {
4068 last_time = rte_rdtsc_precise() - start_time;
4071 } while (unlikely(burst_sz != deq));
4073 *max_time = RTE_MAX(*max_time, last_time);
4074 *min_time = RTE_MIN(*min_time, last_time);
4075 *total_time += last_time;
4078 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4080 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4081 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4083 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4086 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4093 latency_test_enc(struct rte_mempool *mempool,
4094 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4095 uint16_t dev_id, uint16_t queue_id,
4096 const uint16_t num_to_process, uint16_t burst_sz,
4097 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4099 int ret = TEST_SUCCESS;
4100 uint16_t i, j, dequeued;
4101 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4102 uint64_t start_time = 0, last_time = 0;
4104 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4105 uint16_t enq = 0, deq = 0;
4106 bool first_time = true;
4109 if (unlikely(num_to_process - dequeued < burst_sz))
4110 burst_sz = num_to_process - dequeued;
4112 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4113 TEST_ASSERT_SUCCESS(ret,
4114 "rte_bbdev_enc_op_alloc_bulk() failed");
4115 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4116 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4121 /* Set counter to validate the ordering */
4122 for (j = 0; j < burst_sz; ++j)
4123 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4125 start_time = rte_rdtsc_precise();
4127 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4129 TEST_ASSERT(enq == burst_sz,
4130 "Error enqueueing burst, expected %u, got %u",
4135 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4136 &ops_deq[deq], burst_sz - deq);
4137 if (likely(first_time && (deq > 0))) {
4138 last_time += rte_rdtsc_precise() - start_time;
4141 } while (unlikely(burst_sz != deq));
4143 *max_time = RTE_MAX(*max_time, last_time);
4144 *min_time = RTE_MIN(*min_time, last_time);
4145 *total_time += last_time;
4147 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4148 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4149 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4152 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4160 latency_test_ldpc_enc(struct rte_mempool *mempool,
4161 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4162 uint16_t dev_id, uint16_t queue_id,
4163 const uint16_t num_to_process, uint16_t burst_sz,
4164 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4166 int ret = TEST_SUCCESS;
4167 uint16_t i, j, dequeued;
4168 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4169 uint64_t start_time = 0, last_time = 0;
4171 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4172 uint16_t enq = 0, deq = 0;
4173 bool first_time = true;
4176 if (unlikely(num_to_process - dequeued < burst_sz))
4177 burst_sz = num_to_process - dequeued;
4179 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4180 TEST_ASSERT_SUCCESS(ret,
4181 "rte_bbdev_enc_op_alloc_bulk() failed");
4182 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4183 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4188 /* Set counter to validate the ordering */
4189 for (j = 0; j < burst_sz; ++j)
4190 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4192 start_time = rte_rdtsc_precise();
4194 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4195 &ops_enq[enq], burst_sz);
4196 TEST_ASSERT(enq == burst_sz,
4197 "Error enqueueing burst, expected %u, got %u",
4202 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4203 &ops_deq[deq], burst_sz - deq);
4204 if (likely(first_time && (deq > 0))) {
4205 last_time += rte_rdtsc_precise() - start_time;
4208 } while (unlikely(burst_sz != deq));
4210 *max_time = RTE_MAX(*max_time, last_time);
4211 *min_time = RTE_MIN(*min_time, last_time);
4212 *total_time += last_time;
4214 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4215 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4216 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4219 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4226 /* Common function for running validation and latency test cases */
4228 validation_latency_test(struct active_device *ad,
4229 struct test_op_params *op_params, bool latency_flag)
4232 uint16_t burst_sz = op_params->burst_sz;
4233 const uint16_t num_to_process = op_params->num_to_process;
4234 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4235 const uint16_t queue_id = ad->queue_ids[0];
4236 struct test_buffers *bufs = NULL;
4237 struct rte_bbdev_info info;
4238 uint64_t total_time, min_time, max_time;
4239 const char *op_type_str;
4241 total_time = max_time = 0;
4242 min_time = UINT64_MAX;
4244 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4245 "BURST_SIZE should be <= %u", MAX_BURST);
4247 rte_bbdev_info_get(ad->dev_id, &info);
4248 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4250 op_type_str = rte_bbdev_op_type_str(op_type);
4251 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4253 printf("+ ------------------------------------------------------- +\n");
4255 printf("== test: latency\ndev:");
4257 printf("== test: validation\ndev:");
4258 printf("%s, burst size: %u, num ops: %u, op type: %s\n",
4259 info.dev_name, burst_sz, num_to_process, op_type_str);
4261 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4262 iter = latency_test_dec(op_params->mp, bufs,
4263 op_params->ref_dec_op, op_params->vector_mask,
4264 ad->dev_id, queue_id, num_to_process,
4265 burst_sz, &total_time, &min_time, &max_time);
4266 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4267 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4268 op_params->ref_enc_op, ad->dev_id, queue_id,
4269 num_to_process, burst_sz, &total_time,
4270 &min_time, &max_time);
4271 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4272 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4273 op_params->ref_dec_op, op_params->vector_mask,
4274 ad->dev_id, queue_id, num_to_process,
4275 burst_sz, &total_time, &min_time, &max_time,
4277 else /* RTE_BBDEV_OP_TURBO_ENC */
4278 iter = latency_test_enc(op_params->mp, bufs,
4279 op_params->ref_enc_op,
4280 ad->dev_id, queue_id,
4281 num_to_process, burst_sz, &total_time,
4282 &min_time, &max_time);
4287 printf("Operation latency:\n"
4288 "\tavg: %lg cycles, %lg us\n"
4289 "\tmin: %lg cycles, %lg us\n"
4290 "\tmax: %lg cycles, %lg us\n",
4291 (double)total_time / (double)iter,
4292 (double)(total_time * 1000000) / (double)iter /
4293 (double)rte_get_tsc_hz(), (double)min_time,
4294 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4295 (double)max_time, (double)(max_time * 1000000) /
4296 (double)rte_get_tsc_hz());
4298 return TEST_SUCCESS;
4302 latency_test(struct active_device *ad, struct test_op_params *op_params)
4304 return validation_latency_test(ad, op_params, true);
4308 validation_test(struct active_device *ad, struct test_op_params *op_params)
4310 return validation_latency_test(ad, op_params, false);
4313 #ifdef RTE_BBDEV_OFFLOAD_COST
4315 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4316 struct rte_bbdev_stats *stats)
4318 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4319 struct rte_bbdev_stats *q_stats;
4321 if (queue_id >= dev->data->num_queues)
4324 q_stats = &dev->data->queues[queue_id].queue_stats;
4326 stats->enqueued_count = q_stats->enqueued_count;
4327 stats->dequeued_count = q_stats->dequeued_count;
4328 stats->enqueue_err_count = q_stats->enqueue_err_count;
4329 stats->dequeue_err_count = q_stats->dequeue_err_count;
4330 stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4336 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4337 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4338 uint16_t queue_id, const uint16_t num_to_process,
4339 uint16_t burst_sz, struct test_time_stats *time_st)
4341 int i, dequeued, ret;
4342 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4343 uint64_t enq_start_time, deq_start_time;
4344 uint64_t enq_sw_last_time, deq_last_time;
4345 struct rte_bbdev_stats stats;
4347 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4348 uint16_t enq = 0, deq = 0;
4350 if (unlikely(num_to_process - dequeued < burst_sz))
4351 burst_sz = num_to_process - dequeued;
4353 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4354 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4355 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4361 /* Start time meas for enqueue function offload latency */
4362 enq_start_time = rte_rdtsc_precise();
4364 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4365 &ops_enq[enq], burst_sz - enq);
4366 } while (unlikely(burst_sz != enq));
4368 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4369 TEST_ASSERT_SUCCESS(ret,
4370 "Failed to get stats for queue (%u) of device (%u)",
4373 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4374 stats.acc_offload_cycles;
4375 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4377 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4379 time_st->enq_sw_total_time += enq_sw_last_time;
4381 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4382 stats.acc_offload_cycles);
4383 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4384 stats.acc_offload_cycles);
4385 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4387 /* give time for device to process ops */
4390 /* Start time meas for dequeue function offload latency */
4391 deq_start_time = rte_rdtsc_precise();
4392 /* Dequeue one operation */
4394 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4396 } while (unlikely(deq != 1));
4398 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4399 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4401 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4403 time_st->deq_total_time += deq_last_time;
4405 /* Dequeue remaining operations if needed*/
4406 while (burst_sz != deq)
4407 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4408 &ops_deq[deq], burst_sz - deq);
4410 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4418 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4419 struct test_buffers *bufs,
4420 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4421 uint16_t queue_id, const uint16_t num_to_process,
4422 uint16_t burst_sz, struct test_time_stats *time_st)
4424 int i, dequeued, ret;
4425 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4426 uint64_t enq_start_time, deq_start_time;
4427 uint64_t enq_sw_last_time, deq_last_time;
4428 struct rte_bbdev_stats stats;
4429 bool extDdr = ldpc_cap_flags &
4430 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4432 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4433 uint16_t enq = 0, deq = 0;
4435 if (unlikely(num_to_process - dequeued < burst_sz))
4436 burst_sz = num_to_process - dequeued;
4438 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4439 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4440 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4449 preload_harq_ddr(dev_id, queue_id, ops_enq,
4452 /* Start time meas for enqueue function offload latency */
4453 enq_start_time = rte_rdtsc_precise();
4455 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4456 &ops_enq[enq], burst_sz - enq);
4457 } while (unlikely(burst_sz != enq));
4459 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4460 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4461 TEST_ASSERT_SUCCESS(ret,
4462 "Failed to get stats for queue (%u) of device (%u)",
4465 enq_sw_last_time -= stats.acc_offload_cycles;
4466 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4468 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4470 time_st->enq_sw_total_time += enq_sw_last_time;
4472 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4473 stats.acc_offload_cycles);
4474 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4475 stats.acc_offload_cycles);
4476 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4478 /* give time for device to process ops */
4481 /* Start time meas for dequeue function offload latency */
4482 deq_start_time = rte_rdtsc_precise();
4483 /* Dequeue one operation */
4485 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4487 } while (unlikely(deq != 1));
4489 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4490 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4492 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4494 time_st->deq_total_time += deq_last_time;
4496 /* Dequeue remaining operations if needed*/
4497 while (burst_sz != deq)
4498 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4499 &ops_deq[deq], burst_sz - deq);
4502 /* Read loopback is not thread safe */
4503 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4506 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4514 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4515 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4516 uint16_t queue_id, const uint16_t num_to_process,
4517 uint16_t burst_sz, struct test_time_stats *time_st)
4519 int i, dequeued, ret;
4520 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4521 uint64_t enq_start_time, deq_start_time;
4522 uint64_t enq_sw_last_time, deq_last_time;
4523 struct rte_bbdev_stats stats;
4525 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4526 uint16_t enq = 0, deq = 0;
4528 if (unlikely(num_to_process - dequeued < burst_sz))
4529 burst_sz = num_to_process - dequeued;
4531 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4532 TEST_ASSERT_SUCCESS(ret,
4533 "rte_bbdev_enc_op_alloc_bulk() failed");
4534 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4535 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4540 /* Start time meas for enqueue function offload latency */
4541 enq_start_time = rte_rdtsc_precise();
4543 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4544 &ops_enq[enq], burst_sz - enq);
4545 } while (unlikely(burst_sz != enq));
4547 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4549 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4550 TEST_ASSERT_SUCCESS(ret,
4551 "Failed to get stats for queue (%u) of device (%u)",
4553 enq_sw_last_time -= stats.acc_offload_cycles;
4554 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4556 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4558 time_st->enq_sw_total_time += enq_sw_last_time;
4560 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4561 stats.acc_offload_cycles);
4562 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4563 stats.acc_offload_cycles);
4564 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4566 /* give time for device to process ops */
4569 /* Start time meas for dequeue function offload latency */
4570 deq_start_time = rte_rdtsc_precise();
4571 /* Dequeue one operation */
4573 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4575 } while (unlikely(deq != 1));
4577 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4578 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4580 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4582 time_st->deq_total_time += deq_last_time;
4584 while (burst_sz != deq)
4585 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4586 &ops_deq[deq], burst_sz - deq);
4588 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4596 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4597 struct test_buffers *bufs,
4598 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4599 uint16_t queue_id, const uint16_t num_to_process,
4600 uint16_t burst_sz, struct test_time_stats *time_st)
4602 int i, dequeued, ret;
4603 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4604 uint64_t enq_start_time, deq_start_time;
4605 uint64_t enq_sw_last_time, deq_last_time;
4606 struct rte_bbdev_stats stats;
4608 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4609 uint16_t enq = 0, deq = 0;
4611 if (unlikely(num_to_process - dequeued < burst_sz))
4612 burst_sz = num_to_process - dequeued;
4614 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4615 TEST_ASSERT_SUCCESS(ret,
4616 "rte_bbdev_enc_op_alloc_bulk() failed");
4617 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4618 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4623 /* Start time meas for enqueue function offload latency */
4624 enq_start_time = rte_rdtsc_precise();
4626 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4627 &ops_enq[enq], burst_sz - enq);
4628 } while (unlikely(burst_sz != enq));
4630 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4631 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4632 TEST_ASSERT_SUCCESS(ret,
4633 "Failed to get stats for queue (%u) of device (%u)",
4636 enq_sw_last_time -= stats.acc_offload_cycles;
4637 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4639 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4641 time_st->enq_sw_total_time += enq_sw_last_time;
4643 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4644 stats.acc_offload_cycles);
4645 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4646 stats.acc_offload_cycles);
4647 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4649 /* give time for device to process ops */
4652 /* Start time meas for dequeue function offload latency */
4653 deq_start_time = rte_rdtsc_precise();
4654 /* Dequeue one operation */
4656 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4658 } while (unlikely(deq != 1));
4660 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4661 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4663 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4665 time_st->deq_total_time += deq_last_time;
4667 while (burst_sz != deq)
4668 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4669 &ops_deq[deq], burst_sz - deq);
4671 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4680 offload_cost_test(struct active_device *ad,
4681 struct test_op_params *op_params)
4683 #ifndef RTE_BBDEV_OFFLOAD_COST
4685 RTE_SET_USED(op_params);
4686 printf("Offload latency test is disabled.\n");
4687 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4688 return TEST_SKIPPED;
4691 uint16_t burst_sz = op_params->burst_sz;
4692 const uint16_t num_to_process = op_params->num_to_process;
4693 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4694 const uint16_t queue_id = ad->queue_ids[0];
4695 struct test_buffers *bufs = NULL;
4696 struct rte_bbdev_info info;
4697 const char *op_type_str;
4698 struct test_time_stats time_st;
4700 memset(&time_st, 0, sizeof(struct test_time_stats));
4701 time_st.enq_sw_min_time = UINT64_MAX;
4702 time_st.enq_acc_min_time = UINT64_MAX;
4703 time_st.deq_min_time = UINT64_MAX;
4705 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4706 "BURST_SIZE should be <= %u", MAX_BURST);
4708 rte_bbdev_info_get(ad->dev_id, &info);
4709 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4711 op_type_str = rte_bbdev_op_type_str(op_type);
4712 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4714 printf("+ ------------------------------------------------------- +\n");
4715 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4716 info.dev_name, burst_sz, num_to_process, op_type_str);
4718 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4719 iter = offload_latency_test_dec(op_params->mp, bufs,
4720 op_params->ref_dec_op, ad->dev_id, queue_id,
4721 num_to_process, burst_sz, &time_st);
4722 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4723 iter = offload_latency_test_enc(op_params->mp, bufs,
4724 op_params->ref_enc_op, ad->dev_id, queue_id,
4725 num_to_process, burst_sz, &time_st);
4726 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4727 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4728 op_params->ref_enc_op, ad->dev_id, queue_id,
4729 num_to_process, burst_sz, &time_st);
4730 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4731 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4732 op_params->ref_dec_op, ad->dev_id, queue_id,
4733 num_to_process, burst_sz, &time_st);
4735 iter = offload_latency_test_enc(op_params->mp, bufs,
4736 op_params->ref_enc_op, ad->dev_id, queue_id,
4737 num_to_process, burst_sz, &time_st);
4742 printf("Enqueue driver offload cost latency:\n"
4743 "\tavg: %lg cycles, %lg us\n"
4744 "\tmin: %lg cycles, %lg us\n"
4745 "\tmax: %lg cycles, %lg us\n"
4746 "Enqueue accelerator offload cost latency:\n"
4747 "\tavg: %lg cycles, %lg us\n"
4748 "\tmin: %lg cycles, %lg us\n"
4749 "\tmax: %lg cycles, %lg us\n",
4750 (double)time_st.enq_sw_total_time / (double)iter,
4751 (double)(time_st.enq_sw_total_time * 1000000) /
4752 (double)iter / (double)rte_get_tsc_hz(),
4753 (double)time_st.enq_sw_min_time,
4754 (double)(time_st.enq_sw_min_time * 1000000) /
4755 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4756 (double)(time_st.enq_sw_max_time * 1000000) /
4757 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4759 (double)(time_st.enq_acc_total_time * 1000000) /
4760 (double)iter / (double)rte_get_tsc_hz(),
4761 (double)time_st.enq_acc_min_time,
4762 (double)(time_st.enq_acc_min_time * 1000000) /
4763 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4764 (double)(time_st.enq_acc_max_time * 1000000) /
4767 printf("Dequeue offload cost latency - one op:\n"
4768 "\tavg: %lg cycles, %lg us\n"
4769 "\tmin: %lg cycles, %lg us\n"
4770 "\tmax: %lg cycles, %lg us\n",
4771 (double)time_st.deq_total_time / (double)iter,
4772 (double)(time_st.deq_total_time * 1000000) /
4773 (double)iter / (double)rte_get_tsc_hz(),
4774 (double)time_st.deq_min_time,
4775 (double)(time_st.deq_min_time * 1000000) /
4776 rte_get_tsc_hz(), (double)time_st.deq_max_time,
4777 (double)(time_st.deq_max_time * 1000000) /
4780 struct rte_bbdev_stats stats = {0};
4781 get_bbdev_queue_stats(ad->dev_id, queue_id, &stats);
4782 if (op_type != RTE_BBDEV_OP_LDPC_DEC) {
4783 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process,
4784 "Mismatch in enqueue count %10"PRIu64" %d",
4785 stats.enqueued_count, num_to_process);
4786 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process,
4787 "Mismatch in dequeue count %10"PRIu64" %d",
4788 stats.dequeued_count, num_to_process);
4790 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0,
4791 "Enqueue count Error %10"PRIu64"",
4792 stats.enqueue_err_count);
4793 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0,
4794 "Dequeue count Error (%10"PRIu64"",
4795 stats.dequeue_err_count);
4797 return TEST_SUCCESS;
4801 #ifdef RTE_BBDEV_OFFLOAD_COST
4803 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4804 const uint16_t num_to_process, uint16_t burst_sz,
4805 uint64_t *deq_total_time, uint64_t *deq_min_time,
4806 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4809 struct rte_bbdev_dec_op *ops[MAX_BURST];
4810 uint64_t deq_start_time, deq_last_time;
4812 /* Test deq offload latency from an empty queue */
4814 for (i = 0, deq_total = 0; deq_total < num_to_process;
4815 ++i, deq_total += burst_sz) {
4816 deq_start_time = rte_rdtsc_precise();
4818 if (unlikely(num_to_process - deq_total < burst_sz))
4819 burst_sz = num_to_process - deq_total;
4820 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4821 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4824 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4827 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4828 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4829 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4830 *deq_total_time += deq_last_time;
4837 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4838 const uint16_t num_to_process, uint16_t burst_sz,
4839 uint64_t *deq_total_time, uint64_t *deq_min_time,
4840 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4843 struct rte_bbdev_enc_op *ops[MAX_BURST];
4844 uint64_t deq_start_time, deq_last_time;
4846 /* Test deq offload latency from an empty queue */
4847 for (i = 0, deq_total = 0; deq_total < num_to_process;
4848 ++i, deq_total += burst_sz) {
4849 deq_start_time = rte_rdtsc_precise();
4851 if (unlikely(num_to_process - deq_total < burst_sz))
4852 burst_sz = num_to_process - deq_total;
4853 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4854 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4857 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4860 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4861 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4862 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4863 *deq_total_time += deq_last_time;
4872 offload_latency_empty_q_test(struct active_device *ad,
4873 struct test_op_params *op_params)
4875 #ifndef RTE_BBDEV_OFFLOAD_COST
4877 RTE_SET_USED(op_params);
4878 printf("Offload latency empty dequeue test is disabled.\n");
4879 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4880 return TEST_SKIPPED;
4883 uint64_t deq_total_time, deq_min_time, deq_max_time;
4884 uint16_t burst_sz = op_params->burst_sz;
4885 const uint16_t num_to_process = op_params->num_to_process;
4886 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4887 const uint16_t queue_id = ad->queue_ids[0];
4888 struct rte_bbdev_info info;
4889 const char *op_type_str;
4891 deq_total_time = deq_max_time = 0;
4892 deq_min_time = UINT64_MAX;
4894 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4895 "BURST_SIZE should be <= %u", MAX_BURST);
4897 rte_bbdev_info_get(ad->dev_id, &info);
4899 op_type_str = rte_bbdev_op_type_str(op_type);
4900 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4902 printf("+ ------------------------------------------------------- +\n");
4903 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4904 info.dev_name, burst_sz, num_to_process, op_type_str);
4906 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4907 op_type == RTE_BBDEV_OP_LDPC_DEC)
4908 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4909 num_to_process, burst_sz, &deq_total_time,
4910 &deq_min_time, &deq_max_time, op_type);
4912 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4913 num_to_process, burst_sz, &deq_total_time,
4914 &deq_min_time, &deq_max_time, op_type);
4919 printf("Empty dequeue offload:\n"
4920 "\tavg: %lg cycles, %lg us\n"
4921 "\tmin: %lg cycles, %lg us\n"
4922 "\tmax: %lg cycles, %lg us\n",
4923 (double)deq_total_time / (double)iter,
4924 (double)(deq_total_time * 1000000) / (double)iter /
4925 (double)rte_get_tsc_hz(), (double)deq_min_time,
4926 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4927 (double)deq_max_time, (double)(deq_max_time * 1000000) /
4930 return TEST_SUCCESS;
4937 return run_test_case(bler_test);
4943 return run_test_case(throughput_test);
4947 offload_cost_tc(void)
4949 return run_test_case(offload_cost_test);
4953 offload_latency_empty_q_tc(void)
4955 return run_test_case(offload_latency_empty_q_test);
4961 return run_test_case(latency_test);
4967 return run_test_case(validation_test);
4973 return run_test_case(throughput_test);
4976 static struct unit_test_suite bbdev_bler_testsuite = {
4977 .suite_name = "BBdev BLER Tests",
4978 .setup = testsuite_setup,
4979 .teardown = testsuite_teardown,
4980 .unit_test_cases = {
4981 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
4982 TEST_CASES_END() /**< NULL terminate unit test array */
4986 static struct unit_test_suite bbdev_throughput_testsuite = {
4987 .suite_name = "BBdev Throughput Tests",
4988 .setup = testsuite_setup,
4989 .teardown = testsuite_teardown,
4990 .unit_test_cases = {
4991 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
4992 TEST_CASES_END() /**< NULL terminate unit test array */
4996 static struct unit_test_suite bbdev_validation_testsuite = {
4997 .suite_name = "BBdev Validation Tests",
4998 .setup = testsuite_setup,
4999 .teardown = testsuite_teardown,
5000 .unit_test_cases = {
5001 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc),
5002 TEST_CASES_END() /**< NULL terminate unit test array */
5006 static struct unit_test_suite bbdev_latency_testsuite = {
5007 .suite_name = "BBdev Latency Tests",
5008 .setup = testsuite_setup,
5009 .teardown = testsuite_teardown,
5010 .unit_test_cases = {
5011 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
5012 TEST_CASES_END() /**< NULL terminate unit test array */
5016 static struct unit_test_suite bbdev_offload_cost_testsuite = {
5017 .suite_name = "BBdev Offload Cost Tests",
5018 .setup = testsuite_setup,
5019 .teardown = testsuite_teardown,
5020 .unit_test_cases = {
5021 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
5022 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
5023 TEST_CASES_END() /**< NULL terminate unit test array */
5027 static struct unit_test_suite bbdev_interrupt_testsuite = {
5028 .suite_name = "BBdev Interrupt Tests",
5029 .setup = interrupt_testsuite_setup,
5030 .teardown = testsuite_teardown,
5031 .unit_test_cases = {
5032 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
5033 TEST_CASES_END() /**< NULL terminate unit test array */
5037 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
5038 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
5039 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
5040 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
5041 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
5042 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);