1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
19 #include <rte_interrupts.h>
22 #include "test_bbdev_vector.h"
24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
26 #define MAX_QUEUES RTE_MAX_LCORE
27 #define TEST_REPETITIONS 100
28 #define WAIT_OFFLOAD_US 1000
30 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
31 #include <fpga_lte_fec.h>
32 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
33 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
34 #define VF_UL_4G_QUEUE_VALUE 4
35 #define VF_DL_4G_QUEUE_VALUE 4
36 #define UL_4G_BANDWIDTH 3
37 #define DL_4G_BANDWIDTH 3
38 #define UL_4G_LOAD_BALANCE 128
39 #define DL_4G_LOAD_BALANCE 128
40 #define FLR_4G_TIMEOUT 610
43 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
44 #include <rte_pmd_fpga_5gnr_fec.h>
45 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
46 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
47 #define VF_UL_5G_QUEUE_VALUE 4
48 #define VF_DL_5G_QUEUE_VALUE 4
49 #define UL_5G_BANDWIDTH 3
50 #define DL_5G_BANDWIDTH 3
51 #define UL_5G_LOAD_BALANCE 128
52 #define DL_5G_LOAD_BALANCE 128
53 #define FLR_5G_TIMEOUT 610
56 #ifdef RTE_BASEBAND_ACC100
57 #include <rte_acc100_cfg.h>
58 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf")
59 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf")
60 #define ACC100_QMGR_NUM_AQS 16
61 #define ACC100_QMGR_NUM_QGS 2
62 #define ACC100_QMGR_AQ_DEPTH 5
63 #define ACC100_QMGR_INVALID_IDX -1
64 #define ACC100_QMGR_RR 1
65 #define ACC100_QOS_GBR 0
68 #define OPS_CACHE_SIZE 256U
69 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
73 #define INVALID_OPAQUE -1
75 #define INVALID_QUEUE_ID -1
76 /* Increment for next code block in external HARQ memory */
77 #define HARQ_INCR 32768
78 /* Headroom for filler LLRs insertion in HARQ buffer */
79 #define FILLER_HEADROOM 1024
80 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
81 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
82 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
83 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
84 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
85 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
86 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
87 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
88 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
90 static struct test_bbdev_vector test_vector;
92 /* Switch between PMD and Interrupt for throughput TC */
93 static bool intr_enabled;
95 /* LLR arithmetic representation for numerical conversion */
96 static int ldpc_llr_decimals;
97 static int ldpc_llr_size;
98 /* Keep track of the LDPC decoder device capability flag */
99 static uint32_t ldpc_cap_flags;
101 /* Represents tested active devices */
102 static struct active_device {
103 const char *driver_name;
105 uint16_t supported_ops;
106 uint16_t queue_ids[MAX_QUEUES];
108 struct rte_mempool *ops_mempool;
109 struct rte_mempool *in_mbuf_pool;
110 struct rte_mempool *hard_out_mbuf_pool;
111 struct rte_mempool *soft_out_mbuf_pool;
112 struct rte_mempool *harq_in_mbuf_pool;
113 struct rte_mempool *harq_out_mbuf_pool;
114 } active_devs[RTE_BBDEV_MAX_DEVS];
116 static uint8_t nb_active_devs;
118 /* Data buffers used by BBDEV ops */
119 struct test_buffers {
120 struct rte_bbdev_op_data *inputs;
121 struct rte_bbdev_op_data *hard_outputs;
122 struct rte_bbdev_op_data *soft_outputs;
123 struct rte_bbdev_op_data *harq_inputs;
124 struct rte_bbdev_op_data *harq_outputs;
127 /* Operation parameters specific for given test case */
128 struct test_op_params {
129 struct rte_mempool *mp;
130 struct rte_bbdev_dec_op *ref_dec_op;
131 struct rte_bbdev_enc_op *ref_enc_op;
133 uint16_t num_to_process;
137 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
140 /* Contains per lcore params */
141 struct thread_params {
151 rte_atomic16_t nb_dequeued;
152 rte_atomic16_t processing_status;
153 rte_atomic16_t burst_sz;
154 struct test_op_params *op_params;
155 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
156 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
159 #ifdef RTE_BBDEV_OFFLOAD_COST
160 /* Stores time statistics */
161 struct test_time_stats {
162 /* Stores software enqueue total working time */
163 uint64_t enq_sw_total_time;
164 /* Stores minimum value of software enqueue working time */
165 uint64_t enq_sw_min_time;
166 /* Stores maximum value of software enqueue working time */
167 uint64_t enq_sw_max_time;
168 /* Stores turbo enqueue total working time */
169 uint64_t enq_acc_total_time;
170 /* Stores minimum value of accelerator enqueue working time */
171 uint64_t enq_acc_min_time;
172 /* Stores maximum value of accelerator enqueue working time */
173 uint64_t enq_acc_max_time;
174 /* Stores dequeue total working time */
175 uint64_t deq_total_time;
176 /* Stores minimum value of dequeue working time */
177 uint64_t deq_min_time;
178 /* Stores maximum value of dequeue working time */
179 uint64_t deq_max_time;
183 typedef int (test_case_function)(struct active_device *ad,
184 struct test_op_params *op_params);
187 mbuf_reset(struct rte_mbuf *m)
197 /* Read flag value 0/1 from bitmap */
199 check_bit(uint32_t bitmap, uint32_t bitmask)
201 return bitmap & bitmask;
205 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
207 ad->supported_ops |= (1 << op_type);
211 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
213 return ad->supported_ops & (1 << op_type);
217 flags_match(uint32_t flags_req, uint32_t flags_present)
219 return (flags_req & flags_present) == flags_req;
223 clear_soft_out_cap(uint32_t *op_flags)
225 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
226 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
227 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
231 check_dev_cap(const struct rte_bbdev_info *dev_info)
234 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
235 nb_harq_inputs, nb_harq_outputs;
236 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
238 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
239 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
240 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
241 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
242 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
244 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
245 if (op_cap->type != test_vector.op_type)
248 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
249 const struct rte_bbdev_op_cap_turbo_dec *cap =
250 &op_cap->cap.turbo_dec;
251 /* Ignore lack of soft output capability, just skip
252 * checking if soft output is valid.
254 if ((test_vector.turbo_dec.op_flags &
255 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
256 !(cap->capability_flags &
257 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
259 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
262 &test_vector.turbo_dec.op_flags);
265 if (!flags_match(test_vector.turbo_dec.op_flags,
266 cap->capability_flags))
268 if (nb_inputs > cap->num_buffers_src) {
269 printf("Too many inputs defined: %u, max: %u\n",
270 nb_inputs, cap->num_buffers_src);
273 if (nb_soft_outputs > cap->num_buffers_soft_out &&
274 (test_vector.turbo_dec.op_flags &
275 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
277 "Too many soft outputs defined: %u, max: %u\n",
279 cap->num_buffers_soft_out);
282 if (nb_hard_outputs > cap->num_buffers_hard_out) {
284 "Too many hard outputs defined: %u, max: %u\n",
286 cap->num_buffers_hard_out);
289 if (intr_enabled && !(cap->capability_flags &
290 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
292 "Dequeue interrupts are not supported!\n");
297 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
298 const struct rte_bbdev_op_cap_turbo_enc *cap =
299 &op_cap->cap.turbo_enc;
301 if (!flags_match(test_vector.turbo_enc.op_flags,
302 cap->capability_flags))
304 if (nb_inputs > cap->num_buffers_src) {
305 printf("Too many inputs defined: %u, max: %u\n",
306 nb_inputs, cap->num_buffers_src);
309 if (nb_hard_outputs > cap->num_buffers_dst) {
311 "Too many hard outputs defined: %u, max: %u\n",
312 nb_hard_outputs, cap->num_buffers_dst);
315 if (intr_enabled && !(cap->capability_flags &
316 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
318 "Dequeue interrupts are not supported!\n");
323 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
324 const struct rte_bbdev_op_cap_ldpc_enc *cap =
325 &op_cap->cap.ldpc_enc;
327 if (!flags_match(test_vector.ldpc_enc.op_flags,
328 cap->capability_flags)){
329 printf("Flag Mismatch\n");
332 if (nb_inputs > cap->num_buffers_src) {
333 printf("Too many inputs defined: %u, max: %u\n",
334 nb_inputs, cap->num_buffers_src);
337 if (nb_hard_outputs > cap->num_buffers_dst) {
339 "Too many hard outputs defined: %u, max: %u\n",
340 nb_hard_outputs, cap->num_buffers_dst);
343 if (intr_enabled && !(cap->capability_flags &
344 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
346 "Dequeue interrupts are not supported!\n");
351 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
352 const struct rte_bbdev_op_cap_ldpc_dec *cap =
353 &op_cap->cap.ldpc_dec;
355 if (!flags_match(test_vector.ldpc_dec.op_flags,
356 cap->capability_flags)){
357 printf("Flag Mismatch\n");
360 if (nb_inputs > cap->num_buffers_src) {
361 printf("Too many inputs defined: %u, max: %u\n",
362 nb_inputs, cap->num_buffers_src);
365 if (nb_hard_outputs > cap->num_buffers_hard_out) {
367 "Too many hard outputs defined: %u, max: %u\n",
369 cap->num_buffers_hard_out);
372 if (nb_harq_inputs > cap->num_buffers_hard_out) {
374 "Too many HARQ inputs defined: %u, max: %u\n",
376 cap->num_buffers_hard_out);
379 if (nb_harq_outputs > cap->num_buffers_hard_out) {
381 "Too many HARQ outputs defined: %u, max: %u\n",
383 cap->num_buffers_hard_out);
386 if (intr_enabled && !(cap->capability_flags &
387 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
389 "Dequeue interrupts are not supported!\n");
392 if (intr_enabled && (test_vector.ldpc_dec.op_flags &
393 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
394 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
395 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
397 printf("Skip loop-back with interrupt\n");
404 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
405 return TEST_SUCCESS; /* Special case for NULL device */
410 /* calculates optimal mempool size not smaller than the val */
412 optimal_mempool_size(unsigned int val)
414 return rte_align32pow2(val + 1) - 1;
417 /* allocates mbuf mempool for inputs and outputs */
418 static struct rte_mempool *
419 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
420 int socket_id, unsigned int mbuf_pool_size,
421 const char *op_type_str)
424 uint32_t max_seg_sz = 0;
425 char pool_name[RTE_MEMPOOL_NAMESIZE];
427 /* find max input segment size */
428 for (i = 0; i < entries->nb_segments; ++i)
429 if (entries->segments[i].length > max_seg_sz)
430 max_seg_sz = entries->segments[i].length;
432 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
434 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
435 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
437 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
441 create_mempools(struct active_device *ad, int socket_id,
442 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
444 struct rte_mempool *mp;
445 unsigned int ops_pool_size, mbuf_pool_size = 0;
446 char pool_name[RTE_MEMPOOL_NAMESIZE];
447 const char *op_type_str;
448 enum rte_bbdev_op_type op_type = org_op_type;
450 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
451 struct op_data_entries *hard_out =
452 &test_vector.entries[DATA_HARD_OUTPUT];
453 struct op_data_entries *soft_out =
454 &test_vector.entries[DATA_SOFT_OUTPUT];
455 struct op_data_entries *harq_in =
456 &test_vector.entries[DATA_HARQ_INPUT];
457 struct op_data_entries *harq_out =
458 &test_vector.entries[DATA_HARQ_OUTPUT];
460 /* allocate ops mempool */
461 ops_pool_size = optimal_mempool_size(RTE_MAX(
462 /* Ops used plus 1 reference op */
463 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
464 /* Minimal cache size plus 1 reference op */
465 (unsigned int)(1.5 * rte_lcore_count() *
466 OPS_CACHE_SIZE + 1)),
469 if (org_op_type == RTE_BBDEV_OP_NONE)
470 op_type = RTE_BBDEV_OP_TURBO_ENC;
472 op_type_str = rte_bbdev_op_type_str(op_type);
473 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
475 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
477 mp = rte_bbdev_op_pool_create(pool_name, op_type,
478 ops_pool_size, OPS_CACHE_SIZE, socket_id);
479 TEST_ASSERT_NOT_NULL(mp,
480 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
484 ad->ops_mempool = mp;
486 /* Do not create inputs and outputs mbufs for BaseBand Null Device */
487 if (org_op_type == RTE_BBDEV_OP_NONE)
491 if (in->nb_segments > 0) {
492 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
494 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
495 mbuf_pool_size, "in");
496 TEST_ASSERT_NOT_NULL(mp,
497 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
501 ad->in_mbuf_pool = mp;
505 if (hard_out->nb_segments > 0) {
506 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
507 hard_out->nb_segments);
508 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
511 TEST_ASSERT_NOT_NULL(mp,
512 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
516 ad->hard_out_mbuf_pool = mp;
520 if (soft_out->nb_segments > 0) {
521 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
522 soft_out->nb_segments);
523 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
526 TEST_ASSERT_NOT_NULL(mp,
527 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
531 ad->soft_out_mbuf_pool = mp;
535 if (harq_in->nb_segments > 0) {
536 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
537 harq_in->nb_segments);
538 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
541 TEST_ASSERT_NOT_NULL(mp,
542 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
546 ad->harq_in_mbuf_pool = mp;
550 if (harq_out->nb_segments > 0) {
551 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
552 harq_out->nb_segments);
553 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
556 TEST_ASSERT_NOT_NULL(mp,
557 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
561 ad->harq_out_mbuf_pool = mp;
568 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
569 struct test_bbdev_vector *vector)
572 unsigned int queue_id;
573 struct rte_bbdev_queue_conf qconf;
574 struct active_device *ad = &active_devs[nb_active_devs];
575 unsigned int nb_queues;
576 enum rte_bbdev_op_type op_type = vector->op_type;
578 /* Configure fpga lte fec with PF & VF values
579 * if '-i' flag is set and using fpga device
581 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
582 if ((get_init_device() == true) &&
583 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
584 struct rte_fpga_lte_fec_conf conf;
587 printf("Configure FPGA LTE FEC Driver %s with default values\n",
588 info->drv.driver_name);
590 /* clear default configuration before initialization */
591 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf));
594 * true if PF is used for data plane
597 conf.pf_mode_en = true;
599 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
600 /* Number of UL queues per VF (fpga supports 8 VFs) */
601 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
602 /* Number of DL queues per VF (fpga supports 8 VFs) */
603 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
606 /* UL bandwidth. Needed for schedule algorithm */
607 conf.ul_bandwidth = UL_4G_BANDWIDTH;
609 conf.dl_bandwidth = DL_4G_BANDWIDTH;
611 /* UL & DL load Balance Factor to 64 */
612 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
613 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
615 /**< FLR timeout value */
616 conf.flr_time_out = FLR_4G_TIMEOUT;
618 /* setup FPGA PF with configuration information */
619 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf);
620 TEST_ASSERT_SUCCESS(ret,
621 "Failed to configure 4G FPGA PF for bbdev %s",
625 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
626 if ((get_init_device() == true) &&
627 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
628 struct rte_fpga_5gnr_fec_conf conf;
631 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
632 info->drv.driver_name);
634 /* clear default configuration before initialization */
635 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
638 * true if PF is used for data plane
641 conf.pf_mode_en = true;
643 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
644 /* Number of UL queues per VF (fpga supports 8 VFs) */
645 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
646 /* Number of DL queues per VF (fpga supports 8 VFs) */
647 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
650 /* UL bandwidth. Needed for schedule algorithm */
651 conf.ul_bandwidth = UL_5G_BANDWIDTH;
653 conf.dl_bandwidth = DL_5G_BANDWIDTH;
655 /* UL & DL load Balance Factor to 64 */
656 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
657 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
659 /**< FLR timeout value */
660 conf.flr_time_out = FLR_5G_TIMEOUT;
662 /* setup FPGA PF with configuration information */
663 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf);
664 TEST_ASSERT_SUCCESS(ret,
665 "Failed to configure 5G FPGA PF for bbdev %s",
669 #ifdef RTE_BASEBAND_ACC100
670 if ((get_init_device() == true) &&
671 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) {
672 struct rte_acc100_conf conf;
675 printf("Configure ACC100 FEC Driver %s with default values\n",
676 info->drv.driver_name);
678 /* clear default configuration before initialization */
679 memset(&conf, 0, sizeof(struct rte_acc100_conf));
681 /* Always set in PF mode for built-in configuration */
682 conf.pf_mode_en = true;
683 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) {
684 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
685 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
686 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR;
687 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
688 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
689 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR;
690 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
691 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
692 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR;
693 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
694 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
695 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR;
698 conf.input_pos_llr_1_bit = true;
699 conf.output_pos_llr_1_bit = true;
700 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */
702 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
703 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
704 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
705 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
706 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
707 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
708 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
709 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
710 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
711 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
712 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
713 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
714 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
715 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
716 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
717 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
719 /* setup PF with configuration information */
720 ret = rte_acc100_configure(info->dev_name, &conf);
721 TEST_ASSERT_SUCCESS(ret,
722 "Failed to configure ACC100 PF for bbdev %s",
726 /* Let's refresh this now this is configured */
727 rte_bbdev_info_get(dev_id, info);
728 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
729 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
732 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
734 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
735 dev_id, nb_queues, info->socket_id, ret);
739 /* configure interrupts if needed */
741 ret = rte_bbdev_intr_enable(dev_id);
743 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
749 /* setup device queues */
750 qconf.socket = info->socket_id;
751 qconf.queue_size = info->drv.default_queue_conf.queue_size;
753 qconf.deferred_start = 0;
754 qconf.op_type = op_type;
756 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
757 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
760 "Allocated all queues (id=%u) at prio%u on dev%u\n",
761 queue_id, qconf.priority, dev_id);
763 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
767 printf("All queues on dev %u allocated: %u\n",
771 ad->queue_ids[queue_id] = queue_id;
773 TEST_ASSERT(queue_id != 0,
774 "ERROR Failed to configure any queues on dev %u",
776 ad->nb_queues = queue_id;
778 set_avail_op(ad, op_type);
784 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
785 struct test_bbdev_vector *vector)
789 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
790 active_devs[nb_active_devs].dev_id = dev_id;
792 ret = add_bbdev_dev(dev_id, info, vector);
793 if (ret == TEST_SUCCESS)
799 populate_active_devices(void)
803 uint8_t nb_devs_added = 0;
804 struct rte_bbdev_info info;
806 RTE_BBDEV_FOREACH(dev_id) {
807 rte_bbdev_info_get(dev_id, &info);
809 if (check_dev_cap(&info)) {
811 "Device %d (%s) does not support specified capabilities\n",
812 dev_id, info.dev_name);
816 ret = add_active_device(dev_id, &info, &test_vector);
818 printf("Adding active bbdev %s skipped\n",
825 return nb_devs_added;
829 read_test_vector(void)
833 memset(&test_vector, 0, sizeof(test_vector));
834 printf("Test vector file = %s\n", get_vector_filename());
835 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
836 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
837 get_vector_filename());
843 testsuite_setup(void)
845 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
847 if (populate_active_devices() == 0) {
848 printf("No suitable devices found!\n");
856 interrupt_testsuite_setup(void)
858 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
860 /* Enable interrupts */
863 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
864 if (populate_active_devices() == 0 ||
865 test_vector.op_type == RTE_BBDEV_OP_NONE) {
866 intr_enabled = false;
867 printf("No suitable devices found!\n");
875 testsuite_teardown(void)
879 /* Unconfigure devices */
880 RTE_BBDEV_FOREACH(dev_id)
881 rte_bbdev_close(dev_id);
883 /* Clear active devices structs. */
884 memset(active_devs, 0, sizeof(active_devs));
887 /* Disable interrupts */
888 intr_enabled = false;
896 for (i = 0; i < nb_active_devs; i++) {
897 dev_id = active_devs[i].dev_id;
898 /* reset bbdev stats */
899 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
900 "Failed to reset stats of bbdev %u", dev_id);
901 /* start the device */
902 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
903 "Failed to start bbdev %u", dev_id);
913 struct rte_bbdev_stats stats;
915 for (i = 0; i < nb_active_devs; i++) {
916 dev_id = active_devs[i].dev_id;
917 /* read stats and print */
918 rte_bbdev_stats_get(dev_id, &stats);
919 /* Stop the device */
920 rte_bbdev_stop(dev_id);
925 init_op_data_objs(struct rte_bbdev_op_data *bufs,
926 struct op_data_entries *ref_entries,
927 struct rte_mempool *mbuf_pool, const uint16_t n,
928 enum op_data_type op_type, uint16_t min_alignment)
932 bool large_input = false;
934 for (i = 0; i < n; ++i) {
936 struct op_data_buf *seg = &ref_entries->segments[0];
937 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
938 TEST_ASSERT_NOT_NULL(m_head,
939 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
940 op_type, n * ref_entries->nb_segments,
943 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
945 * Special case when DPDK mbuf cannot handle
946 * the required input size
948 printf("Warning: Larger input size than DPDK mbuf %d\n",
952 bufs[i].data = m_head;
956 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
957 if ((op_type == DATA_INPUT) && large_input) {
958 /* Allocate a fake overused mbuf */
959 data = rte_malloc(NULL, seg->length, 0);
960 memcpy(data, seg->addr, seg->length);
961 m_head->buf_addr = data;
962 m_head->buf_iova = rte_malloc_virt2iova(data);
963 m_head->data_off = 0;
964 m_head->data_len = seg->length;
966 data = rte_pktmbuf_append(m_head, seg->length);
967 TEST_ASSERT_NOT_NULL(data,
968 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
969 seg->length, op_type);
971 TEST_ASSERT(data == RTE_PTR_ALIGN(
972 data, min_alignment),
973 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
974 data, min_alignment);
975 rte_memcpy(data, seg->addr, seg->length);
978 bufs[i].length += seg->length;
980 for (j = 1; j < ref_entries->nb_segments; ++j) {
981 struct rte_mbuf *m_tail =
982 rte_pktmbuf_alloc(mbuf_pool);
983 TEST_ASSERT_NOT_NULL(m_tail,
984 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
986 n * ref_entries->nb_segments,
990 data = rte_pktmbuf_append(m_tail, seg->length);
991 TEST_ASSERT_NOT_NULL(data,
992 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
993 seg->length, op_type);
995 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
997 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
998 data, min_alignment);
999 rte_memcpy(data, seg->addr, seg->length);
1000 bufs[i].length += seg->length;
1002 ret = rte_pktmbuf_chain(m_head, m_tail);
1003 TEST_ASSERT_SUCCESS(ret,
1004 "Couldn't chain mbufs from %d data type mbuf pool",
1009 /* allocate chained-mbuf for output buffer */
1010 for (j = 1; j < ref_entries->nb_segments; ++j) {
1011 struct rte_mbuf *m_tail =
1012 rte_pktmbuf_alloc(mbuf_pool);
1013 TEST_ASSERT_NOT_NULL(m_tail,
1014 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1016 n * ref_entries->nb_segments,
1019 ret = rte_pktmbuf_chain(m_head, m_tail);
1020 TEST_ASSERT_SUCCESS(ret,
1021 "Couldn't chain mbufs from %d data type mbuf pool",
1031 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
1036 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
1037 if (*buffers == NULL) {
1038 printf("WARNING: Failed to allocate op_data on socket %d\n",
1040 /* try to allocate memory on other detected sockets */
1041 for (i = 0; i < socket; i++) {
1042 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
1043 if (*buffers != NULL)
1048 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
1052 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
1053 const uint16_t n, const int8_t max_llr_modulus)
1055 uint16_t i, byte_idx;
1057 for (i = 0; i < n; ++i) {
1058 struct rte_mbuf *m = input_ops[i].data;
1060 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1061 input_ops[i].offset);
1062 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1064 llr[byte_idx] = round((double)max_llr_modulus *
1065 llr[byte_idx] / INT8_MAX);
1073 * We may have to insert filler bits
1074 * when they are required by the HARQ assumption
1077 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1078 const uint16_t n, struct test_op_params *op_params)
1080 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1082 if (input_ops == NULL)
1084 /* No need to add filler if not required by device */
1085 if (!(ldpc_cap_flags &
1086 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1088 /* No need to add filler for loopback operation */
1089 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1092 uint16_t i, j, parity_offset;
1093 for (i = 0; i < n; ++i) {
1094 struct rte_mbuf *m = input_ops[i].data;
1095 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1096 input_ops[i].offset);
1097 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1098 * dec.z_c - dec.n_filler;
1099 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1100 m->data_len = new_hin_size;
1101 input_ops[i].length = new_hin_size;
1102 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1104 llr[j] = llr[j - dec.n_filler];
1105 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1106 for (j = 0; j < dec.n_filler; j++)
1107 llr[parity_offset + j] = llr_max_pre_scaling;
1112 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1113 const uint16_t n, const int8_t llr_size,
1114 const int8_t llr_decimals)
1116 if (input_ops == NULL)
1119 uint16_t i, byte_idx;
1121 int16_t llr_max, llr_min, llr_tmp;
1122 llr_max = (1 << (llr_size - 1)) - 1;
1124 for (i = 0; i < n; ++i) {
1125 struct rte_mbuf *m = input_ops[i].data;
1127 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1128 input_ops[i].offset);
1129 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1132 llr_tmp = llr[byte_idx];
1133 if (llr_decimals == 4)
1135 else if (llr_decimals == 2)
1137 else if (llr_decimals == 0)
1139 llr_tmp = RTE_MIN(llr_max,
1140 RTE_MAX(llr_min, llr_tmp));
1141 llr[byte_idx] = (int8_t) llr_tmp;
1152 fill_queue_buffers(struct test_op_params *op_params,
1153 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1154 struct rte_mempool *soft_out_mp,
1155 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1157 const struct rte_bbdev_op_cap *capabilities,
1158 uint16_t min_alignment, const int socket_id)
1161 enum op_data_type type;
1162 const uint16_t n = op_params->num_to_process;
1164 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1172 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1173 &op_params->q_bufs[socket_id][queue_id].inputs,
1174 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1175 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1176 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1177 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1180 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1181 struct op_data_entries *ref_entries =
1182 &test_vector.entries[type];
1183 if (ref_entries->nb_segments == 0)
1186 ret = allocate_buffers_on_socket(queue_ops[type],
1187 n * sizeof(struct rte_bbdev_op_data),
1189 TEST_ASSERT_SUCCESS(ret,
1190 "Couldn't allocate memory for rte_bbdev_op_data structs");
1192 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1193 mbuf_pools[type], n, type, min_alignment);
1194 TEST_ASSERT_SUCCESS(ret,
1195 "Couldn't init rte_bbdev_op_data structs");
1198 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1199 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1200 capabilities->cap.turbo_dec.max_llr_modulus);
1202 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1203 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1204 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1205 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1206 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1207 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1208 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1209 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1210 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1211 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1212 if (!loopback && !llr_comp)
1213 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1214 ldpc_llr_size, ldpc_llr_decimals);
1215 if (!loopback && !harq_comp)
1216 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1217 ldpc_llr_size, ldpc_llr_decimals);
1219 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1227 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1231 rte_mempool_free(ad->ops_mempool);
1232 rte_mempool_free(ad->in_mbuf_pool);
1233 rte_mempool_free(ad->hard_out_mbuf_pool);
1234 rte_mempool_free(ad->soft_out_mbuf_pool);
1235 rte_mempool_free(ad->harq_in_mbuf_pool);
1236 rte_mempool_free(ad->harq_out_mbuf_pool);
1238 for (i = 0; i < rte_lcore_count(); ++i) {
1239 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1240 rte_free(op_params->q_bufs[j][i].inputs);
1241 rte_free(op_params->q_bufs[j][i].hard_outputs);
1242 rte_free(op_params->q_bufs[j][i].soft_outputs);
1243 rte_free(op_params->q_bufs[j][i].harq_inputs);
1244 rte_free(op_params->q_bufs[j][i].harq_outputs);
1250 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1251 unsigned int start_idx,
1252 struct rte_bbdev_op_data *inputs,
1253 struct rte_bbdev_op_data *hard_outputs,
1254 struct rte_bbdev_op_data *soft_outputs,
1255 struct rte_bbdev_dec_op *ref_op)
1258 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1260 for (i = 0; i < n; ++i) {
1261 if (turbo_dec->code_block_mode == 0) {
1262 ops[i]->turbo_dec.tb_params.ea =
1263 turbo_dec->tb_params.ea;
1264 ops[i]->turbo_dec.tb_params.eb =
1265 turbo_dec->tb_params.eb;
1266 ops[i]->turbo_dec.tb_params.k_pos =
1267 turbo_dec->tb_params.k_pos;
1268 ops[i]->turbo_dec.tb_params.k_neg =
1269 turbo_dec->tb_params.k_neg;
1270 ops[i]->turbo_dec.tb_params.c =
1271 turbo_dec->tb_params.c;
1272 ops[i]->turbo_dec.tb_params.c_neg =
1273 turbo_dec->tb_params.c_neg;
1274 ops[i]->turbo_dec.tb_params.cab =
1275 turbo_dec->tb_params.cab;
1276 ops[i]->turbo_dec.tb_params.r =
1277 turbo_dec->tb_params.r;
1279 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1280 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1283 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1284 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1285 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1286 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1287 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1288 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1289 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1291 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1292 ops[i]->turbo_dec.input = inputs[start_idx + i];
1293 if (soft_outputs != NULL)
1294 ops[i]->turbo_dec.soft_output =
1295 soft_outputs[start_idx + i];
1300 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1301 unsigned int start_idx,
1302 struct rte_bbdev_op_data *inputs,
1303 struct rte_bbdev_op_data *outputs,
1304 struct rte_bbdev_enc_op *ref_op)
1307 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1308 for (i = 0; i < n; ++i) {
1309 if (turbo_enc->code_block_mode == 0) {
1310 ops[i]->turbo_enc.tb_params.ea =
1311 turbo_enc->tb_params.ea;
1312 ops[i]->turbo_enc.tb_params.eb =
1313 turbo_enc->tb_params.eb;
1314 ops[i]->turbo_enc.tb_params.k_pos =
1315 turbo_enc->tb_params.k_pos;
1316 ops[i]->turbo_enc.tb_params.k_neg =
1317 turbo_enc->tb_params.k_neg;
1318 ops[i]->turbo_enc.tb_params.c =
1319 turbo_enc->tb_params.c;
1320 ops[i]->turbo_enc.tb_params.c_neg =
1321 turbo_enc->tb_params.c_neg;
1322 ops[i]->turbo_enc.tb_params.cab =
1323 turbo_enc->tb_params.cab;
1324 ops[i]->turbo_enc.tb_params.ncb_pos =
1325 turbo_enc->tb_params.ncb_pos;
1326 ops[i]->turbo_enc.tb_params.ncb_neg =
1327 turbo_enc->tb_params.ncb_neg;
1328 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1330 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1331 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1332 ops[i]->turbo_enc.cb_params.ncb =
1333 turbo_enc->cb_params.ncb;
1335 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1336 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1337 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1339 ops[i]->turbo_enc.output = outputs[start_idx + i];
1340 ops[i]->turbo_enc.input = inputs[start_idx + i];
1345 /* Returns a random number drawn from a normal distribution
1346 * with mean of 0 and variance of 1
1347 * Marsaglia algorithm
1352 double S, Z, U1, U2, u, v, fac;
1355 U1 = (double)rand() / RAND_MAX;
1356 U2 = (double)rand() / RAND_MAX;
1360 } while (S >= 1 || S == 0);
1361 fac = sqrt(-2. * log(S) / S);
1362 Z = (n % 2) ? u * fac : v * fac;
1366 static inline double
1367 maxstar(double A, double B)
1369 if (fabs(A - B) > 5)
1370 return RTE_MAX(A, B);
1372 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1376 * Generate Qm LLRS for Qm==8
1377 * Modulation, AWGN and LLR estimation from max log development
1380 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1385 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1386 /* 5.1.4 of TS38.211 */
1387 const double symbols_I[256] = {
1388 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1389 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1390 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1391 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1392 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1393 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1394 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1395 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1396 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1397 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1398 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1399 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1400 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1401 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1402 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1403 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1404 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1405 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1406 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1407 -13, -13, -15, -15, -13, -13, -15, -15};
1408 const double symbols_Q[256] = {
1409 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1410 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1411 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1412 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1413 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1414 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1415 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1416 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1417 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1418 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1419 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1420 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1421 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1422 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1423 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1424 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1425 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1426 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1427 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1428 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1429 /* Average constellation point energy */
1431 for (k = 0; k < qm; k++)
1432 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1433 /* 5.1.4 of TS38.211 */
1434 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1435 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1436 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1437 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1439 I += sqrt(N0 / 2) * randn(0);
1440 Q += sqrt(N0 / 2) * randn(1);
1442 * Calculate the log of the probability that each of
1443 * the constellation points was transmitted
1445 for (m = 0; m < qam; m++)
1446 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1447 + pow(Q - symbols_Q[m], 2.0)) / N0;
1448 /* Calculate an LLR for each of the k_64QAM bits in the set */
1449 for (k = 0; k < qm; k++) {
1452 /* For each constellation point */
1453 for (m = 0; m < qam; m++) {
1454 if ((m >> (qm - k - 1)) & 1)
1455 p1 = maxstar(p1, log_syml_prob[m]);
1457 p0 = maxstar(p0, log_syml_prob[m]);
1459 /* Calculate the LLR */
1461 llr_ *= (1 << ldpc_llr_decimals);
1465 if (llr_ < -llr_max)
1467 llrs[qm * i + k] = (int8_t) llr_;
1473 * Generate Qm LLRS for Qm==6
1474 * Modulation, AWGN and LLR estimation from max log development
1477 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1482 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1483 /* 5.1.4 of TS38.211 */
1484 const double symbols_I[64] = {
1485 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1486 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1487 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1488 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1489 -5, -5, -7, -7, -5, -5, -7, -7};
1490 const double symbols_Q[64] = {
1491 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1492 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1493 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1494 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1495 -3, -1, -3, -1, -5, -7, -5, -7};
1496 /* Average constellation point energy */
1498 for (k = 0; k < qm; k++)
1499 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1500 /* 5.1.4 of TS38.211 */
1501 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1502 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1504 I += sqrt(N0 / 2) * randn(0);
1505 Q += sqrt(N0 / 2) * randn(1);
1507 * Calculate the log of the probability that each of
1508 * the constellation points was transmitted
1510 for (m = 0; m < qam; m++)
1511 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1512 + pow(Q - symbols_Q[m], 2.0)) / N0;
1513 /* Calculate an LLR for each of the k_64QAM bits in the set */
1514 for (k = 0; k < qm; k++) {
1517 /* For each constellation point */
1518 for (m = 0; m < qam; m++) {
1519 if ((m >> (qm - k - 1)) & 1)
1520 p1 = maxstar(p1, log_syml_prob[m]);
1522 p0 = maxstar(p0, log_syml_prob[m]);
1524 /* Calculate the LLR */
1526 llr_ *= (1 << ldpc_llr_decimals);
1530 if (llr_ < -llr_max)
1532 llrs[qm * i + k] = (int8_t) llr_;
1537 * Generate Qm LLRS for Qm==4
1538 * Modulation, AWGN and LLR estimation from max log development
1541 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1546 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1547 /* 5.1.4 of TS38.211 */
1548 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1549 -1, -1, -3, -3, -1, -1, -3, -3};
1550 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1551 1, 3, 1, 3, -1, -3, -1, -3};
1552 /* Average constellation point energy */
1554 for (k = 0; k < qm; k++)
1555 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1556 /* 5.1.4 of TS38.211 */
1557 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1558 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1560 I += sqrt(N0 / 2) * randn(0);
1561 Q += sqrt(N0 / 2) * randn(1);
1563 * Calculate the log of the probability that each of
1564 * the constellation points was transmitted
1566 for (m = 0; m < qam; m++)
1567 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1568 + pow(Q - symbols_Q[m], 2.0)) / N0;
1569 /* Calculate an LLR for each of the k_64QAM bits in the set */
1570 for (k = 0; k < qm; k++) {
1573 /* For each constellation point */
1574 for (m = 0; m < qam; m++) {
1575 if ((m >> (qm - k - 1)) & 1)
1576 p1 = maxstar(p1, log_syml_prob[m]);
1578 p0 = maxstar(p0, log_syml_prob[m]);
1580 /* Calculate the LLR */
1582 llr_ *= (1 << ldpc_llr_decimals);
1586 if (llr_ < -llr_max)
1588 llrs[qm * i + k] = (int8_t) llr_;
1593 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1596 double coeff = 2.0 * sqrt(N0);
1598 /* Ignore in vectors rare quasi null LLRs not to be saturated */
1599 if (llrs[j] < 8 && llrs[j] > -8)
1602 /* Note don't change sign here */
1604 b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1606 b = b1 * (1 << ldpc_llr_decimals);
1612 llrs[j] = (int8_t) b;
1615 /* Generate LLR for a given SNR */
1617 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1618 struct rte_bbdev_dec_op *ref_op)
1622 uint32_t i, j, e, range;
1625 e = ref_op->ldpc_dec.cb_params.e;
1626 qm = ref_op->ldpc_dec.q_m;
1627 llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1629 N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1631 for (i = 0; i < n; ++i) {
1633 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1635 for (j = 0; j < range; ++j)
1636 gen_qm8_llr(llrs, j, N0, llr_max);
1637 } else if (qm == 6) {
1638 for (j = 0; j < range; ++j)
1639 gen_qm6_llr(llrs, j, N0, llr_max);
1640 } else if (qm == 4) {
1641 for (j = 0; j < range; ++j)
1642 gen_qm4_llr(llrs, j, N0, llr_max);
1644 for (j = 0; j < e; ++j)
1645 gen_qm2_llr(llrs, j, N0, llr_max);
1651 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1652 unsigned int start_idx,
1653 struct rte_bbdev_op_data *inputs,
1654 struct rte_bbdev_op_data *hard_outputs,
1655 struct rte_bbdev_op_data *soft_outputs,
1656 struct rte_bbdev_op_data *harq_inputs,
1657 struct rte_bbdev_op_data *harq_outputs,
1658 struct rte_bbdev_dec_op *ref_op)
1661 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1663 for (i = 0; i < n; ++i) {
1664 if (ldpc_dec->code_block_mode == 0) {
1665 ops[i]->ldpc_dec.tb_params.ea =
1666 ldpc_dec->tb_params.ea;
1667 ops[i]->ldpc_dec.tb_params.eb =
1668 ldpc_dec->tb_params.eb;
1669 ops[i]->ldpc_dec.tb_params.c =
1670 ldpc_dec->tb_params.c;
1671 ops[i]->ldpc_dec.tb_params.cab =
1672 ldpc_dec->tb_params.cab;
1673 ops[i]->ldpc_dec.tb_params.r =
1674 ldpc_dec->tb_params.r;
1676 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1679 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1680 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1681 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1682 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1683 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1684 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1685 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1686 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1687 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1689 if (hard_outputs != NULL)
1690 ops[i]->ldpc_dec.hard_output =
1691 hard_outputs[start_idx + i];
1693 ops[i]->ldpc_dec.input =
1694 inputs[start_idx + i];
1695 if (soft_outputs != NULL)
1696 ops[i]->ldpc_dec.soft_output =
1697 soft_outputs[start_idx + i];
1698 if (harq_inputs != NULL)
1699 ops[i]->ldpc_dec.harq_combined_input =
1700 harq_inputs[start_idx + i];
1701 if (harq_outputs != NULL)
1702 ops[i]->ldpc_dec.harq_combined_output =
1703 harq_outputs[start_idx + i];
1709 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1710 unsigned int start_idx,
1711 struct rte_bbdev_op_data *inputs,
1712 struct rte_bbdev_op_data *outputs,
1713 struct rte_bbdev_enc_op *ref_op)
1716 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1717 for (i = 0; i < n; ++i) {
1718 if (ldpc_enc->code_block_mode == 0) {
1719 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1720 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1721 ops[i]->ldpc_enc.tb_params.cab =
1722 ldpc_enc->tb_params.cab;
1723 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1724 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1726 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1728 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1729 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1730 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1731 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1732 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1733 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1734 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1735 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1736 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1737 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1742 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1743 unsigned int order_idx, const int expected_status)
1745 int status = op->status;
1746 /* ignore parity mismatch false alarms for long iterations */
1747 if (get_iter_max() >= 10) {
1748 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1749 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1750 printf("WARNING: Ignore Syndrome Check mismatch\n");
1751 status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1753 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1754 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1755 printf("WARNING: Ignore Syndrome Check mismatch\n");
1756 status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1760 TEST_ASSERT(status == expected_status,
1761 "op_status (%d) != expected_status (%d)",
1762 op->status, expected_status);
1764 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1765 "Ordering error, expected %p, got %p",
1766 (void *)(uintptr_t)order_idx, op->opaque_data);
1768 return TEST_SUCCESS;
1772 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1773 unsigned int order_idx, const int expected_status)
1775 TEST_ASSERT(op->status == expected_status,
1776 "op_status (%d) != expected_status (%d)",
1777 op->status, expected_status);
1779 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1780 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1781 "Ordering error, expected %p, got %p",
1782 (void *)(uintptr_t)order_idx, op->opaque_data);
1784 return TEST_SUCCESS;
1788 validate_op_chain(struct rte_bbdev_op_data *op,
1789 struct op_data_entries *orig_op)
1792 struct rte_mbuf *m = op->data;
1793 uint8_t nb_dst_segments = orig_op->nb_segments;
1794 uint32_t total_data_size = 0;
1796 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1797 "Number of segments differ in original (%u) and filled (%u) op",
1798 nb_dst_segments, m->nb_segs);
1800 /* Validate each mbuf segment length */
1801 for (i = 0; i < nb_dst_segments; ++i) {
1802 /* Apply offset to the first mbuf segment */
1803 uint16_t offset = (i == 0) ? op->offset : 0;
1804 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1805 total_data_size += orig_op->segments[i].length;
1807 TEST_ASSERT(orig_op->segments[i].length == data_len,
1808 "Length of segment differ in original (%u) and filled (%u) op",
1809 orig_op->segments[i].length, data_len);
1810 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1811 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1813 "Output buffers (CB=%u) are not equal", i);
1817 /* Validate total mbuf pkt length */
1818 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1819 TEST_ASSERT(total_data_size == pkt_len,
1820 "Length of data differ in original (%u) and filled (%u) op",
1821 total_data_size, pkt_len);
1823 return TEST_SUCCESS;
1827 * Compute K0 for a given configuration for HARQ output length computation
1828 * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1830 static inline uint16_t
1831 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1835 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1838 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1839 else if (rv_index == 2)
1840 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1842 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1844 /* LBRM case - includes a division by N */
1846 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1848 else if (rv_index == 2)
1849 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1852 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1856 /* HARQ output length including the Filler bits */
1857 static inline uint16_t
1858 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1861 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1862 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1863 /* Compute RM out size and number of rows */
1864 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1865 * ops_ld->z_c - ops_ld->n_filler;
1866 uint16_t deRmOutSize = RTE_MIN(
1867 k0 + ops_ld->cb_params.e +
1868 ((k0 > parity_offset) ?
1869 0 : ops_ld->n_filler),
1871 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1873 uint16_t harq_output_len = numRows * ops_ld->z_c;
1874 return harq_output_len;
1878 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1879 struct op_data_entries *orig_op,
1880 struct rte_bbdev_op_ldpc_dec *ops_ld)
1884 struct rte_mbuf *m = op->data;
1885 uint8_t nb_dst_segments = orig_op->nb_segments;
1886 uint32_t total_data_size = 0;
1887 int8_t *harq_orig, *harq_out, abs_harq_origin;
1888 uint32_t byte_error = 0, cum_error = 0, error;
1889 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1890 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1891 uint16_t parity_offset;
1893 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1894 "Number of segments differ in original (%u) and filled (%u) op",
1895 nb_dst_segments, m->nb_segs);
1897 /* Validate each mbuf segment length */
1898 for (i = 0; i < nb_dst_segments; ++i) {
1899 /* Apply offset to the first mbuf segment */
1900 uint16_t offset = (i == 0) ? op->offset : 0;
1901 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1902 total_data_size += orig_op->segments[i].length;
1904 TEST_ASSERT(orig_op->segments[i].length <
1905 (uint32_t)(data_len + 64),
1906 "Length of segment differ in original (%u) and filled (%u) op",
1907 orig_op->segments[i].length, data_len);
1908 harq_orig = (int8_t *) orig_op->segments[i].addr;
1909 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1911 if (!(ldpc_cap_flags &
1912 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1913 ) || (ops_ld->op_flags &
1914 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1915 data_len -= ops_ld->z_c;
1916 parity_offset = data_len;
1918 /* Compute RM out size and number of rows */
1919 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1920 * ops_ld->z_c - ops_ld->n_filler;
1921 uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1923 if (data_len > deRmOutSize)
1924 data_len = deRmOutSize;
1925 if (data_len > orig_op->segments[i].length)
1926 data_len = orig_op->segments[i].length;
1929 * HARQ output can have minor differences
1930 * due to integer representation and related scaling
1932 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1933 if (j == parity_offset) {
1934 /* Special Handling of the filler bits */
1935 for (k = 0; k < ops_ld->n_filler; k++) {
1937 llr_max_pre_scaling) {
1938 printf("HARQ Filler issue %d: %d %d\n",
1946 if (!(ops_ld->op_flags &
1947 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1948 if (ldpc_llr_decimals > 1)
1949 harq_out[jj] = (harq_out[jj] + 1)
1950 >> (ldpc_llr_decimals - 1);
1951 /* Saturated to S7 */
1952 if (harq_orig[j] > llr_max)
1953 harq_orig[j] = llr_max;
1954 if (harq_orig[j] < -llr_max)
1955 harq_orig[j] = -llr_max;
1957 if (harq_orig[j] != harq_out[jj]) {
1958 error = (harq_orig[j] > harq_out[jj]) ?
1959 harq_orig[j] - harq_out[jj] :
1960 harq_out[jj] - harq_orig[j];
1961 abs_harq_origin = harq_orig[j] > 0 ?
1964 /* Residual quantization error */
1965 if ((error > 8 && (abs_harq_origin <
1968 printf("HARQ mismatch %d: exp %d act %d => %d\n",
1970 harq_out[jj], error);
1980 TEST_ASSERT(byte_error <= 1,
1981 "HARQ output mismatch (%d) %d",
1982 byte_error, cum_error);
1984 /* Validate total mbuf pkt length */
1985 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1986 TEST_ASSERT(total_data_size < pkt_len + 64,
1987 "Length of data differ in original (%u) and filled (%u) op",
1988 total_data_size, pkt_len);
1990 return TEST_SUCCESS;
1994 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1995 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1999 struct op_data_entries *hard_data_orig =
2000 &test_vector.entries[DATA_HARD_OUTPUT];
2001 struct op_data_entries *soft_data_orig =
2002 &test_vector.entries[DATA_SOFT_OUTPUT];
2003 struct rte_bbdev_op_turbo_dec *ops_td;
2004 struct rte_bbdev_op_data *hard_output;
2005 struct rte_bbdev_op_data *soft_output;
2006 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
2008 for (i = 0; i < n; ++i) {
2009 ops_td = &ops[i]->turbo_dec;
2010 hard_output = &ops_td->hard_output;
2011 soft_output = &ops_td->soft_output;
2013 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2014 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2015 "Returned iter_count (%d) > expected iter_count (%d)",
2016 ops_td->iter_count, ref_td->iter_count);
2017 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2018 TEST_ASSERT_SUCCESS(ret,
2019 "Checking status and ordering for decoder failed");
2021 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2023 "Hard output buffers (CB=%u) are not equal",
2026 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
2027 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2029 "Soft output buffers (CB=%u) are not equal",
2033 return TEST_SUCCESS;
2036 /* Check Number of code blocks errors */
2038 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
2041 struct op_data_entries *hard_data_orig =
2042 &test_vector.entries[DATA_HARD_OUTPUT];
2043 struct rte_bbdev_op_ldpc_dec *ops_td;
2044 struct rte_bbdev_op_data *hard_output;
2048 for (i = 0; i < n; ++i) {
2049 ops_td = &ops[i]->ldpc_dec;
2050 hard_output = &ops_td->hard_output;
2051 m = hard_output->data;
2052 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
2053 hard_data_orig->segments[0].addr,
2054 hard_data_orig->segments[0].length))
2061 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2062 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2066 struct op_data_entries *hard_data_orig =
2067 &test_vector.entries[DATA_HARD_OUTPUT];
2068 struct op_data_entries *soft_data_orig =
2069 &test_vector.entries[DATA_SOFT_OUTPUT];
2070 struct op_data_entries *harq_data_orig =
2071 &test_vector.entries[DATA_HARQ_OUTPUT];
2072 struct rte_bbdev_op_ldpc_dec *ops_td;
2073 struct rte_bbdev_op_data *hard_output;
2074 struct rte_bbdev_op_data *harq_output;
2075 struct rte_bbdev_op_data *soft_output;
2076 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2078 for (i = 0; i < n; ++i) {
2079 ops_td = &ops[i]->ldpc_dec;
2080 hard_output = &ops_td->hard_output;
2081 harq_output = &ops_td->harq_combined_output;
2082 soft_output = &ops_td->soft_output;
2084 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2085 TEST_ASSERT_SUCCESS(ret,
2086 "Checking status and ordering for decoder failed");
2087 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2088 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2089 "Returned iter_count (%d) > expected iter_count (%d)",
2090 ops_td->iter_count, ref_td->iter_count);
2092 * We can ignore output data when the decoding failed to
2093 * converge or for loop-back cases
2095 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2096 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2098 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2100 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2102 "Hard output buffers (CB=%u) are not equal",
2105 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2106 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2108 "Soft output buffers (CB=%u) are not equal",
2110 if (ref_op->ldpc_dec.op_flags &
2111 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2112 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2113 harq_data_orig, ops_td),
2114 "HARQ output buffers (CB=%u) are not equal",
2117 if (ref_op->ldpc_dec.op_flags &
2118 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2119 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2120 harq_data_orig, ops_td),
2121 "HARQ output buffers (CB=%u) are not equal",
2126 return TEST_SUCCESS;
2131 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2132 struct rte_bbdev_enc_op *ref_op)
2136 struct op_data_entries *hard_data_orig =
2137 &test_vector.entries[DATA_HARD_OUTPUT];
2139 for (i = 0; i < n; ++i) {
2140 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2141 TEST_ASSERT_SUCCESS(ret,
2142 "Checking status and ordering for encoder failed");
2143 TEST_ASSERT_SUCCESS(validate_op_chain(
2144 &ops[i]->turbo_enc.output,
2146 "Output buffers (CB=%u) are not equal",
2150 return TEST_SUCCESS;
2154 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2155 struct rte_bbdev_enc_op *ref_op)
2159 struct op_data_entries *hard_data_orig =
2160 &test_vector.entries[DATA_HARD_OUTPUT];
2162 for (i = 0; i < n; ++i) {
2163 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2164 TEST_ASSERT_SUCCESS(ret,
2165 "Checking status and ordering for encoder failed");
2166 TEST_ASSERT_SUCCESS(validate_op_chain(
2167 &ops[i]->ldpc_enc.output,
2169 "Output buffers (CB=%u) are not equal",
2173 return TEST_SUCCESS;
2177 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2180 struct op_data_entries *entry;
2182 op->turbo_dec = test_vector.turbo_dec;
2183 entry = &test_vector.entries[DATA_INPUT];
2184 for (i = 0; i < entry->nb_segments; ++i)
2185 op->turbo_dec.input.length +=
2186 entry->segments[i].length;
2190 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2193 struct op_data_entries *entry;
2195 op->ldpc_dec = test_vector.ldpc_dec;
2196 entry = &test_vector.entries[DATA_INPUT];
2197 for (i = 0; i < entry->nb_segments; ++i)
2198 op->ldpc_dec.input.length +=
2199 entry->segments[i].length;
2200 if (test_vector.ldpc_dec.op_flags &
2201 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2202 entry = &test_vector.entries[DATA_HARQ_INPUT];
2203 for (i = 0; i < entry->nb_segments; ++i)
2204 op->ldpc_dec.harq_combined_input.length +=
2205 entry->segments[i].length;
2211 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2214 struct op_data_entries *entry;
2216 op->turbo_enc = test_vector.turbo_enc;
2217 entry = &test_vector.entries[DATA_INPUT];
2218 for (i = 0; i < entry->nb_segments; ++i)
2219 op->turbo_enc.input.length +=
2220 entry->segments[i].length;
2224 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2227 struct op_data_entries *entry;
2229 op->ldpc_enc = test_vector.ldpc_enc;
2230 entry = &test_vector.entries[DATA_INPUT];
2231 for (i = 0; i < entry->nb_segments; ++i)
2232 op->ldpc_enc.input.length +=
2233 entry->segments[i].length;
2237 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2240 uint32_t c, r, tb_size = 0;
2242 if (op->turbo_dec.code_block_mode) {
2243 tb_size = op->turbo_dec.tb_params.k_neg;
2245 c = op->turbo_dec.tb_params.c;
2246 r = op->turbo_dec.tb_params.r;
2247 for (i = 0; i < c-r; i++)
2248 tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2249 op->turbo_dec.tb_params.k_neg :
2250 op->turbo_dec.tb_params.k_pos;
2256 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2259 uint32_t c, r, tb_size = 0;
2260 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2262 if (op->ldpc_dec.code_block_mode) {
2263 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2265 c = op->ldpc_dec.tb_params.c;
2266 r = op->ldpc_dec.tb_params.r;
2267 for (i = 0; i < c-r; i++)
2268 tb_size += sys_cols * op->ldpc_dec.z_c
2269 - op->ldpc_dec.n_filler;
2275 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2278 uint32_t c, r, tb_size = 0;
2280 if (op->turbo_enc.code_block_mode) {
2281 tb_size = op->turbo_enc.tb_params.k_neg;
2283 c = op->turbo_enc.tb_params.c;
2284 r = op->turbo_enc.tb_params.r;
2285 for (i = 0; i < c-r; i++)
2286 tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2287 op->turbo_enc.tb_params.k_neg :
2288 op->turbo_enc.tb_params.k_pos;
2294 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2297 uint32_t c, r, tb_size = 0;
2298 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2300 if (op->turbo_enc.code_block_mode) {
2301 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2303 c = op->turbo_enc.tb_params.c;
2304 r = op->turbo_enc.tb_params.r;
2305 for (i = 0; i < c-r; i++)
2306 tb_size += sys_cols * op->ldpc_enc.z_c
2307 - op->ldpc_enc.n_filler;
2314 init_test_op_params(struct test_op_params *op_params,
2315 enum rte_bbdev_op_type op_type, const int expected_status,
2316 const int vector_mask, struct rte_mempool *ops_mp,
2317 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2320 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2321 op_type == RTE_BBDEV_OP_LDPC_DEC)
2322 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2323 &op_params->ref_dec_op, 1);
2325 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2326 &op_params->ref_enc_op, 1);
2328 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2330 op_params->mp = ops_mp;
2331 op_params->burst_sz = burst_sz;
2332 op_params->num_to_process = num_to_process;
2333 op_params->num_lcores = num_lcores;
2334 op_params->vector_mask = vector_mask;
2335 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2336 op_type == RTE_BBDEV_OP_LDPC_DEC)
2337 op_params->ref_dec_op->status = expected_status;
2338 else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2339 || op_type == RTE_BBDEV_OP_LDPC_ENC)
2340 op_params->ref_enc_op->status = expected_status;
2345 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2346 struct test_op_params *op_params)
2348 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2350 struct active_device *ad;
2351 unsigned int burst_sz = get_burst_sz();
2352 enum rte_bbdev_op_type op_type = test_vector.op_type;
2353 const struct rte_bbdev_op_cap *capabilities = NULL;
2355 ad = &active_devs[dev_id];
2357 /* Check if device supports op_type */
2358 if (!is_avail_op(ad, test_vector.op_type))
2359 return TEST_SUCCESS;
2361 struct rte_bbdev_info info;
2362 rte_bbdev_info_get(ad->dev_id, &info);
2363 socket_id = GET_SOCKET(info.socket_id);
2365 f_ret = create_mempools(ad, socket_id, op_type,
2367 if (f_ret != TEST_SUCCESS) {
2368 printf("Couldn't create mempools");
2371 if (op_type == RTE_BBDEV_OP_NONE)
2372 op_type = RTE_BBDEV_OP_TURBO_ENC;
2374 f_ret = init_test_op_params(op_params, test_vector.op_type,
2375 test_vector.expected_status,
2381 if (f_ret != TEST_SUCCESS) {
2382 printf("Couldn't init test op params");
2387 /* Find capabilities */
2388 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2389 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2390 if (cap->type == test_vector.op_type) {
2396 TEST_ASSERT_NOT_NULL(capabilities,
2397 "Couldn't find capabilities");
2399 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2400 create_reference_dec_op(op_params->ref_dec_op);
2401 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2402 create_reference_enc_op(op_params->ref_enc_op);
2403 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2404 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2405 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2406 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2408 for (i = 0; i < ad->nb_queues; ++i) {
2409 f_ret = fill_queue_buffers(op_params,
2411 ad->hard_out_mbuf_pool,
2412 ad->soft_out_mbuf_pool,
2413 ad->harq_in_mbuf_pool,
2414 ad->harq_out_mbuf_pool,
2417 info.drv.min_alignment,
2419 if (f_ret != TEST_SUCCESS) {
2420 printf("Couldn't init queue buffers");
2425 /* Run test case function */
2426 t_ret = test_case_func(ad, op_params);
2428 /* Free active device resources and return */
2429 free_buffers(ad, op_params);
2433 free_buffers(ad, op_params);
2437 /* Run given test function per active device per supported op type
2441 run_test_case(test_case_function *test_case_func)
2446 /* Alloc op_params */
2447 struct test_op_params *op_params = rte_zmalloc(NULL,
2448 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2449 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2450 RTE_ALIGN(sizeof(struct test_op_params),
2451 RTE_CACHE_LINE_SIZE));
2453 /* For each device run test case function */
2454 for (dev = 0; dev < nb_active_devs; ++dev)
2455 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2457 rte_free(op_params);
2463 /* Push back the HARQ output from DDR to host */
2465 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2466 struct rte_bbdev_dec_op **ops,
2470 int save_status, ret;
2471 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2472 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2473 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2474 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2475 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2476 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2477 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2478 for (j = 0; j < n; ++j) {
2479 if ((loopback && mem_out) || hc_out) {
2480 save_status = ops[j]->status;
2481 ops[j]->ldpc_dec.op_flags =
2482 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2483 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2485 ops[j]->ldpc_dec.op_flags +=
2486 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2487 ops[j]->ldpc_dec.harq_combined_input.offset =
2489 ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2490 harq_offset += HARQ_INCR;
2492 ops[j]->ldpc_dec.harq_combined_input.length =
2493 ops[j]->ldpc_dec.harq_combined_output.length;
2494 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2498 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2501 ops[j]->ldpc_dec.op_flags = flags;
2502 ops[j]->status = save_status;
2508 * Push back the HARQ output from HW DDR to Host
2509 * Preload HARQ memory input and adjust HARQ offset
2512 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2513 struct rte_bbdev_dec_op **ops, const uint16_t n,
2518 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2519 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS];
2520 struct rte_bbdev_dec_op *ops_deq[MAX_OPS];
2521 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2522 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2523 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2524 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2525 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2526 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2527 if ((mem_in || hc_in) && preload) {
2528 for (j = 0; j < n; ++j) {
2529 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input;
2530 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output;
2531 ops[j]->ldpc_dec.op_flags =
2532 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2533 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2535 ops[j]->ldpc_dec.op_flags +=
2536 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2537 ops[j]->ldpc_dec.harq_combined_output.offset =
2539 ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2540 harq_offset += HARQ_INCR;
2542 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n);
2545 deq += rte_bbdev_dequeue_ldpc_dec_ops(
2546 dev_id, queue_id, &ops_deq[deq],
2548 /* Restore the operations */
2549 for (j = 0; j < n; ++j) {
2550 ops[j]->ldpc_dec.op_flags = flags;
2551 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j];
2552 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j];
2555 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2556 for (j = 0; j < n; ++j) {
2557 /* Adjust HARQ offset when we reach external DDR */
2558 if (mem_in || hc_in)
2559 ops[j]->ldpc_dec.harq_combined_input.offset
2561 if (mem_out || hc_out)
2562 ops[j]->ldpc_dec.harq_combined_output.offset
2564 harq_offset += HARQ_INCR;
2569 dequeue_event_callback(uint16_t dev_id,
2570 enum rte_bbdev_event_type event, void *cb_arg,
2575 uint64_t total_time;
2576 uint16_t deq, burst_sz, num_ops;
2577 uint16_t queue_id = *(uint16_t *) ret_param;
2578 struct rte_bbdev_info info;
2580 struct thread_params *tp = cb_arg;
2582 /* Find matching thread params using queue_id */
2583 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2584 if (tp->queue_id == queue_id)
2587 if (i == MAX_QUEUES) {
2588 printf("%s: Queue_id from interrupt details was not found!\n",
2593 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2594 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2596 "Dequeue interrupt handler called for incorrect event!\n");
2600 burst_sz = rte_atomic16_read(&tp->burst_sz);
2601 num_ops = tp->op_params->num_to_process;
2603 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2604 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2606 rte_atomic16_read(&tp->nb_dequeued)],
2608 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2609 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2611 rte_atomic16_read(&tp->nb_dequeued)],
2613 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2614 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2616 rte_atomic16_read(&tp->nb_dequeued)],
2618 else /*RTE_BBDEV_OP_TURBO_ENC*/
2619 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2621 rte_atomic16_read(&tp->nb_dequeued)],
2624 if (deq < burst_sz) {
2626 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2628 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2632 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2633 rte_atomic16_add(&tp->nb_dequeued, deq);
2637 total_time = rte_rdtsc_precise() - tp->start_time;
2639 rte_bbdev_info_get(dev_id, &info);
2643 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2644 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2645 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2646 tp->op_params->vector_mask);
2647 /* get the max of iter_count for all dequeued ops */
2648 for (i = 0; i < num_ops; ++i)
2649 tp->iter_count = RTE_MAX(
2650 tp->dec_ops[i]->turbo_dec.iter_count,
2652 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2653 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2654 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2655 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2656 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2657 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2658 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2659 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2660 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2661 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2662 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2663 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2664 tp->op_params->vector_mask);
2665 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2669 printf("Buffers validation failed\n");
2670 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2673 switch (test_vector.op_type) {
2674 case RTE_BBDEV_OP_TURBO_DEC:
2675 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2677 case RTE_BBDEV_OP_TURBO_ENC:
2678 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2680 case RTE_BBDEV_OP_LDPC_DEC:
2681 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2683 case RTE_BBDEV_OP_LDPC_ENC:
2684 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2686 case RTE_BBDEV_OP_NONE:
2690 printf("Unknown op type: %d\n", test_vector.op_type);
2691 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2695 tp->ops_per_sec += ((double)num_ops) /
2696 ((double)total_time / (double)rte_get_tsc_hz());
2697 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2698 ((double)total_time / (double)rte_get_tsc_hz());
2700 rte_atomic16_add(&tp->nb_dequeued, deq);
2704 throughput_intr_lcore_ldpc_dec(void *arg)
2706 struct thread_params *tp = arg;
2707 unsigned int enqueued;
2708 const uint16_t queue_id = tp->queue_id;
2709 const uint16_t burst_sz = tp->op_params->burst_sz;
2710 const uint16_t num_to_process = tp->op_params->num_to_process;
2711 struct rte_bbdev_dec_op *ops[num_to_process];
2712 struct test_buffers *bufs = NULL;
2713 struct rte_bbdev_info info;
2715 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2716 uint16_t num_to_enq, enq;
2718 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2719 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2720 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2721 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2723 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2724 "BURST_SIZE should be <= %u", MAX_BURST);
2726 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2727 "Failed to enable interrupts for dev: %u, queue_id: %u",
2728 tp->dev_id, queue_id);
2730 rte_bbdev_info_get(tp->dev_id, &info);
2732 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2733 "NUM_OPS cannot exceed %u for this device",
2734 info.drv.queue_size_lim);
2736 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2738 rte_atomic16_clear(&tp->processing_status);
2739 rte_atomic16_clear(&tp->nb_dequeued);
2741 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2744 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2746 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2748 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2749 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2750 bufs->hard_outputs, bufs->soft_outputs,
2751 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2753 /* Set counter to validate the ordering */
2754 for (j = 0; j < num_to_process; ++j)
2755 ops[j]->opaque_data = (void *)(uintptr_t)j;
2757 for (j = 0; j < TEST_REPETITIONS; ++j) {
2758 for (i = 0; i < num_to_process; ++i) {
2761 ops[i]->ldpc_dec.hard_output.data);
2762 if (hc_out || loopback)
2764 ops[i]->ldpc_dec.harq_combined_output.data);
2767 tp->start_time = rte_rdtsc_precise();
2768 for (enqueued = 0; enqueued < num_to_process;) {
2769 num_to_enq = burst_sz;
2771 if (unlikely(num_to_process - enqueued < num_to_enq))
2772 num_to_enq = num_to_process - enqueued;
2776 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2778 queue_id, &ops[enqueued],
2780 } while (unlikely(num_to_enq != enq));
2783 /* Write to thread burst_sz current number of enqueued
2784 * descriptors. It ensures that proper number of
2785 * descriptors will be dequeued in callback
2786 * function - needed for last batch in case where
2787 * the number of operations is not a multiple of
2790 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2792 /* Wait until processing of previous batch is
2795 while (rte_atomic16_read(&tp->nb_dequeued) !=
2799 if (j != TEST_REPETITIONS - 1)
2800 rte_atomic16_clear(&tp->nb_dequeued);
2803 return TEST_SUCCESS;
2807 throughput_intr_lcore_dec(void *arg)
2809 struct thread_params *tp = arg;
2810 unsigned int enqueued;
2811 const uint16_t queue_id = tp->queue_id;
2812 const uint16_t burst_sz = tp->op_params->burst_sz;
2813 const uint16_t num_to_process = tp->op_params->num_to_process;
2814 struct rte_bbdev_dec_op *ops[num_to_process];
2815 struct test_buffers *bufs = NULL;
2816 struct rte_bbdev_info info;
2818 uint16_t num_to_enq, enq;
2820 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2821 "BURST_SIZE should be <= %u", MAX_BURST);
2823 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2824 "Failed to enable interrupts for dev: %u, queue_id: %u",
2825 tp->dev_id, queue_id);
2827 rte_bbdev_info_get(tp->dev_id, &info);
2829 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2830 "NUM_OPS cannot exceed %u for this device",
2831 info.drv.queue_size_lim);
2833 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2835 rte_atomic16_clear(&tp->processing_status);
2836 rte_atomic16_clear(&tp->nb_dequeued);
2838 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2841 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2843 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2845 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2846 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2847 bufs->hard_outputs, bufs->soft_outputs,
2848 tp->op_params->ref_dec_op);
2850 /* Set counter to validate the ordering */
2851 for (j = 0; j < num_to_process; ++j)
2852 ops[j]->opaque_data = (void *)(uintptr_t)j;
2854 for (j = 0; j < TEST_REPETITIONS; ++j) {
2855 for (i = 0; i < num_to_process; ++i)
2856 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2858 tp->start_time = rte_rdtsc_precise();
2859 for (enqueued = 0; enqueued < num_to_process;) {
2860 num_to_enq = burst_sz;
2862 if (unlikely(num_to_process - enqueued < num_to_enq))
2863 num_to_enq = num_to_process - enqueued;
2867 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2868 queue_id, &ops[enqueued],
2870 } while (unlikely(num_to_enq != enq));
2873 /* Write to thread burst_sz current number of enqueued
2874 * descriptors. It ensures that proper number of
2875 * descriptors will be dequeued in callback
2876 * function - needed for last batch in case where
2877 * the number of operations is not a multiple of
2880 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2882 /* Wait until processing of previous batch is
2885 while (rte_atomic16_read(&tp->nb_dequeued) !=
2889 if (j != TEST_REPETITIONS - 1)
2890 rte_atomic16_clear(&tp->nb_dequeued);
2893 return TEST_SUCCESS;
2897 throughput_intr_lcore_enc(void *arg)
2899 struct thread_params *tp = arg;
2900 unsigned int enqueued;
2901 const uint16_t queue_id = tp->queue_id;
2902 const uint16_t burst_sz = tp->op_params->burst_sz;
2903 const uint16_t num_to_process = tp->op_params->num_to_process;
2904 struct rte_bbdev_enc_op *ops[num_to_process];
2905 struct test_buffers *bufs = NULL;
2906 struct rte_bbdev_info info;
2908 uint16_t num_to_enq, enq;
2910 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2911 "BURST_SIZE should be <= %u", MAX_BURST);
2913 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2914 "Failed to enable interrupts for dev: %u, queue_id: %u",
2915 tp->dev_id, queue_id);
2917 rte_bbdev_info_get(tp->dev_id, &info);
2919 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2920 "NUM_OPS cannot exceed %u for this device",
2921 info.drv.queue_size_lim);
2923 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2925 rte_atomic16_clear(&tp->processing_status);
2926 rte_atomic16_clear(&tp->nb_dequeued);
2928 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2931 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2933 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2935 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2936 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2937 bufs->hard_outputs, tp->op_params->ref_enc_op);
2939 /* Set counter to validate the ordering */
2940 for (j = 0; j < num_to_process; ++j)
2941 ops[j]->opaque_data = (void *)(uintptr_t)j;
2943 for (j = 0; j < TEST_REPETITIONS; ++j) {
2944 for (i = 0; i < num_to_process; ++i)
2945 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2947 tp->start_time = rte_rdtsc_precise();
2948 for (enqueued = 0; enqueued < num_to_process;) {
2949 num_to_enq = burst_sz;
2951 if (unlikely(num_to_process - enqueued < num_to_enq))
2952 num_to_enq = num_to_process - enqueued;
2956 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2957 queue_id, &ops[enqueued],
2959 } while (unlikely(enq != num_to_enq));
2962 /* Write to thread burst_sz current number of enqueued
2963 * descriptors. It ensures that proper number of
2964 * descriptors will be dequeued in callback
2965 * function - needed for last batch in case where
2966 * the number of operations is not a multiple of
2969 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2971 /* Wait until processing of previous batch is
2974 while (rte_atomic16_read(&tp->nb_dequeued) !=
2978 if (j != TEST_REPETITIONS - 1)
2979 rte_atomic16_clear(&tp->nb_dequeued);
2982 return TEST_SUCCESS;
2987 throughput_intr_lcore_ldpc_enc(void *arg)
2989 struct thread_params *tp = arg;
2990 unsigned int enqueued;
2991 const uint16_t queue_id = tp->queue_id;
2992 const uint16_t burst_sz = tp->op_params->burst_sz;
2993 const uint16_t num_to_process = tp->op_params->num_to_process;
2994 struct rte_bbdev_enc_op *ops[num_to_process];
2995 struct test_buffers *bufs = NULL;
2996 struct rte_bbdev_info info;
2998 uint16_t num_to_enq, enq;
3000 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3001 "BURST_SIZE should be <= %u", MAX_BURST);
3003 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
3004 "Failed to enable interrupts for dev: %u, queue_id: %u",
3005 tp->dev_id, queue_id);
3007 rte_bbdev_info_get(tp->dev_id, &info);
3009 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
3010 "NUM_OPS cannot exceed %u for this device",
3011 info.drv.queue_size_lim);
3013 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3015 rte_atomic16_clear(&tp->processing_status);
3016 rte_atomic16_clear(&tp->nb_dequeued);
3018 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3021 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
3023 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3025 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3026 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
3027 bufs->inputs, bufs->hard_outputs,
3028 tp->op_params->ref_enc_op);
3030 /* Set counter to validate the ordering */
3031 for (j = 0; j < num_to_process; ++j)
3032 ops[j]->opaque_data = (void *)(uintptr_t)j;
3034 for (j = 0; j < TEST_REPETITIONS; ++j) {
3035 for (i = 0; i < num_to_process; ++i)
3036 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
3038 tp->start_time = rte_rdtsc_precise();
3039 for (enqueued = 0; enqueued < num_to_process;) {
3040 num_to_enq = burst_sz;
3042 if (unlikely(num_to_process - enqueued < num_to_enq))
3043 num_to_enq = num_to_process - enqueued;
3047 enq += rte_bbdev_enqueue_ldpc_enc_ops(
3049 queue_id, &ops[enqueued],
3051 } while (unlikely(enq != num_to_enq));
3054 /* Write to thread burst_sz current number of enqueued
3055 * descriptors. It ensures that proper number of
3056 * descriptors will be dequeued in callback
3057 * function - needed for last batch in case where
3058 * the number of operations is not a multiple of
3061 rte_atomic16_set(&tp->burst_sz, num_to_enq);
3063 /* Wait until processing of previous batch is
3066 while (rte_atomic16_read(&tp->nb_dequeued) !=
3070 if (j != TEST_REPETITIONS - 1)
3071 rte_atomic16_clear(&tp->nb_dequeued);
3074 return TEST_SUCCESS;
3078 throughput_pmd_lcore_dec(void *arg)
3080 struct thread_params *tp = arg;
3082 uint64_t total_time = 0, start_time;
3083 const uint16_t queue_id = tp->queue_id;
3084 const uint16_t burst_sz = tp->op_params->burst_sz;
3085 const uint16_t num_ops = tp->op_params->num_to_process;
3086 struct rte_bbdev_dec_op *ops_enq[num_ops];
3087 struct rte_bbdev_dec_op *ops_deq[num_ops];
3088 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3089 struct test_buffers *bufs = NULL;
3091 struct rte_bbdev_info info;
3092 uint16_t num_to_enq;
3094 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3095 "BURST_SIZE should be <= %u", MAX_BURST);
3097 rte_bbdev_info_get(tp->dev_id, &info);
3099 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3100 "NUM_OPS cannot exceed %u for this device",
3101 info.drv.queue_size_lim);
3103 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3105 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3108 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3109 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3111 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3112 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3113 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3115 /* Set counter to validate the ordering */
3116 for (j = 0; j < num_ops; ++j)
3117 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3119 for (i = 0; i < TEST_REPETITIONS; ++i) {
3121 for (j = 0; j < num_ops; ++j)
3122 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3124 start_time = rte_rdtsc_precise();
3126 for (enq = 0, deq = 0; enq < num_ops;) {
3127 num_to_enq = burst_sz;
3129 if (unlikely(num_ops - enq < num_to_enq))
3130 num_to_enq = num_ops - enq;
3132 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3133 queue_id, &ops_enq[enq], num_to_enq);
3135 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3136 queue_id, &ops_deq[deq], enq - deq);
3139 /* dequeue the remaining */
3141 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3142 queue_id, &ops_deq[deq], enq - deq);
3145 total_time += rte_rdtsc_precise() - start_time;
3149 /* get the max of iter_count for all dequeued ops */
3150 for (i = 0; i < num_ops; ++i) {
3151 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3155 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3156 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3157 tp->op_params->vector_mask);
3158 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3161 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3163 double tb_len_bits = calc_dec_TB_size(ref_op);
3165 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3166 ((double)total_time / (double)rte_get_tsc_hz());
3167 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3168 1000000.0) / ((double)total_time /
3169 (double)rte_get_tsc_hz());
3171 return TEST_SUCCESS;
3175 bler_pmd_lcore_ldpc_dec(void *arg)
3177 struct thread_params *tp = arg;
3179 uint64_t total_time = 0, start_time;
3180 const uint16_t queue_id = tp->queue_id;
3181 const uint16_t burst_sz = tp->op_params->burst_sz;
3182 const uint16_t num_ops = tp->op_params->num_to_process;
3183 struct rte_bbdev_dec_op *ops_enq[num_ops];
3184 struct rte_bbdev_dec_op *ops_deq[num_ops];
3185 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3186 struct test_buffers *bufs = NULL;
3188 float parity_bler = 0;
3189 struct rte_bbdev_info info;
3190 uint16_t num_to_enq;
3191 bool extDdr = check_bit(ldpc_cap_flags,
3192 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3193 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3194 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3195 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3196 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3198 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3199 "BURST_SIZE should be <= %u", MAX_BURST);
3201 rte_bbdev_info_get(tp->dev_id, &info);
3203 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3204 "NUM_OPS cannot exceed %u for this device",
3205 info.drv.queue_size_lim);
3207 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3209 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3212 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3213 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3215 /* For BLER tests we need to enable early termination */
3216 if (!check_bit(ref_op->ldpc_dec.op_flags,
3217 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3218 ref_op->ldpc_dec.op_flags +=
3219 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3220 ref_op->ldpc_dec.iter_max = get_iter_max();
3221 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3223 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3224 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3225 bufs->hard_outputs, bufs->soft_outputs,
3226 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3227 generate_llr_input(num_ops, bufs->inputs, ref_op);
3229 /* Set counter to validate the ordering */
3230 for (j = 0; j < num_ops; ++j)
3231 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3233 for (i = 0; i < 1; ++i) { /* Could add more iterations */
3234 for (j = 0; j < num_ops; ++j) {
3237 ops_enq[j]->ldpc_dec.hard_output.data);
3238 if (hc_out || loopback)
3240 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3243 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3245 start_time = rte_rdtsc_precise();
3247 for (enq = 0, deq = 0; enq < num_ops;) {
3248 num_to_enq = burst_sz;
3250 if (unlikely(num_ops - enq < num_to_enq))
3251 num_to_enq = num_ops - enq;
3253 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3254 queue_id, &ops_enq[enq], num_to_enq);
3256 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3257 queue_id, &ops_deq[deq], enq - deq);
3260 /* dequeue the remaining */
3262 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3263 queue_id, &ops_deq[deq], enq - deq);
3266 total_time += rte_rdtsc_precise() - start_time;
3270 tp->iter_average = 0;
3271 /* get the max of iter_count for all dequeued ops */
3272 for (i = 0; i < num_ops; ++i) {
3273 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3275 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3276 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3280 parity_bler /= num_ops; /* This one is based on SYND */
3281 tp->iter_average /= num_ops;
3282 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3284 if (test_vector.op_type != RTE_BBDEV_OP_NONE
3288 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3289 tp->op_params->vector_mask);
3290 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3293 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3295 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3296 tp->ops_per_sec = ((double)num_ops * 1) /
3297 ((double)total_time / (double)rte_get_tsc_hz());
3298 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3299 1000000.0) / ((double)total_time /
3300 (double)rte_get_tsc_hz());
3302 return TEST_SUCCESS;
3306 throughput_pmd_lcore_ldpc_dec(void *arg)
3308 struct thread_params *tp = arg;
3310 uint64_t total_time = 0, start_time;
3311 const uint16_t queue_id = tp->queue_id;
3312 const uint16_t burst_sz = tp->op_params->burst_sz;
3313 const uint16_t num_ops = tp->op_params->num_to_process;
3314 struct rte_bbdev_dec_op *ops_enq[num_ops];
3315 struct rte_bbdev_dec_op *ops_deq[num_ops];
3316 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3317 struct test_buffers *bufs = NULL;
3319 struct rte_bbdev_info info;
3320 uint16_t num_to_enq;
3321 bool extDdr = check_bit(ldpc_cap_flags,
3322 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3323 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3324 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3325 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3326 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3328 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3329 "BURST_SIZE should be <= %u", MAX_BURST);
3331 rte_bbdev_info_get(tp->dev_id, &info);
3333 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3334 "NUM_OPS cannot exceed %u for this device",
3335 info.drv.queue_size_lim);
3337 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3339 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3342 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3343 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3345 /* For throughput tests we need to disable early termination */
3346 if (check_bit(ref_op->ldpc_dec.op_flags,
3347 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3348 ref_op->ldpc_dec.op_flags -=
3349 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3350 ref_op->ldpc_dec.iter_max = get_iter_max();
3351 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3353 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3354 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3355 bufs->hard_outputs, bufs->soft_outputs,
3356 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3358 /* Set counter to validate the ordering */
3359 for (j = 0; j < num_ops; ++j)
3360 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3362 for (i = 0; i < TEST_REPETITIONS; ++i) {
3363 for (j = 0; j < num_ops; ++j) {
3366 ops_enq[j]->ldpc_dec.hard_output.data);
3367 if (hc_out || loopback)
3369 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3372 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3374 start_time = rte_rdtsc_precise();
3376 for (enq = 0, deq = 0; enq < num_ops;) {
3377 num_to_enq = burst_sz;
3379 if (unlikely(num_ops - enq < num_to_enq))
3380 num_to_enq = num_ops - enq;
3382 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3383 queue_id, &ops_enq[enq], num_to_enq);
3385 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3386 queue_id, &ops_deq[deq], enq - deq);
3389 /* dequeue the remaining */
3391 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3392 queue_id, &ops_deq[deq], enq - deq);
3395 total_time += rte_rdtsc_precise() - start_time;
3399 /* get the max of iter_count for all dequeued ops */
3400 for (i = 0; i < num_ops; ++i) {
3401 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3405 /* Read loopback is not thread safe */
3406 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3409 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3410 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3411 tp->op_params->vector_mask);
3412 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3415 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3417 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3419 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3420 ((double)total_time / (double)rte_get_tsc_hz());
3421 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3422 1000000.0) / ((double)total_time /
3423 (double)rte_get_tsc_hz());
3425 return TEST_SUCCESS;
3429 throughput_pmd_lcore_enc(void *arg)
3431 struct thread_params *tp = arg;
3433 uint64_t total_time = 0, start_time;
3434 const uint16_t queue_id = tp->queue_id;
3435 const uint16_t burst_sz = tp->op_params->burst_sz;
3436 const uint16_t num_ops = tp->op_params->num_to_process;
3437 struct rte_bbdev_enc_op *ops_enq[num_ops];
3438 struct rte_bbdev_enc_op *ops_deq[num_ops];
3439 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3440 struct test_buffers *bufs = NULL;
3442 struct rte_bbdev_info info;
3443 uint16_t num_to_enq;
3445 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3446 "BURST_SIZE should be <= %u", MAX_BURST);
3448 rte_bbdev_info_get(tp->dev_id, &info);
3450 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3451 "NUM_OPS cannot exceed %u for this device",
3452 info.drv.queue_size_lim);
3454 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3456 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3459 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3461 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3463 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3464 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3465 bufs->hard_outputs, ref_op);
3467 /* Set counter to validate the ordering */
3468 for (j = 0; j < num_ops; ++j)
3469 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3471 for (i = 0; i < TEST_REPETITIONS; ++i) {
3473 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3474 for (j = 0; j < num_ops; ++j)
3475 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3477 start_time = rte_rdtsc_precise();
3479 for (enq = 0, deq = 0; enq < num_ops;) {
3480 num_to_enq = burst_sz;
3482 if (unlikely(num_ops - enq < num_to_enq))
3483 num_to_enq = num_ops - enq;
3485 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3486 queue_id, &ops_enq[enq], num_to_enq);
3488 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3489 queue_id, &ops_deq[deq], enq - deq);
3492 /* dequeue the remaining */
3494 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3495 queue_id, &ops_deq[deq], enq - deq);
3498 total_time += rte_rdtsc_precise() - start_time;
3501 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3502 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3503 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3506 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3508 double tb_len_bits = calc_enc_TB_size(ref_op);
3510 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3511 ((double)total_time / (double)rte_get_tsc_hz());
3512 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3513 / 1000000.0) / ((double)total_time /
3514 (double)rte_get_tsc_hz());
3516 return TEST_SUCCESS;
3520 throughput_pmd_lcore_ldpc_enc(void *arg)
3522 struct thread_params *tp = arg;
3524 uint64_t total_time = 0, start_time;
3525 const uint16_t queue_id = tp->queue_id;
3526 const uint16_t burst_sz = tp->op_params->burst_sz;
3527 const uint16_t num_ops = tp->op_params->num_to_process;
3528 struct rte_bbdev_enc_op *ops_enq[num_ops];
3529 struct rte_bbdev_enc_op *ops_deq[num_ops];
3530 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3531 struct test_buffers *bufs = NULL;
3533 struct rte_bbdev_info info;
3534 uint16_t num_to_enq;
3536 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3537 "BURST_SIZE should be <= %u", MAX_BURST);
3539 rte_bbdev_info_get(tp->dev_id, &info);
3541 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3542 "NUM_OPS cannot exceed %u for this device",
3543 info.drv.queue_size_lim);
3545 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3547 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3550 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3552 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3554 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3555 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3556 bufs->hard_outputs, ref_op);
3558 /* Set counter to validate the ordering */
3559 for (j = 0; j < num_ops; ++j)
3560 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3562 for (i = 0; i < TEST_REPETITIONS; ++i) {
3564 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3565 for (j = 0; j < num_ops; ++j)
3566 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3568 start_time = rte_rdtsc_precise();
3570 for (enq = 0, deq = 0; enq < num_ops;) {
3571 num_to_enq = burst_sz;
3573 if (unlikely(num_ops - enq < num_to_enq))
3574 num_to_enq = num_ops - enq;
3576 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3577 queue_id, &ops_enq[enq], num_to_enq);
3579 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3580 queue_id, &ops_deq[deq], enq - deq);
3583 /* dequeue the remaining */
3585 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3586 queue_id, &ops_deq[deq], enq - deq);
3589 total_time += rte_rdtsc_precise() - start_time;
3592 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3593 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3594 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3597 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3599 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3601 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3602 ((double)total_time / (double)rte_get_tsc_hz());
3603 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3604 / 1000000.0) / ((double)total_time /
3605 (double)rte_get_tsc_hz());
3607 return TEST_SUCCESS;
3611 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3613 unsigned int iter = 0;
3614 double total_mops = 0, total_mbps = 0;
3616 for (iter = 0; iter < used_cores; iter++) {
3618 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3619 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3620 t_params[iter].mbps);
3621 total_mops += t_params[iter].ops_per_sec;
3622 total_mbps += t_params[iter].mbps;
3625 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3626 used_cores, total_mops, total_mbps);
3629 /* Aggregate the performance results over the number of cores used */
3631 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3633 unsigned int core_idx = 0;
3634 double total_mops = 0, total_mbps = 0;
3635 uint8_t iter_count = 0;
3637 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3639 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3640 t_params[core_idx].lcore_id,
3641 t_params[core_idx].ops_per_sec,
3642 t_params[core_idx].mbps,
3643 t_params[core_idx].iter_count);
3644 total_mops += t_params[core_idx].ops_per_sec;
3645 total_mbps += t_params[core_idx].mbps;
3646 iter_count = RTE_MAX(iter_count,
3647 t_params[core_idx].iter_count);
3650 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3651 used_cores, total_mops, total_mbps, iter_count);
3654 /* Aggregate the performance results over the number of cores used */
3656 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3658 unsigned int core_idx = 0;
3659 double total_mbps = 0, total_bler = 0, total_iter = 0;
3660 double snr = get_snr();
3662 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3663 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3664 t_params[core_idx].lcore_id,
3665 t_params[core_idx].bler * 100,
3666 t_params[core_idx].iter_average,
3667 t_params[core_idx].mbps,
3668 get_vector_filename());
3669 total_mbps += t_params[core_idx].mbps;
3670 total_bler += t_params[core_idx].bler;
3671 total_iter += t_params[core_idx].iter_average;
3673 total_bler /= used_cores;
3674 total_iter /= used_cores;
3676 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3677 snr, total_bler * 100, total_iter, get_iter_max(),
3678 total_mbps, get_vector_filename());
3682 * Test function that determines BLER wireless performance
3685 bler_test(struct active_device *ad,
3686 struct test_op_params *op_params)
3689 unsigned int lcore_id, used_cores = 0;
3690 struct thread_params *t_params;
3691 struct rte_bbdev_info info;
3692 lcore_function_t *bler_function;
3693 uint16_t num_lcores;
3694 const char *op_type_str;
3696 rte_bbdev_info_get(ad->dev_id, &info);
3698 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3699 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3700 test_vector.op_type);
3702 printf("+ ------------------------------------------------------- +\n");
3703 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3704 info.dev_name, ad->nb_queues, op_params->burst_sz,
3705 op_params->num_to_process, op_params->num_lcores,
3707 intr_enabled ? "Interrupt mode" : "PMD mode",
3708 (double)rte_get_tsc_hz() / 1000000000.0);
3710 /* Set number of lcores */
3711 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3713 : op_params->num_lcores;
3715 /* Allocate memory for thread parameters structure */
3716 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3717 RTE_CACHE_LINE_SIZE);
3718 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3719 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3720 RTE_CACHE_LINE_SIZE));
3722 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) &&
3723 !check_bit(test_vector.ldpc_dec.op_flags,
3724 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
3725 && !check_bit(test_vector.ldpc_dec.op_flags,
3726 RTE_BBDEV_LDPC_LLR_COMPRESSION))
3727 bler_function = bler_pmd_lcore_ldpc_dec;
3729 return TEST_SKIPPED;
3731 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3733 /* Main core is set at first entry */
3734 t_params[0].dev_id = ad->dev_id;
3735 t_params[0].lcore_id = rte_lcore_id();
3736 t_params[0].op_params = op_params;
3737 t_params[0].queue_id = ad->queue_ids[used_cores++];
3738 t_params[0].iter_count = 0;
3740 RTE_LCORE_FOREACH_WORKER(lcore_id) {
3741 if (used_cores >= num_lcores)
3744 t_params[used_cores].dev_id = ad->dev_id;
3745 t_params[used_cores].lcore_id = lcore_id;
3746 t_params[used_cores].op_params = op_params;
3747 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3748 t_params[used_cores].iter_count = 0;
3750 rte_eal_remote_launch(bler_function,
3751 &t_params[used_cores++], lcore_id);
3754 rte_atomic16_set(&op_params->sync, SYNC_START);
3755 ret = bler_function(&t_params[0]);
3757 /* Main core is always used */
3758 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3759 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3761 print_dec_bler(t_params, num_lcores);
3763 /* Return if test failed */
3769 /* Function to print something here*/
3775 * Test function that determines how long an enqueue + dequeue of a burst
3776 * takes on available lcores.
3779 throughput_test(struct active_device *ad,
3780 struct test_op_params *op_params)
3783 unsigned int lcore_id, used_cores = 0;
3784 struct thread_params *t_params, *tp;
3785 struct rte_bbdev_info info;
3786 lcore_function_t *throughput_function;
3787 uint16_t num_lcores;
3788 const char *op_type_str;
3790 rte_bbdev_info_get(ad->dev_id, &info);
3792 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3793 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3794 test_vector.op_type);
3796 printf("+ ------------------------------------------------------- +\n");
3797 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3798 info.dev_name, ad->nb_queues, op_params->burst_sz,
3799 op_params->num_to_process, op_params->num_lcores,
3801 intr_enabled ? "Interrupt mode" : "PMD mode",
3802 (double)rte_get_tsc_hz() / 1000000000.0);
3804 /* Set number of lcores */
3805 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3807 : op_params->num_lcores;
3809 /* Allocate memory for thread parameters structure */
3810 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3811 RTE_CACHE_LINE_SIZE);
3812 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3813 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3814 RTE_CACHE_LINE_SIZE));
3817 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3818 throughput_function = throughput_intr_lcore_dec;
3819 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3820 throughput_function = throughput_intr_lcore_ldpc_dec;
3821 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3822 throughput_function = throughput_intr_lcore_enc;
3823 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3824 throughput_function = throughput_intr_lcore_ldpc_enc;
3826 throughput_function = throughput_intr_lcore_enc;
3828 /* Dequeue interrupt callback registration */
3829 ret = rte_bbdev_callback_register(ad->dev_id,
3830 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3837 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3838 throughput_function = throughput_pmd_lcore_dec;
3839 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3840 throughput_function = throughput_pmd_lcore_ldpc_dec;
3841 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3842 throughput_function = throughput_pmd_lcore_enc;
3843 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3844 throughput_function = throughput_pmd_lcore_ldpc_enc;
3846 throughput_function = throughput_pmd_lcore_enc;
3849 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3851 /* Main core is set at first entry */
3852 t_params[0].dev_id = ad->dev_id;
3853 t_params[0].lcore_id = rte_lcore_id();
3854 t_params[0].op_params = op_params;
3855 t_params[0].queue_id = ad->queue_ids[used_cores++];
3856 t_params[0].iter_count = 0;
3858 RTE_LCORE_FOREACH_WORKER(lcore_id) {
3859 if (used_cores >= num_lcores)
3862 t_params[used_cores].dev_id = ad->dev_id;
3863 t_params[used_cores].lcore_id = lcore_id;
3864 t_params[used_cores].op_params = op_params;
3865 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3866 t_params[used_cores].iter_count = 0;
3868 rte_eal_remote_launch(throughput_function,
3869 &t_params[used_cores++], lcore_id);
3872 rte_atomic16_set(&op_params->sync, SYNC_START);
3873 ret = throughput_function(&t_params[0]);
3875 /* Main core is always used */
3876 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3877 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3879 /* Return if test failed */
3885 /* Print throughput if interrupts are disabled and test passed */
3886 if (!intr_enabled) {
3887 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3888 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3889 print_dec_throughput(t_params, num_lcores);
3891 print_enc_throughput(t_params, num_lcores);
3896 /* In interrupt TC we need to wait for the interrupt callback to deqeue
3897 * all pending operations. Skip waiting for queues which reported an
3898 * error using processing_status variable.
3899 * Wait for main lcore operations.
3902 while ((rte_atomic16_read(&tp->nb_dequeued) <
3903 op_params->num_to_process) &&
3904 (rte_atomic16_read(&tp->processing_status) !=
3908 tp->ops_per_sec /= TEST_REPETITIONS;
3909 tp->mbps /= TEST_REPETITIONS;
3910 ret |= (int)rte_atomic16_read(&tp->processing_status);
3912 /* Wait for worker lcores operations */
3913 for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3914 tp = &t_params[used_cores];
3916 while ((rte_atomic16_read(&tp->nb_dequeued) <
3917 op_params->num_to_process) &&
3918 (rte_atomic16_read(&tp->processing_status) !=
3922 tp->ops_per_sec /= TEST_REPETITIONS;
3923 tp->mbps /= TEST_REPETITIONS;
3924 ret |= (int)rte_atomic16_read(&tp->processing_status);
3927 /* Print throughput if test passed */
3929 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3930 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3931 print_dec_throughput(t_params, num_lcores);
3932 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3933 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3934 print_enc_throughput(t_params, num_lcores);
3942 latency_test_dec(struct rte_mempool *mempool,
3943 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3944 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3945 const uint16_t num_to_process, uint16_t burst_sz,
3946 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3948 int ret = TEST_SUCCESS;
3949 uint16_t i, j, dequeued;
3950 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3951 uint64_t start_time = 0, last_time = 0;
3953 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3954 uint16_t enq = 0, deq = 0;
3955 bool first_time = true;
3958 if (unlikely(num_to_process - dequeued < burst_sz))
3959 burst_sz = num_to_process - dequeued;
3961 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3962 TEST_ASSERT_SUCCESS(ret,
3963 "rte_bbdev_dec_op_alloc_bulk() failed");
3964 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3965 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3971 /* Set counter to validate the ordering */
3972 for (j = 0; j < burst_sz; ++j)
3973 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3975 start_time = rte_rdtsc_precise();
3977 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
3979 TEST_ASSERT(enq == burst_sz,
3980 "Error enqueueing burst, expected %u, got %u",
3985 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3986 &ops_deq[deq], burst_sz - deq);
3987 if (likely(first_time && (deq > 0))) {
3988 last_time = rte_rdtsc_precise() - start_time;
3991 } while (unlikely(burst_sz != deq));
3993 *max_time = RTE_MAX(*max_time, last_time);
3994 *min_time = RTE_MIN(*min_time, last_time);
3995 *total_time += last_time;
3997 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3998 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
4000 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4003 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4010 /* Test case for latency/validation for LDPC Decoder */
4012 latency_test_ldpc_dec(struct rte_mempool *mempool,
4013 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
4014 int vector_mask, uint16_t dev_id, uint16_t queue_id,
4015 const uint16_t num_to_process, uint16_t burst_sz,
4016 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time,
4019 int ret = TEST_SUCCESS;
4020 uint16_t i, j, dequeued;
4021 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4022 uint64_t start_time = 0, last_time = 0;
4023 bool extDdr = ldpc_cap_flags &
4024 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4026 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4027 uint16_t enq = 0, deq = 0;
4028 bool first_time = true;
4031 if (unlikely(num_to_process - dequeued < burst_sz))
4032 burst_sz = num_to_process - dequeued;
4034 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4035 TEST_ASSERT_SUCCESS(ret,
4036 "rte_bbdev_dec_op_alloc_bulk() failed");
4038 /* For latency tests we need to disable early termination */
4039 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags,
4040 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
4041 ref_op->ldpc_dec.op_flags -=
4042 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
4043 ref_op->ldpc_dec.iter_max = get_iter_max();
4044 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
4046 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4047 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4056 preload_harq_ddr(dev_id, queue_id, ops_enq,
4059 /* Set counter to validate the ordering */
4060 for (j = 0; j < burst_sz; ++j)
4061 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4063 start_time = rte_rdtsc_precise();
4065 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4066 &ops_enq[enq], burst_sz);
4067 TEST_ASSERT(enq == burst_sz,
4068 "Error enqueueing burst, expected %u, got %u",
4073 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4074 &ops_deq[deq], burst_sz - deq);
4075 if (likely(first_time && (deq > 0))) {
4076 last_time = rte_rdtsc_precise() - start_time;
4079 } while (unlikely(burst_sz != deq));
4081 *max_time = RTE_MAX(*max_time, last_time);
4082 *min_time = RTE_MIN(*min_time, last_time);
4083 *total_time += last_time;
4086 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4088 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4089 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4091 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4094 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4101 latency_test_enc(struct rte_mempool *mempool,
4102 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4103 uint16_t dev_id, uint16_t queue_id,
4104 const uint16_t num_to_process, uint16_t burst_sz,
4105 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4107 int ret = TEST_SUCCESS;
4108 uint16_t i, j, dequeued;
4109 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4110 uint64_t start_time = 0, last_time = 0;
4112 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4113 uint16_t enq = 0, deq = 0;
4114 bool first_time = true;
4117 if (unlikely(num_to_process - dequeued < burst_sz))
4118 burst_sz = num_to_process - dequeued;
4120 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4121 TEST_ASSERT_SUCCESS(ret,
4122 "rte_bbdev_enc_op_alloc_bulk() failed");
4123 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4124 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4129 /* Set counter to validate the ordering */
4130 for (j = 0; j < burst_sz; ++j)
4131 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4133 start_time = rte_rdtsc_precise();
4135 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4137 TEST_ASSERT(enq == burst_sz,
4138 "Error enqueueing burst, expected %u, got %u",
4143 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4144 &ops_deq[deq], burst_sz - deq);
4145 if (likely(first_time && (deq > 0))) {
4146 last_time += rte_rdtsc_precise() - start_time;
4149 } while (unlikely(burst_sz != deq));
4151 *max_time = RTE_MAX(*max_time, last_time);
4152 *min_time = RTE_MIN(*min_time, last_time);
4153 *total_time += last_time;
4155 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4156 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4157 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4160 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4168 latency_test_ldpc_enc(struct rte_mempool *mempool,
4169 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4170 uint16_t dev_id, uint16_t queue_id,
4171 const uint16_t num_to_process, uint16_t burst_sz,
4172 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4174 int ret = TEST_SUCCESS;
4175 uint16_t i, j, dequeued;
4176 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4177 uint64_t start_time = 0, last_time = 0;
4179 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4180 uint16_t enq = 0, deq = 0;
4181 bool first_time = true;
4184 if (unlikely(num_to_process - dequeued < burst_sz))
4185 burst_sz = num_to_process - dequeued;
4187 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4188 TEST_ASSERT_SUCCESS(ret,
4189 "rte_bbdev_enc_op_alloc_bulk() failed");
4190 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4191 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4196 /* Set counter to validate the ordering */
4197 for (j = 0; j < burst_sz; ++j)
4198 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4200 start_time = rte_rdtsc_precise();
4202 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4203 &ops_enq[enq], burst_sz);
4204 TEST_ASSERT(enq == burst_sz,
4205 "Error enqueueing burst, expected %u, got %u",
4210 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4211 &ops_deq[deq], burst_sz - deq);
4212 if (likely(first_time && (deq > 0))) {
4213 last_time += rte_rdtsc_precise() - start_time;
4216 } while (unlikely(burst_sz != deq));
4218 *max_time = RTE_MAX(*max_time, last_time);
4219 *min_time = RTE_MIN(*min_time, last_time);
4220 *total_time += last_time;
4222 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4223 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4224 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4227 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4234 /* Common function for running validation and latency test cases */
4236 validation_latency_test(struct active_device *ad,
4237 struct test_op_params *op_params, bool latency_flag)
4240 uint16_t burst_sz = op_params->burst_sz;
4241 const uint16_t num_to_process = op_params->num_to_process;
4242 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4243 const uint16_t queue_id = ad->queue_ids[0];
4244 struct test_buffers *bufs = NULL;
4245 struct rte_bbdev_info info;
4246 uint64_t total_time, min_time, max_time;
4247 const char *op_type_str;
4249 total_time = max_time = 0;
4250 min_time = UINT64_MAX;
4252 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4253 "BURST_SIZE should be <= %u", MAX_BURST);
4255 rte_bbdev_info_get(ad->dev_id, &info);
4256 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4258 op_type_str = rte_bbdev_op_type_str(op_type);
4259 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4261 printf("+ ------------------------------------------------------- +\n");
4263 printf("== test: latency\ndev:");
4265 printf("== test: validation\ndev:");
4266 printf("%s, burst size: %u, num ops: %u, op type: %s\n",
4267 info.dev_name, burst_sz, num_to_process, op_type_str);
4269 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4270 iter = latency_test_dec(op_params->mp, bufs,
4271 op_params->ref_dec_op, op_params->vector_mask,
4272 ad->dev_id, queue_id, num_to_process,
4273 burst_sz, &total_time, &min_time, &max_time);
4274 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4275 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4276 op_params->ref_enc_op, ad->dev_id, queue_id,
4277 num_to_process, burst_sz, &total_time,
4278 &min_time, &max_time);
4279 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4280 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4281 op_params->ref_dec_op, op_params->vector_mask,
4282 ad->dev_id, queue_id, num_to_process,
4283 burst_sz, &total_time, &min_time, &max_time,
4285 else /* RTE_BBDEV_OP_TURBO_ENC */
4286 iter = latency_test_enc(op_params->mp, bufs,
4287 op_params->ref_enc_op,
4288 ad->dev_id, queue_id,
4289 num_to_process, burst_sz, &total_time,
4290 &min_time, &max_time);
4295 printf("Operation latency:\n"
4296 "\tavg: %lg cycles, %lg us\n"
4297 "\tmin: %lg cycles, %lg us\n"
4298 "\tmax: %lg cycles, %lg us\n",
4299 (double)total_time / (double)iter,
4300 (double)(total_time * 1000000) / (double)iter /
4301 (double)rte_get_tsc_hz(), (double)min_time,
4302 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4303 (double)max_time, (double)(max_time * 1000000) /
4304 (double)rte_get_tsc_hz());
4306 return TEST_SUCCESS;
4310 latency_test(struct active_device *ad, struct test_op_params *op_params)
4312 return validation_latency_test(ad, op_params, true);
4316 validation_test(struct active_device *ad, struct test_op_params *op_params)
4318 return validation_latency_test(ad, op_params, false);
4321 #ifdef RTE_BBDEV_OFFLOAD_COST
4323 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4324 struct rte_bbdev_stats *stats)
4326 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4327 struct rte_bbdev_stats *q_stats;
4329 if (queue_id >= dev->data->num_queues)
4332 q_stats = &dev->data->queues[queue_id].queue_stats;
4334 stats->enqueued_count = q_stats->enqueued_count;
4335 stats->dequeued_count = q_stats->dequeued_count;
4336 stats->enqueue_err_count = q_stats->enqueue_err_count;
4337 stats->dequeue_err_count = q_stats->dequeue_err_count;
4338 stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4344 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4345 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4346 uint16_t queue_id, const uint16_t num_to_process,
4347 uint16_t burst_sz, struct test_time_stats *time_st)
4349 int i, dequeued, ret;
4350 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4351 uint64_t enq_start_time, deq_start_time;
4352 uint64_t enq_sw_last_time, deq_last_time;
4353 struct rte_bbdev_stats stats;
4355 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4356 uint16_t enq = 0, deq = 0;
4358 if (unlikely(num_to_process - dequeued < burst_sz))
4359 burst_sz = num_to_process - dequeued;
4361 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4362 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4363 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4369 /* Start time meas for enqueue function offload latency */
4370 enq_start_time = rte_rdtsc_precise();
4372 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4373 &ops_enq[enq], burst_sz - enq);
4374 } while (unlikely(burst_sz != enq));
4376 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4377 TEST_ASSERT_SUCCESS(ret,
4378 "Failed to get stats for queue (%u) of device (%u)",
4381 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4382 stats.acc_offload_cycles;
4383 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4385 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4387 time_st->enq_sw_total_time += enq_sw_last_time;
4389 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4390 stats.acc_offload_cycles);
4391 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4392 stats.acc_offload_cycles);
4393 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4395 /* give time for device to process ops */
4396 rte_delay_us(WAIT_OFFLOAD_US);
4398 /* Start time meas for dequeue function offload latency */
4399 deq_start_time = rte_rdtsc_precise();
4400 /* Dequeue one operation */
4402 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4403 &ops_deq[deq], enq);
4404 } while (unlikely(deq == 0));
4406 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4407 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4409 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4411 time_st->deq_total_time += deq_last_time;
4413 /* Dequeue remaining operations if needed*/
4414 while (burst_sz != deq)
4415 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4416 &ops_deq[deq], burst_sz - deq);
4418 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4426 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4427 struct test_buffers *bufs,
4428 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4429 uint16_t queue_id, const uint16_t num_to_process,
4430 uint16_t burst_sz, struct test_time_stats *time_st)
4432 int i, dequeued, ret;
4433 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4434 uint64_t enq_start_time, deq_start_time;
4435 uint64_t enq_sw_last_time, deq_last_time;
4436 struct rte_bbdev_stats stats;
4437 bool extDdr = ldpc_cap_flags &
4438 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4440 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4441 uint16_t enq = 0, deq = 0;
4443 if (unlikely(num_to_process - dequeued < burst_sz))
4444 burst_sz = num_to_process - dequeued;
4446 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4447 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4448 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4457 preload_harq_ddr(dev_id, queue_id, ops_enq,
4460 /* Start time meas for enqueue function offload latency */
4461 enq_start_time = rte_rdtsc_precise();
4463 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4464 &ops_enq[enq], burst_sz - enq);
4465 } while (unlikely(burst_sz != enq));
4467 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4468 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4469 TEST_ASSERT_SUCCESS(ret,
4470 "Failed to get stats for queue (%u) of device (%u)",
4473 enq_sw_last_time -= stats.acc_offload_cycles;
4474 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4476 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4478 time_st->enq_sw_total_time += enq_sw_last_time;
4480 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4481 stats.acc_offload_cycles);
4482 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4483 stats.acc_offload_cycles);
4484 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4486 /* give time for device to process ops */
4487 rte_delay_us(WAIT_OFFLOAD_US);
4489 /* Start time meas for dequeue function offload latency */
4490 deq_start_time = rte_rdtsc_precise();
4491 /* Dequeue one operation */
4493 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4494 &ops_deq[deq], enq);
4495 } while (unlikely(deq == 0));
4497 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4498 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4500 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4502 time_st->deq_total_time += deq_last_time;
4504 /* Dequeue remaining operations if needed*/
4505 while (burst_sz != deq)
4506 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4507 &ops_deq[deq], burst_sz - deq);
4510 /* Read loopback is not thread safe */
4511 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4514 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4522 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4523 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4524 uint16_t queue_id, const uint16_t num_to_process,
4525 uint16_t burst_sz, struct test_time_stats *time_st)
4527 int i, dequeued, ret;
4528 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4529 uint64_t enq_start_time, deq_start_time;
4530 uint64_t enq_sw_last_time, deq_last_time;
4531 struct rte_bbdev_stats stats;
4533 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4534 uint16_t enq = 0, deq = 0;
4536 if (unlikely(num_to_process - dequeued < burst_sz))
4537 burst_sz = num_to_process - dequeued;
4539 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4540 TEST_ASSERT_SUCCESS(ret,
4541 "rte_bbdev_enc_op_alloc_bulk() failed");
4542 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4543 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4548 /* Start time meas for enqueue function offload latency */
4549 enq_start_time = rte_rdtsc_precise();
4551 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4552 &ops_enq[enq], burst_sz - enq);
4553 } while (unlikely(burst_sz != enq));
4555 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4557 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4558 TEST_ASSERT_SUCCESS(ret,
4559 "Failed to get stats for queue (%u) of device (%u)",
4561 enq_sw_last_time -= stats.acc_offload_cycles;
4562 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4564 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4566 time_st->enq_sw_total_time += enq_sw_last_time;
4568 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4569 stats.acc_offload_cycles);
4570 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4571 stats.acc_offload_cycles);
4572 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4574 /* give time for device to process ops */
4575 rte_delay_us(WAIT_OFFLOAD_US);
4577 /* Start time meas for dequeue function offload latency */
4578 deq_start_time = rte_rdtsc_precise();
4579 /* Dequeue one operation */
4581 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4582 &ops_deq[deq], enq);
4583 } while (unlikely(deq == 0));
4585 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4586 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4588 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4590 time_st->deq_total_time += deq_last_time;
4592 while (burst_sz != deq)
4593 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4594 &ops_deq[deq], burst_sz - deq);
4596 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4604 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4605 struct test_buffers *bufs,
4606 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4607 uint16_t queue_id, const uint16_t num_to_process,
4608 uint16_t burst_sz, struct test_time_stats *time_st)
4610 int i, dequeued, ret;
4611 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4612 uint64_t enq_start_time, deq_start_time;
4613 uint64_t enq_sw_last_time, deq_last_time;
4614 struct rte_bbdev_stats stats;
4616 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4617 uint16_t enq = 0, deq = 0;
4619 if (unlikely(num_to_process - dequeued < burst_sz))
4620 burst_sz = num_to_process - dequeued;
4622 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4623 TEST_ASSERT_SUCCESS(ret,
4624 "rte_bbdev_enc_op_alloc_bulk() failed");
4625 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4626 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4631 /* Start time meas for enqueue function offload latency */
4632 enq_start_time = rte_rdtsc_precise();
4634 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4635 &ops_enq[enq], burst_sz - enq);
4636 } while (unlikely(burst_sz != enq));
4638 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4639 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4640 TEST_ASSERT_SUCCESS(ret,
4641 "Failed to get stats for queue (%u) of device (%u)",
4644 enq_sw_last_time -= stats.acc_offload_cycles;
4645 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4647 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4649 time_st->enq_sw_total_time += enq_sw_last_time;
4651 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4652 stats.acc_offload_cycles);
4653 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4654 stats.acc_offload_cycles);
4655 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4657 /* give time for device to process ops */
4658 rte_delay_us(WAIT_OFFLOAD_US);
4660 /* Start time meas for dequeue function offload latency */
4661 deq_start_time = rte_rdtsc_precise();
4662 /* Dequeue one operation */
4664 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4665 &ops_deq[deq], enq);
4666 } while (unlikely(deq == 0));
4668 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4669 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4671 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4673 time_st->deq_total_time += deq_last_time;
4675 while (burst_sz != deq)
4676 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4677 &ops_deq[deq], burst_sz - deq);
4679 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4688 offload_cost_test(struct active_device *ad,
4689 struct test_op_params *op_params)
4691 #ifndef RTE_BBDEV_OFFLOAD_COST
4693 RTE_SET_USED(op_params);
4694 printf("Offload latency test is disabled.\n");
4695 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4696 return TEST_SKIPPED;
4699 uint16_t burst_sz = op_params->burst_sz;
4700 const uint16_t num_to_process = op_params->num_to_process;
4701 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4702 const uint16_t queue_id = ad->queue_ids[0];
4703 struct test_buffers *bufs = NULL;
4704 struct rte_bbdev_info info;
4705 const char *op_type_str;
4706 struct test_time_stats time_st;
4708 memset(&time_st, 0, sizeof(struct test_time_stats));
4709 time_st.enq_sw_min_time = UINT64_MAX;
4710 time_st.enq_acc_min_time = UINT64_MAX;
4711 time_st.deq_min_time = UINT64_MAX;
4713 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4714 "BURST_SIZE should be <= %u", MAX_BURST);
4716 rte_bbdev_info_get(ad->dev_id, &info);
4717 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4719 op_type_str = rte_bbdev_op_type_str(op_type);
4720 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4722 printf("+ ------------------------------------------------------- +\n");
4723 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4724 info.dev_name, burst_sz, num_to_process, op_type_str);
4726 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4727 iter = offload_latency_test_dec(op_params->mp, bufs,
4728 op_params->ref_dec_op, ad->dev_id, queue_id,
4729 num_to_process, burst_sz, &time_st);
4730 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4731 iter = offload_latency_test_enc(op_params->mp, bufs,
4732 op_params->ref_enc_op, ad->dev_id, queue_id,
4733 num_to_process, burst_sz, &time_st);
4734 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4735 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4736 op_params->ref_enc_op, ad->dev_id, queue_id,
4737 num_to_process, burst_sz, &time_st);
4738 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4739 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4740 op_params->ref_dec_op, ad->dev_id, queue_id,
4741 num_to_process, burst_sz, &time_st);
4743 iter = offload_latency_test_enc(op_params->mp, bufs,
4744 op_params->ref_enc_op, ad->dev_id, queue_id,
4745 num_to_process, burst_sz, &time_st);
4750 printf("Enqueue driver offload cost latency:\n"
4751 "\tavg: %lg cycles, %lg us\n"
4752 "\tmin: %lg cycles, %lg us\n"
4753 "\tmax: %lg cycles, %lg us\n"
4754 "Enqueue accelerator offload cost latency:\n"
4755 "\tavg: %lg cycles, %lg us\n"
4756 "\tmin: %lg cycles, %lg us\n"
4757 "\tmax: %lg cycles, %lg us\n",
4758 (double)time_st.enq_sw_total_time / (double)iter,
4759 (double)(time_st.enq_sw_total_time * 1000000) /
4760 (double)iter / (double)rte_get_tsc_hz(),
4761 (double)time_st.enq_sw_min_time,
4762 (double)(time_st.enq_sw_min_time * 1000000) /
4763 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4764 (double)(time_st.enq_sw_max_time * 1000000) /
4765 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4767 (double)(time_st.enq_acc_total_time * 1000000) /
4768 (double)iter / (double)rte_get_tsc_hz(),
4769 (double)time_st.enq_acc_min_time,
4770 (double)(time_st.enq_acc_min_time * 1000000) /
4771 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4772 (double)(time_st.enq_acc_max_time * 1000000) /
4775 printf("Dequeue offload cost latency - one op:\n"
4776 "\tavg: %lg cycles, %lg us\n"
4777 "\tmin: %lg cycles, %lg us\n"
4778 "\tmax: %lg cycles, %lg us\n",
4779 (double)time_st.deq_total_time / (double)iter,
4780 (double)(time_st.deq_total_time * 1000000) /
4781 (double)iter / (double)rte_get_tsc_hz(),
4782 (double)time_st.deq_min_time,
4783 (double)(time_st.deq_min_time * 1000000) /
4784 rte_get_tsc_hz(), (double)time_st.deq_max_time,
4785 (double)(time_st.deq_max_time * 1000000) /
4788 struct rte_bbdev_stats stats = {0};
4789 get_bbdev_queue_stats(ad->dev_id, queue_id, &stats);
4790 if (op_type != RTE_BBDEV_OP_LDPC_DEC) {
4791 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process,
4792 "Mismatch in enqueue count %10"PRIu64" %d",
4793 stats.enqueued_count, num_to_process);
4794 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process,
4795 "Mismatch in dequeue count %10"PRIu64" %d",
4796 stats.dequeued_count, num_to_process);
4798 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0,
4799 "Enqueue count Error %10"PRIu64"",
4800 stats.enqueue_err_count);
4801 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0,
4802 "Dequeue count Error (%10"PRIu64"",
4803 stats.dequeue_err_count);
4805 return TEST_SUCCESS;
4809 #ifdef RTE_BBDEV_OFFLOAD_COST
4811 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4812 const uint16_t num_to_process, uint16_t burst_sz,
4813 uint64_t *deq_total_time, uint64_t *deq_min_time,
4814 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4817 struct rte_bbdev_dec_op *ops[MAX_BURST];
4818 uint64_t deq_start_time, deq_last_time;
4820 /* Test deq offload latency from an empty queue */
4822 for (i = 0, deq_total = 0; deq_total < num_to_process;
4823 ++i, deq_total += burst_sz) {
4824 deq_start_time = rte_rdtsc_precise();
4826 if (unlikely(num_to_process - deq_total < burst_sz))
4827 burst_sz = num_to_process - deq_total;
4828 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4829 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4832 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4835 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4836 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4837 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4838 *deq_total_time += deq_last_time;
4845 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4846 const uint16_t num_to_process, uint16_t burst_sz,
4847 uint64_t *deq_total_time, uint64_t *deq_min_time,
4848 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4851 struct rte_bbdev_enc_op *ops[MAX_BURST];
4852 uint64_t deq_start_time, deq_last_time;
4854 /* Test deq offload latency from an empty queue */
4855 for (i = 0, deq_total = 0; deq_total < num_to_process;
4856 ++i, deq_total += burst_sz) {
4857 deq_start_time = rte_rdtsc_precise();
4859 if (unlikely(num_to_process - deq_total < burst_sz))
4860 burst_sz = num_to_process - deq_total;
4861 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4862 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4865 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4868 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4869 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4870 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4871 *deq_total_time += deq_last_time;
4880 offload_latency_empty_q_test(struct active_device *ad,
4881 struct test_op_params *op_params)
4883 #ifndef RTE_BBDEV_OFFLOAD_COST
4885 RTE_SET_USED(op_params);
4886 printf("Offload latency empty dequeue test is disabled.\n");
4887 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4888 return TEST_SKIPPED;
4891 uint64_t deq_total_time, deq_min_time, deq_max_time;
4892 uint16_t burst_sz = op_params->burst_sz;
4893 const uint16_t num_to_process = op_params->num_to_process;
4894 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4895 const uint16_t queue_id = ad->queue_ids[0];
4896 struct rte_bbdev_info info;
4897 const char *op_type_str;
4899 deq_total_time = deq_max_time = 0;
4900 deq_min_time = UINT64_MAX;
4902 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4903 "BURST_SIZE should be <= %u", MAX_BURST);
4905 rte_bbdev_info_get(ad->dev_id, &info);
4907 op_type_str = rte_bbdev_op_type_str(op_type);
4908 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4910 printf("+ ------------------------------------------------------- +\n");
4911 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4912 info.dev_name, burst_sz, num_to_process, op_type_str);
4914 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4915 op_type == RTE_BBDEV_OP_LDPC_DEC)
4916 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4917 num_to_process, burst_sz, &deq_total_time,
4918 &deq_min_time, &deq_max_time, op_type);
4920 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4921 num_to_process, burst_sz, &deq_total_time,
4922 &deq_min_time, &deq_max_time, op_type);
4927 printf("Empty dequeue offload:\n"
4928 "\tavg: %lg cycles, %lg us\n"
4929 "\tmin: %lg cycles, %lg us\n"
4930 "\tmax: %lg cycles, %lg us\n",
4931 (double)deq_total_time / (double)iter,
4932 (double)(deq_total_time * 1000000) / (double)iter /
4933 (double)rte_get_tsc_hz(), (double)deq_min_time,
4934 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4935 (double)deq_max_time, (double)(deq_max_time * 1000000) /
4938 return TEST_SUCCESS;
4945 return run_test_case(bler_test);
4951 return run_test_case(throughput_test);
4955 offload_cost_tc(void)
4957 return run_test_case(offload_cost_test);
4961 offload_latency_empty_q_tc(void)
4963 return run_test_case(offload_latency_empty_q_test);
4969 return run_test_case(latency_test);
4975 return run_test_case(validation_test);
4981 return run_test_case(throughput_test);
4984 static struct unit_test_suite bbdev_bler_testsuite = {
4985 .suite_name = "BBdev BLER Tests",
4986 .setup = testsuite_setup,
4987 .teardown = testsuite_teardown,
4988 .unit_test_cases = {
4989 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
4990 TEST_CASES_END() /**< NULL terminate unit test array */
4994 static struct unit_test_suite bbdev_throughput_testsuite = {
4995 .suite_name = "BBdev Throughput Tests",
4996 .setup = testsuite_setup,
4997 .teardown = testsuite_teardown,
4998 .unit_test_cases = {
4999 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
5000 TEST_CASES_END() /**< NULL terminate unit test array */
5004 static struct unit_test_suite bbdev_validation_testsuite = {
5005 .suite_name = "BBdev Validation Tests",
5006 .setup = testsuite_setup,
5007 .teardown = testsuite_teardown,
5008 .unit_test_cases = {
5009 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc),
5010 TEST_CASES_END() /**< NULL terminate unit test array */
5014 static struct unit_test_suite bbdev_latency_testsuite = {
5015 .suite_name = "BBdev Latency Tests",
5016 .setup = testsuite_setup,
5017 .teardown = testsuite_teardown,
5018 .unit_test_cases = {
5019 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
5020 TEST_CASES_END() /**< NULL terminate unit test array */
5024 static struct unit_test_suite bbdev_offload_cost_testsuite = {
5025 .suite_name = "BBdev Offload Cost Tests",
5026 .setup = testsuite_setup,
5027 .teardown = testsuite_teardown,
5028 .unit_test_cases = {
5029 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
5030 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
5031 TEST_CASES_END() /**< NULL terminate unit test array */
5035 static struct unit_test_suite bbdev_interrupt_testsuite = {
5036 .suite_name = "BBdev Interrupt Tests",
5037 .setup = interrupt_testsuite_setup,
5038 .teardown = testsuite_teardown,
5039 .unit_test_cases = {
5040 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
5041 TEST_CASES_END() /**< NULL terminate unit test array */
5045 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
5046 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
5047 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
5048 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
5049 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
5050 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);