1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
19 #include <rte_interrupts.h>
22 #include "test_bbdev_vector.h"
24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
26 #define MAX_QUEUES RTE_MAX_LCORE
27 #define TEST_REPETITIONS 1000
29 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
30 #include <fpga_lte_fec.h>
31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
33 #define VF_UL_4G_QUEUE_VALUE 4
34 #define VF_DL_4G_QUEUE_VALUE 4
35 #define UL_4G_BANDWIDTH 3
36 #define DL_4G_BANDWIDTH 3
37 #define UL_4G_LOAD_BALANCE 128
38 #define DL_4G_LOAD_BALANCE 128
39 #define FLR_4G_TIMEOUT 610
42 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC
43 #include <rte_pmd_fpga_5gnr_fec.h>
44 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
45 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
46 #define VF_UL_5G_QUEUE_VALUE 4
47 #define VF_DL_5G_QUEUE_VALUE 4
48 #define UL_5G_BANDWIDTH 3
49 #define DL_5G_BANDWIDTH 3
50 #define UL_5G_LOAD_BALANCE 128
51 #define DL_5G_LOAD_BALANCE 128
52 #define FLR_5G_TIMEOUT 610
55 #define OPS_CACHE_SIZE 256U
56 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
60 #define INVALID_OPAQUE -1
62 #define INVALID_QUEUE_ID -1
63 /* Increment for next code block in external HARQ memory */
64 #define HARQ_INCR 32768
65 /* Headroom for filler LLRs insertion in HARQ buffer */
66 #define FILLER_HEADROOM 1024
67 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
68 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
69 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
70 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
71 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
72 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
73 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
74 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
75 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
77 static struct test_bbdev_vector test_vector;
79 /* Switch between PMD and Interrupt for throughput TC */
80 static bool intr_enabled;
82 /* LLR arithmetic representation for numerical conversion */
83 static int ldpc_llr_decimals;
84 static int ldpc_llr_size;
85 /* Keep track of the LDPC decoder device capability flag */
86 static uint32_t ldpc_cap_flags;
88 /* Represents tested active devices */
89 static struct active_device {
90 const char *driver_name;
92 uint16_t supported_ops;
93 uint16_t queue_ids[MAX_QUEUES];
95 struct rte_mempool *ops_mempool;
96 struct rte_mempool *in_mbuf_pool;
97 struct rte_mempool *hard_out_mbuf_pool;
98 struct rte_mempool *soft_out_mbuf_pool;
99 struct rte_mempool *harq_in_mbuf_pool;
100 struct rte_mempool *harq_out_mbuf_pool;
101 } active_devs[RTE_BBDEV_MAX_DEVS];
103 static uint8_t nb_active_devs;
105 /* Data buffers used by BBDEV ops */
106 struct test_buffers {
107 struct rte_bbdev_op_data *inputs;
108 struct rte_bbdev_op_data *hard_outputs;
109 struct rte_bbdev_op_data *soft_outputs;
110 struct rte_bbdev_op_data *harq_inputs;
111 struct rte_bbdev_op_data *harq_outputs;
114 /* Operation parameters specific for given test case */
115 struct test_op_params {
116 struct rte_mempool *mp;
117 struct rte_bbdev_dec_op *ref_dec_op;
118 struct rte_bbdev_enc_op *ref_enc_op;
120 uint16_t num_to_process;
124 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
127 /* Contains per lcore params */
128 struct thread_params {
138 rte_atomic16_t nb_dequeued;
139 rte_atomic16_t processing_status;
140 rte_atomic16_t burst_sz;
141 struct test_op_params *op_params;
142 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
143 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
146 #ifdef RTE_BBDEV_OFFLOAD_COST
147 /* Stores time statistics */
148 struct test_time_stats {
149 /* Stores software enqueue total working time */
150 uint64_t enq_sw_total_time;
151 /* Stores minimum value of software enqueue working time */
152 uint64_t enq_sw_min_time;
153 /* Stores maximum value of software enqueue working time */
154 uint64_t enq_sw_max_time;
155 /* Stores turbo enqueue total working time */
156 uint64_t enq_acc_total_time;
157 /* Stores minimum value of accelerator enqueue working time */
158 uint64_t enq_acc_min_time;
159 /* Stores maximum value of accelerator enqueue working time */
160 uint64_t enq_acc_max_time;
161 /* Stores dequeue total working time */
162 uint64_t deq_total_time;
163 /* Stores minimum value of dequeue working time */
164 uint64_t deq_min_time;
165 /* Stores maximum value of dequeue working time */
166 uint64_t deq_max_time;
170 typedef int (test_case_function)(struct active_device *ad,
171 struct test_op_params *op_params);
174 mbuf_reset(struct rte_mbuf *m)
184 /* Read flag value 0/1 from bitmap */
186 check_bit(uint32_t bitmap, uint32_t bitmask)
188 return bitmap & bitmask;
192 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
194 ad->supported_ops |= (1 << op_type);
198 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
200 return ad->supported_ops & (1 << op_type);
204 flags_match(uint32_t flags_req, uint32_t flags_present)
206 return (flags_req & flags_present) == flags_req;
210 clear_soft_out_cap(uint32_t *op_flags)
212 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
213 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
214 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
218 check_dev_cap(const struct rte_bbdev_info *dev_info)
221 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
222 nb_harq_inputs, nb_harq_outputs;
223 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
225 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
226 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
227 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
228 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
229 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
231 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
232 if (op_cap->type != test_vector.op_type)
235 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
236 const struct rte_bbdev_op_cap_turbo_dec *cap =
237 &op_cap->cap.turbo_dec;
238 /* Ignore lack of soft output capability, just skip
239 * checking if soft output is valid.
241 if ((test_vector.turbo_dec.op_flags &
242 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
243 !(cap->capability_flags &
244 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
246 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
249 &test_vector.turbo_dec.op_flags);
252 if (!flags_match(test_vector.turbo_dec.op_flags,
253 cap->capability_flags))
255 if (nb_inputs > cap->num_buffers_src) {
256 printf("Too many inputs defined: %u, max: %u\n",
257 nb_inputs, cap->num_buffers_src);
260 if (nb_soft_outputs > cap->num_buffers_soft_out &&
261 (test_vector.turbo_dec.op_flags &
262 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
264 "Too many soft outputs defined: %u, max: %u\n",
266 cap->num_buffers_soft_out);
269 if (nb_hard_outputs > cap->num_buffers_hard_out) {
271 "Too many hard outputs defined: %u, max: %u\n",
273 cap->num_buffers_hard_out);
276 if (intr_enabled && !(cap->capability_flags &
277 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
279 "Dequeue interrupts are not supported!\n");
284 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
285 const struct rte_bbdev_op_cap_turbo_enc *cap =
286 &op_cap->cap.turbo_enc;
288 if (!flags_match(test_vector.turbo_enc.op_flags,
289 cap->capability_flags))
291 if (nb_inputs > cap->num_buffers_src) {
292 printf("Too many inputs defined: %u, max: %u\n",
293 nb_inputs, cap->num_buffers_src);
296 if (nb_hard_outputs > cap->num_buffers_dst) {
298 "Too many hard outputs defined: %u, max: %u\n",
299 nb_hard_outputs, cap->num_buffers_dst);
302 if (intr_enabled && !(cap->capability_flags &
303 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
305 "Dequeue interrupts are not supported!\n");
310 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
311 const struct rte_bbdev_op_cap_ldpc_enc *cap =
312 &op_cap->cap.ldpc_enc;
314 if (!flags_match(test_vector.ldpc_enc.op_flags,
315 cap->capability_flags)){
316 printf("Flag Mismatch\n");
319 if (nb_inputs > cap->num_buffers_src) {
320 printf("Too many inputs defined: %u, max: %u\n",
321 nb_inputs, cap->num_buffers_src);
324 if (nb_hard_outputs > cap->num_buffers_dst) {
326 "Too many hard outputs defined: %u, max: %u\n",
327 nb_hard_outputs, cap->num_buffers_dst);
330 if (intr_enabled && !(cap->capability_flags &
331 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
333 "Dequeue interrupts are not supported!\n");
338 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
339 const struct rte_bbdev_op_cap_ldpc_dec *cap =
340 &op_cap->cap.ldpc_dec;
342 if (!flags_match(test_vector.ldpc_dec.op_flags,
343 cap->capability_flags)){
344 printf("Flag Mismatch\n");
347 if (nb_inputs > cap->num_buffers_src) {
348 printf("Too many inputs defined: %u, max: %u\n",
349 nb_inputs, cap->num_buffers_src);
352 if (nb_hard_outputs > cap->num_buffers_hard_out) {
354 "Too many hard outputs defined: %u, max: %u\n",
356 cap->num_buffers_hard_out);
359 if (nb_harq_inputs > cap->num_buffers_hard_out) {
361 "Too many HARQ inputs defined: %u, max: %u\n",
363 cap->num_buffers_hard_out);
366 if (nb_harq_outputs > cap->num_buffers_hard_out) {
368 "Too many HARQ outputs defined: %u, max: %u\n",
370 cap->num_buffers_hard_out);
373 if (intr_enabled && !(cap->capability_flags &
374 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
376 "Dequeue interrupts are not supported!\n");
379 if (intr_enabled && (test_vector.ldpc_dec.op_flags &
380 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
381 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
382 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
384 printf("Skip loop-back with interrupt\n");
391 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
392 return TEST_SUCCESS; /* Special case for NULL device */
397 /* calculates optimal mempool size not smaller than the val */
399 optimal_mempool_size(unsigned int val)
401 return rte_align32pow2(val + 1) - 1;
404 /* allocates mbuf mempool for inputs and outputs */
405 static struct rte_mempool *
406 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
407 int socket_id, unsigned int mbuf_pool_size,
408 const char *op_type_str)
411 uint32_t max_seg_sz = 0;
412 char pool_name[RTE_MEMPOOL_NAMESIZE];
414 /* find max input segment size */
415 for (i = 0; i < entries->nb_segments; ++i)
416 if (entries->segments[i].length > max_seg_sz)
417 max_seg_sz = entries->segments[i].length;
419 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
421 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
422 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
424 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
428 create_mempools(struct active_device *ad, int socket_id,
429 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
431 struct rte_mempool *mp;
432 unsigned int ops_pool_size, mbuf_pool_size = 0;
433 char pool_name[RTE_MEMPOOL_NAMESIZE];
434 const char *op_type_str;
435 enum rte_bbdev_op_type op_type = org_op_type;
437 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
438 struct op_data_entries *hard_out =
439 &test_vector.entries[DATA_HARD_OUTPUT];
440 struct op_data_entries *soft_out =
441 &test_vector.entries[DATA_SOFT_OUTPUT];
442 struct op_data_entries *harq_in =
443 &test_vector.entries[DATA_HARQ_INPUT];
444 struct op_data_entries *harq_out =
445 &test_vector.entries[DATA_HARQ_OUTPUT];
447 /* allocate ops mempool */
448 ops_pool_size = optimal_mempool_size(RTE_MAX(
449 /* Ops used plus 1 reference op */
450 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
451 /* Minimal cache size plus 1 reference op */
452 (unsigned int)(1.5 * rte_lcore_count() *
453 OPS_CACHE_SIZE + 1)),
456 if (org_op_type == RTE_BBDEV_OP_NONE)
457 op_type = RTE_BBDEV_OP_TURBO_ENC;
459 op_type_str = rte_bbdev_op_type_str(op_type);
460 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
462 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
464 mp = rte_bbdev_op_pool_create(pool_name, op_type,
465 ops_pool_size, OPS_CACHE_SIZE, socket_id);
466 TEST_ASSERT_NOT_NULL(mp,
467 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
471 ad->ops_mempool = mp;
473 /* Do not create inputs and outputs mbufs for BaseBand Null Device */
474 if (org_op_type == RTE_BBDEV_OP_NONE)
478 if (in->nb_segments > 0) {
479 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
481 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
482 mbuf_pool_size, "in");
483 TEST_ASSERT_NOT_NULL(mp,
484 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
488 ad->in_mbuf_pool = mp;
492 if (hard_out->nb_segments > 0) {
493 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
494 hard_out->nb_segments);
495 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
498 TEST_ASSERT_NOT_NULL(mp,
499 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
503 ad->hard_out_mbuf_pool = mp;
507 if (soft_out->nb_segments > 0) {
508 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
509 soft_out->nb_segments);
510 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
513 TEST_ASSERT_NOT_NULL(mp,
514 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
518 ad->soft_out_mbuf_pool = mp;
522 if (harq_in->nb_segments > 0) {
523 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
524 harq_in->nb_segments);
525 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
528 TEST_ASSERT_NOT_NULL(mp,
529 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
533 ad->harq_in_mbuf_pool = mp;
537 if (harq_out->nb_segments > 0) {
538 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
539 harq_out->nb_segments);
540 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
543 TEST_ASSERT_NOT_NULL(mp,
544 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
548 ad->harq_out_mbuf_pool = mp;
555 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
556 struct test_bbdev_vector *vector)
559 unsigned int queue_id;
560 struct rte_bbdev_queue_conf qconf;
561 struct active_device *ad = &active_devs[nb_active_devs];
562 unsigned int nb_queues;
563 enum rte_bbdev_op_type op_type = vector->op_type;
565 /* Configure fpga lte fec with PF & VF values
566 * if '-i' flag is set and using fpga device
568 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
569 if ((get_init_device() == true) &&
570 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
571 struct fpga_lte_fec_conf conf;
574 printf("Configure FPGA LTE FEC Driver %s with default values\n",
575 info->drv.driver_name);
577 /* clear default configuration before initialization */
578 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
581 * true if PF is used for data plane
584 conf.pf_mode_en = true;
586 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
587 /* Number of UL queues per VF (fpga supports 8 VFs) */
588 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
589 /* Number of DL queues per VF (fpga supports 8 VFs) */
590 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
593 /* UL bandwidth. Needed for schedule algorithm */
594 conf.ul_bandwidth = UL_4G_BANDWIDTH;
596 conf.dl_bandwidth = DL_4G_BANDWIDTH;
598 /* UL & DL load Balance Factor to 64 */
599 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
600 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
602 /**< FLR timeout value */
603 conf.flr_time_out = FLR_4G_TIMEOUT;
605 /* setup FPGA PF with configuration information */
606 ret = fpga_lte_fec_configure(info->dev_name, &conf);
607 TEST_ASSERT_SUCCESS(ret,
608 "Failed to configure 4G FPGA PF for bbdev %s",
612 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC
613 if ((get_init_device() == true) &&
614 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
615 struct fpga_5gnr_fec_conf conf;
618 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
619 info->drv.driver_name);
621 /* clear default configuration before initialization */
622 memset(&conf, 0, sizeof(struct fpga_5gnr_fec_conf));
625 * true if PF is used for data plane
628 conf.pf_mode_en = true;
630 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
631 /* Number of UL queues per VF (fpga supports 8 VFs) */
632 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
633 /* Number of DL queues per VF (fpga supports 8 VFs) */
634 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
637 /* UL bandwidth. Needed for schedule algorithm */
638 conf.ul_bandwidth = UL_5G_BANDWIDTH;
640 conf.dl_bandwidth = DL_5G_BANDWIDTH;
642 /* UL & DL load Balance Factor to 64 */
643 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
644 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
646 /**< FLR timeout value */
647 conf.flr_time_out = FLR_5G_TIMEOUT;
649 /* setup FPGA PF with configuration information */
650 ret = fpga_5gnr_fec_configure(info->dev_name, &conf);
651 TEST_ASSERT_SUCCESS(ret,
652 "Failed to configure 5G FPGA PF for bbdev %s",
656 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
657 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
660 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
662 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
663 dev_id, nb_queues, info->socket_id, ret);
667 /* configure interrupts if needed */
669 ret = rte_bbdev_intr_enable(dev_id);
671 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
677 /* setup device queues */
678 qconf.socket = info->socket_id;
679 qconf.queue_size = info->drv.default_queue_conf.queue_size;
681 qconf.deferred_start = 0;
682 qconf.op_type = op_type;
684 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
685 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
688 "Allocated all queues (id=%u) at prio%u on dev%u\n",
689 queue_id, qconf.priority, dev_id);
691 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
695 printf("All queues on dev %u allocated: %u\n",
699 ad->queue_ids[queue_id] = queue_id;
701 TEST_ASSERT(queue_id != 0,
702 "ERROR Failed to configure any queues on dev %u",
704 ad->nb_queues = queue_id;
706 set_avail_op(ad, op_type);
712 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
713 struct test_bbdev_vector *vector)
717 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
718 active_devs[nb_active_devs].dev_id = dev_id;
720 ret = add_bbdev_dev(dev_id, info, vector);
721 if (ret == TEST_SUCCESS)
727 populate_active_devices(void)
731 uint8_t nb_devs_added = 0;
732 struct rte_bbdev_info info;
734 RTE_BBDEV_FOREACH(dev_id) {
735 rte_bbdev_info_get(dev_id, &info);
737 if (check_dev_cap(&info)) {
739 "Device %d (%s) does not support specified capabilities\n",
740 dev_id, info.dev_name);
744 ret = add_active_device(dev_id, &info, &test_vector);
746 printf("Adding active bbdev %s skipped\n",
753 return nb_devs_added;
757 read_test_vector(void)
761 memset(&test_vector, 0, sizeof(test_vector));
762 printf("Test vector file = %s\n", get_vector_filename());
763 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
764 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
765 get_vector_filename());
771 testsuite_setup(void)
773 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
775 if (populate_active_devices() == 0) {
776 printf("No suitable devices found!\n");
784 interrupt_testsuite_setup(void)
786 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
788 /* Enable interrupts */
791 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
792 if (populate_active_devices() == 0 ||
793 test_vector.op_type == RTE_BBDEV_OP_NONE) {
794 intr_enabled = false;
795 printf("No suitable devices found!\n");
803 testsuite_teardown(void)
807 /* Unconfigure devices */
808 RTE_BBDEV_FOREACH(dev_id)
809 rte_bbdev_close(dev_id);
811 /* Clear active devices structs. */
812 memset(active_devs, 0, sizeof(active_devs));
815 /* Disable interrupts */
816 intr_enabled = false;
824 for (i = 0; i < nb_active_devs; i++) {
825 dev_id = active_devs[i].dev_id;
826 /* reset bbdev stats */
827 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
828 "Failed to reset stats of bbdev %u", dev_id);
829 /* start the device */
830 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
831 "Failed to start bbdev %u", dev_id);
841 struct rte_bbdev_stats stats;
843 for (i = 0; i < nb_active_devs; i++) {
844 dev_id = active_devs[i].dev_id;
845 /* read stats and print */
846 rte_bbdev_stats_get(dev_id, &stats);
847 /* Stop the device */
848 rte_bbdev_stop(dev_id);
853 init_op_data_objs(struct rte_bbdev_op_data *bufs,
854 struct op_data_entries *ref_entries,
855 struct rte_mempool *mbuf_pool, const uint16_t n,
856 enum op_data_type op_type, uint16_t min_alignment)
860 bool large_input = false;
862 for (i = 0; i < n; ++i) {
864 struct op_data_buf *seg = &ref_entries->segments[0];
865 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
866 TEST_ASSERT_NOT_NULL(m_head,
867 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
868 op_type, n * ref_entries->nb_segments,
871 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
873 * Special case when DPDK mbuf cannot handle
874 * the required input size
876 printf("Warning: Larger input size than DPDK mbuf %d\n",
880 bufs[i].data = m_head;
884 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
885 if ((op_type == DATA_INPUT) && large_input) {
886 /* Allocate a fake overused mbuf */
887 data = rte_malloc(NULL, seg->length, 0);
888 memcpy(data, seg->addr, seg->length);
889 m_head->buf_addr = data;
890 m_head->buf_iova = rte_malloc_virt2iova(data);
891 m_head->data_off = 0;
892 m_head->data_len = seg->length;
894 data = rte_pktmbuf_append(m_head, seg->length);
895 TEST_ASSERT_NOT_NULL(data,
896 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
897 seg->length, op_type);
899 TEST_ASSERT(data == RTE_PTR_ALIGN(
900 data, min_alignment),
901 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
902 data, min_alignment);
903 rte_memcpy(data, seg->addr, seg->length);
906 bufs[i].length += seg->length;
908 for (j = 1; j < ref_entries->nb_segments; ++j) {
909 struct rte_mbuf *m_tail =
910 rte_pktmbuf_alloc(mbuf_pool);
911 TEST_ASSERT_NOT_NULL(m_tail,
912 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
914 n * ref_entries->nb_segments,
918 data = rte_pktmbuf_append(m_tail, seg->length);
919 TEST_ASSERT_NOT_NULL(data,
920 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
921 seg->length, op_type);
923 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
925 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
926 data, min_alignment);
927 rte_memcpy(data, seg->addr, seg->length);
928 bufs[i].length += seg->length;
930 ret = rte_pktmbuf_chain(m_head, m_tail);
931 TEST_ASSERT_SUCCESS(ret,
932 "Couldn't chain mbufs from %d data type mbuf pool",
937 /* allocate chained-mbuf for output buffer */
938 for (j = 1; j < ref_entries->nb_segments; ++j) {
939 struct rte_mbuf *m_tail =
940 rte_pktmbuf_alloc(mbuf_pool);
941 TEST_ASSERT_NOT_NULL(m_tail,
942 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
944 n * ref_entries->nb_segments,
947 ret = rte_pktmbuf_chain(m_head, m_tail);
948 TEST_ASSERT_SUCCESS(ret,
949 "Couldn't chain mbufs from %d data type mbuf pool",
959 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
964 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
965 if (*buffers == NULL) {
966 printf("WARNING: Failed to allocate op_data on socket %d\n",
968 /* try to allocate memory on other detected sockets */
969 for (i = 0; i < socket; i++) {
970 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
971 if (*buffers != NULL)
976 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
980 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
981 const uint16_t n, const int8_t max_llr_modulus)
983 uint16_t i, byte_idx;
985 for (i = 0; i < n; ++i) {
986 struct rte_mbuf *m = input_ops[i].data;
988 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
989 input_ops[i].offset);
990 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
992 llr[byte_idx] = round((double)max_llr_modulus *
993 llr[byte_idx] / INT8_MAX);
1001 * We may have to insert filler bits
1002 * when they are required by the HARQ assumption
1005 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1006 const uint16_t n, struct test_op_params *op_params)
1008 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1010 if (input_ops == NULL)
1012 /* No need to add filler if not required by device */
1013 if (!(ldpc_cap_flags &
1014 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1016 /* No need to add filler for loopback operation */
1017 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1020 uint16_t i, j, parity_offset;
1021 for (i = 0; i < n; ++i) {
1022 struct rte_mbuf *m = input_ops[i].data;
1023 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1024 input_ops[i].offset);
1025 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1026 * dec.z_c - dec.n_filler;
1027 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1028 m->data_len = new_hin_size;
1029 input_ops[i].length = new_hin_size;
1030 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1032 llr[j] = llr[j - dec.n_filler];
1033 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1034 for (j = 0; j < dec.n_filler; j++)
1035 llr[parity_offset + j] = llr_max_pre_scaling;
1040 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1041 const uint16_t n, const int8_t llr_size,
1042 const int8_t llr_decimals)
1044 if (input_ops == NULL)
1047 uint16_t i, byte_idx;
1049 int16_t llr_max, llr_min, llr_tmp;
1050 llr_max = (1 << (llr_size - 1)) - 1;
1052 for (i = 0; i < n; ++i) {
1053 struct rte_mbuf *m = input_ops[i].data;
1055 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1056 input_ops[i].offset);
1057 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1060 llr_tmp = llr[byte_idx];
1061 if (llr_decimals == 4)
1063 else if (llr_decimals == 2)
1065 else if (llr_decimals == 0)
1067 llr_tmp = RTE_MIN(llr_max,
1068 RTE_MAX(llr_min, llr_tmp));
1069 llr[byte_idx] = (int8_t) llr_tmp;
1080 fill_queue_buffers(struct test_op_params *op_params,
1081 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1082 struct rte_mempool *soft_out_mp,
1083 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1085 const struct rte_bbdev_op_cap *capabilities,
1086 uint16_t min_alignment, const int socket_id)
1089 enum op_data_type type;
1090 const uint16_t n = op_params->num_to_process;
1092 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1100 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1101 &op_params->q_bufs[socket_id][queue_id].inputs,
1102 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1103 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1104 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1105 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1108 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1109 struct op_data_entries *ref_entries =
1110 &test_vector.entries[type];
1111 if (ref_entries->nb_segments == 0)
1114 ret = allocate_buffers_on_socket(queue_ops[type],
1115 n * sizeof(struct rte_bbdev_op_data),
1117 TEST_ASSERT_SUCCESS(ret,
1118 "Couldn't allocate memory for rte_bbdev_op_data structs");
1120 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1121 mbuf_pools[type], n, type, min_alignment);
1122 TEST_ASSERT_SUCCESS(ret,
1123 "Couldn't init rte_bbdev_op_data structs");
1126 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1127 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1128 capabilities->cap.turbo_dec.max_llr_modulus);
1130 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1131 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1132 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1133 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1134 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1135 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1136 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1137 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1138 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1139 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1140 if (!loopback && !llr_comp)
1141 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1142 ldpc_llr_size, ldpc_llr_decimals);
1143 if (!loopback && !harq_comp)
1144 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1145 ldpc_llr_size, ldpc_llr_decimals);
1147 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1155 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1159 rte_mempool_free(ad->ops_mempool);
1160 rte_mempool_free(ad->in_mbuf_pool);
1161 rte_mempool_free(ad->hard_out_mbuf_pool);
1162 rte_mempool_free(ad->soft_out_mbuf_pool);
1163 rte_mempool_free(ad->harq_in_mbuf_pool);
1164 rte_mempool_free(ad->harq_out_mbuf_pool);
1166 for (i = 0; i < rte_lcore_count(); ++i) {
1167 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1168 rte_free(op_params->q_bufs[j][i].inputs);
1169 rte_free(op_params->q_bufs[j][i].hard_outputs);
1170 rte_free(op_params->q_bufs[j][i].soft_outputs);
1171 rte_free(op_params->q_bufs[j][i].harq_inputs);
1172 rte_free(op_params->q_bufs[j][i].harq_outputs);
1178 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1179 unsigned int start_idx,
1180 struct rte_bbdev_op_data *inputs,
1181 struct rte_bbdev_op_data *hard_outputs,
1182 struct rte_bbdev_op_data *soft_outputs,
1183 struct rte_bbdev_dec_op *ref_op)
1186 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1188 for (i = 0; i < n; ++i) {
1189 if (turbo_dec->code_block_mode == 0) {
1190 ops[i]->turbo_dec.tb_params.ea =
1191 turbo_dec->tb_params.ea;
1192 ops[i]->turbo_dec.tb_params.eb =
1193 turbo_dec->tb_params.eb;
1194 ops[i]->turbo_dec.tb_params.k_pos =
1195 turbo_dec->tb_params.k_pos;
1196 ops[i]->turbo_dec.tb_params.k_neg =
1197 turbo_dec->tb_params.k_neg;
1198 ops[i]->turbo_dec.tb_params.c =
1199 turbo_dec->tb_params.c;
1200 ops[i]->turbo_dec.tb_params.c_neg =
1201 turbo_dec->tb_params.c_neg;
1202 ops[i]->turbo_dec.tb_params.cab =
1203 turbo_dec->tb_params.cab;
1204 ops[i]->turbo_dec.tb_params.r =
1205 turbo_dec->tb_params.r;
1207 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1208 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1211 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1212 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1213 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1214 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1215 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1216 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1217 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1219 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1220 ops[i]->turbo_dec.input = inputs[start_idx + i];
1221 if (soft_outputs != NULL)
1222 ops[i]->turbo_dec.soft_output =
1223 soft_outputs[start_idx + i];
1228 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1229 unsigned int start_idx,
1230 struct rte_bbdev_op_data *inputs,
1231 struct rte_bbdev_op_data *outputs,
1232 struct rte_bbdev_enc_op *ref_op)
1235 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1236 for (i = 0; i < n; ++i) {
1237 if (turbo_enc->code_block_mode == 0) {
1238 ops[i]->turbo_enc.tb_params.ea =
1239 turbo_enc->tb_params.ea;
1240 ops[i]->turbo_enc.tb_params.eb =
1241 turbo_enc->tb_params.eb;
1242 ops[i]->turbo_enc.tb_params.k_pos =
1243 turbo_enc->tb_params.k_pos;
1244 ops[i]->turbo_enc.tb_params.k_neg =
1245 turbo_enc->tb_params.k_neg;
1246 ops[i]->turbo_enc.tb_params.c =
1247 turbo_enc->tb_params.c;
1248 ops[i]->turbo_enc.tb_params.c_neg =
1249 turbo_enc->tb_params.c_neg;
1250 ops[i]->turbo_enc.tb_params.cab =
1251 turbo_enc->tb_params.cab;
1252 ops[i]->turbo_enc.tb_params.ncb_pos =
1253 turbo_enc->tb_params.ncb_pos;
1254 ops[i]->turbo_enc.tb_params.ncb_neg =
1255 turbo_enc->tb_params.ncb_neg;
1256 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1258 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1259 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1260 ops[i]->turbo_enc.cb_params.ncb =
1261 turbo_enc->cb_params.ncb;
1263 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1264 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1265 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1267 ops[i]->turbo_enc.output = outputs[start_idx + i];
1268 ops[i]->turbo_enc.input = inputs[start_idx + i];
1273 /* Returns a random number drawn from a normal distribution
1274 * with mean of 0 and variance of 1
1275 * Marsaglia algorithm
1280 double S, Z, U1, U2, u, v, fac;
1283 U1 = (double)rand() / RAND_MAX;
1284 U2 = (double)rand() / RAND_MAX;
1288 } while (S >= 1 || S == 0);
1289 fac = sqrt(-2. * log(S) / S);
1290 Z = (n % 2) ? u * fac : v * fac;
1294 static inline double
1295 maxstar(double A, double B)
1297 if (fabs(A - B) > 5)
1298 return RTE_MAX(A, B);
1300 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1304 * Generate Qm LLRS for Qm==8
1305 * Modulation, AWGN and LLR estimation from max log development
1308 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1313 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1314 /* 5.1.4 of TS38.211 */
1315 const double symbols_I[256] = {
1316 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1317 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1318 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1319 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1320 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1321 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1322 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1323 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1324 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1325 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1326 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1327 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1328 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1329 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1330 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1331 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1332 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1333 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1334 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1335 -13, -13, -15, -15, -13, -13, -15, -15};
1336 const double symbols_Q[256] = {
1337 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1338 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1339 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1340 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1341 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1342 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1343 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1344 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1345 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1346 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1347 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1348 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1349 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1350 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1351 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1352 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1353 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1354 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1355 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1356 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1357 /* Average constellation point energy */
1359 for (k = 0; k < qm; k++)
1360 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1361 /* 5.1.4 of TS38.211 */
1362 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1363 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1364 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1365 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1367 I += sqrt(N0 / 2) * randn(0);
1368 Q += sqrt(N0 / 2) * randn(1);
1370 * Calculate the log of the probability that each of
1371 * the constellation points was transmitted
1373 for (m = 0; m < qam; m++)
1374 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1375 + pow(Q - symbols_Q[m], 2.0)) / N0;
1376 /* Calculate an LLR for each of the k_64QAM bits in the set */
1377 for (k = 0; k < qm; k++) {
1380 /* For each constellation point */
1381 for (m = 0; m < qam; m++) {
1382 if ((m >> (qm - k - 1)) & 1)
1383 p1 = maxstar(p1, log_syml_prob[m]);
1385 p0 = maxstar(p0, log_syml_prob[m]);
1387 /* Calculate the LLR */
1389 llr_ *= (1 << ldpc_llr_decimals);
1393 if (llr_ < -llr_max)
1395 llrs[qm * i + k] = (int8_t) llr_;
1401 * Generate Qm LLRS for Qm==6
1402 * Modulation, AWGN and LLR estimation from max log development
1405 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1410 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1411 /* 5.1.4 of TS38.211 */
1412 const double symbols_I[64] = {
1413 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1414 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1415 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1416 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1417 -5, -5, -7, -7, -5, -5, -7, -7};
1418 const double symbols_Q[64] = {
1419 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1420 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1421 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1422 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1423 -3, -1, -3, -1, -5, -7, -5, -7};
1424 /* Average constellation point energy */
1426 for (k = 0; k < qm; k++)
1427 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1428 /* 5.1.4 of TS38.211 */
1429 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1430 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1432 I += sqrt(N0 / 2) * randn(0);
1433 Q += sqrt(N0 / 2) * randn(1);
1435 * Calculate the log of the probability that each of
1436 * the constellation points was transmitted
1438 for (m = 0; m < qam; m++)
1439 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1440 + pow(Q - symbols_Q[m], 2.0)) / N0;
1441 /* Calculate an LLR for each of the k_64QAM bits in the set */
1442 for (k = 0; k < qm; k++) {
1445 /* For each constellation point */
1446 for (m = 0; m < qam; m++) {
1447 if ((m >> (qm - k - 1)) & 1)
1448 p1 = maxstar(p1, log_syml_prob[m]);
1450 p0 = maxstar(p0, log_syml_prob[m]);
1452 /* Calculate the LLR */
1454 llr_ *= (1 << ldpc_llr_decimals);
1458 if (llr_ < -llr_max)
1460 llrs[qm * i + k] = (int8_t) llr_;
1465 * Generate Qm LLRS for Qm==4
1466 * Modulation, AWGN and LLR estimation from max log development
1469 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1474 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1475 /* 5.1.4 of TS38.211 */
1476 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1477 -1, -1, -3, -3, -1, -1, -3, -3};
1478 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1479 1, 3, 1, 3, -1, -3, -1, -3};
1480 /* Average constellation point energy */
1482 for (k = 0; k < qm; k++)
1483 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1484 /* 5.1.4 of TS38.211 */
1485 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1486 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1488 I += sqrt(N0 / 2) * randn(0);
1489 Q += sqrt(N0 / 2) * randn(1);
1491 * Calculate the log of the probability that each of
1492 * the constellation points was transmitted
1494 for (m = 0; m < qam; m++)
1495 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1496 + pow(Q - symbols_Q[m], 2.0)) / N0;
1497 /* Calculate an LLR for each of the k_64QAM bits in the set */
1498 for (k = 0; k < qm; k++) {
1501 /* For each constellation point */
1502 for (m = 0; m < qam; m++) {
1503 if ((m >> (qm - k - 1)) & 1)
1504 p1 = maxstar(p1, log_syml_prob[m]);
1506 p0 = maxstar(p0, log_syml_prob[m]);
1508 /* Calculate the LLR */
1510 llr_ *= (1 << ldpc_llr_decimals);
1514 if (llr_ < -llr_max)
1516 llrs[qm * i + k] = (int8_t) llr_;
1521 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1524 double coeff = 2.0 * sqrt(N0);
1526 /* Ignore in vectors rare quasi null LLRs not to be saturated */
1527 if (llrs[j] < 8 && llrs[j] > -8)
1530 /* Note don't change sign here */
1532 b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1534 b = b1 * (1 << ldpc_llr_decimals);
1540 llrs[j] = (int8_t) b;
1543 /* Generate LLR for a given SNR */
1545 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1546 struct rte_bbdev_dec_op *ref_op)
1550 uint32_t i, j, e, range;
1553 e = ref_op->ldpc_dec.cb_params.e;
1554 qm = ref_op->ldpc_dec.q_m;
1555 llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1557 N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1559 for (i = 0; i < n; ++i) {
1561 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1563 for (j = 0; j < range; ++j)
1564 gen_qm8_llr(llrs, j, N0, llr_max);
1565 } else if (qm == 6) {
1566 for (j = 0; j < range; ++j)
1567 gen_qm6_llr(llrs, j, N0, llr_max);
1568 } else if (qm == 4) {
1569 for (j = 0; j < range; ++j)
1570 gen_qm4_llr(llrs, j, N0, llr_max);
1572 for (j = 0; j < e; ++j)
1573 gen_qm2_llr(llrs, j, N0, llr_max);
1579 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1580 unsigned int start_idx,
1581 struct rte_bbdev_op_data *inputs,
1582 struct rte_bbdev_op_data *hard_outputs,
1583 struct rte_bbdev_op_data *soft_outputs,
1584 struct rte_bbdev_op_data *harq_inputs,
1585 struct rte_bbdev_op_data *harq_outputs,
1586 struct rte_bbdev_dec_op *ref_op)
1589 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1591 for (i = 0; i < n; ++i) {
1592 if (ldpc_dec->code_block_mode == 0) {
1593 ops[i]->ldpc_dec.tb_params.ea =
1594 ldpc_dec->tb_params.ea;
1595 ops[i]->ldpc_dec.tb_params.eb =
1596 ldpc_dec->tb_params.eb;
1597 ops[i]->ldpc_dec.tb_params.c =
1598 ldpc_dec->tb_params.c;
1599 ops[i]->ldpc_dec.tb_params.cab =
1600 ldpc_dec->tb_params.cab;
1601 ops[i]->ldpc_dec.tb_params.r =
1602 ldpc_dec->tb_params.r;
1604 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1607 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1608 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1609 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1610 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1611 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1612 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1613 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1614 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1615 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1617 if (hard_outputs != NULL)
1618 ops[i]->ldpc_dec.hard_output =
1619 hard_outputs[start_idx + i];
1621 ops[i]->ldpc_dec.input =
1622 inputs[start_idx + i];
1623 if (soft_outputs != NULL)
1624 ops[i]->ldpc_dec.soft_output =
1625 soft_outputs[start_idx + i];
1626 if (harq_inputs != NULL)
1627 ops[i]->ldpc_dec.harq_combined_input =
1628 harq_inputs[start_idx + i];
1629 if (harq_outputs != NULL)
1630 ops[i]->ldpc_dec.harq_combined_output =
1631 harq_outputs[start_idx + i];
1637 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1638 unsigned int start_idx,
1639 struct rte_bbdev_op_data *inputs,
1640 struct rte_bbdev_op_data *outputs,
1641 struct rte_bbdev_enc_op *ref_op)
1644 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1645 for (i = 0; i < n; ++i) {
1646 if (ldpc_enc->code_block_mode == 0) {
1647 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1648 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1649 ops[i]->ldpc_enc.tb_params.cab =
1650 ldpc_enc->tb_params.cab;
1651 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1652 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1654 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1656 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1657 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1658 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1659 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1660 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1661 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1662 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1663 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1664 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1665 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1670 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1671 unsigned int order_idx, const int expected_status)
1673 int status = op->status;
1674 /* ignore parity mismatch false alarms for long iterations */
1675 if (get_iter_max() >= 10) {
1676 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1677 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1678 printf("WARNING: Ignore Syndrome Check mismatch\n");
1679 status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1681 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1682 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1683 printf("WARNING: Ignore Syndrome Check mismatch\n");
1684 status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1688 TEST_ASSERT(status == expected_status,
1689 "op_status (%d) != expected_status (%d)",
1690 op->status, expected_status);
1692 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1693 "Ordering error, expected %p, got %p",
1694 (void *)(uintptr_t)order_idx, op->opaque_data);
1696 return TEST_SUCCESS;
1700 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1701 unsigned int order_idx, const int expected_status)
1703 TEST_ASSERT(op->status == expected_status,
1704 "op_status (%d) != expected_status (%d)",
1705 op->status, expected_status);
1707 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1708 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1709 "Ordering error, expected %p, got %p",
1710 (void *)(uintptr_t)order_idx, op->opaque_data);
1712 return TEST_SUCCESS;
1716 validate_op_chain(struct rte_bbdev_op_data *op,
1717 struct op_data_entries *orig_op)
1720 struct rte_mbuf *m = op->data;
1721 uint8_t nb_dst_segments = orig_op->nb_segments;
1722 uint32_t total_data_size = 0;
1724 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1725 "Number of segments differ in original (%u) and filled (%u) op",
1726 nb_dst_segments, m->nb_segs);
1728 /* Validate each mbuf segment length */
1729 for (i = 0; i < nb_dst_segments; ++i) {
1730 /* Apply offset to the first mbuf segment */
1731 uint16_t offset = (i == 0) ? op->offset : 0;
1732 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1733 total_data_size += orig_op->segments[i].length;
1735 TEST_ASSERT(orig_op->segments[i].length == data_len,
1736 "Length of segment differ in original (%u) and filled (%u) op",
1737 orig_op->segments[i].length, data_len);
1738 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1739 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1741 "Output buffers (CB=%u) are not equal", i);
1745 /* Validate total mbuf pkt length */
1746 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1747 TEST_ASSERT(total_data_size == pkt_len,
1748 "Length of data differ in original (%u) and filled (%u) op",
1749 total_data_size, pkt_len);
1751 return TEST_SUCCESS;
1755 * Compute K0 for a given configuration for HARQ output length computation
1756 * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1758 static inline uint16_t
1759 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1763 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1766 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1767 else if (rv_index == 2)
1768 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1770 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1772 /* LBRM case - includes a division by N */
1774 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1776 else if (rv_index == 2)
1777 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1780 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1784 /* HARQ output length including the Filler bits */
1785 static inline uint16_t
1786 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1789 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1790 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1791 /* Compute RM out size and number of rows */
1792 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1793 * ops_ld->z_c - ops_ld->n_filler;
1794 uint16_t deRmOutSize = RTE_MIN(
1795 k0 + ops_ld->cb_params.e +
1796 ((k0 > parity_offset) ?
1797 0 : ops_ld->n_filler),
1799 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1801 uint16_t harq_output_len = numRows * ops_ld->z_c;
1802 return harq_output_len;
1806 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1807 struct op_data_entries *orig_op,
1808 struct rte_bbdev_op_ldpc_dec *ops_ld)
1812 struct rte_mbuf *m = op->data;
1813 uint8_t nb_dst_segments = orig_op->nb_segments;
1814 uint32_t total_data_size = 0;
1815 int8_t *harq_orig, *harq_out, abs_harq_origin;
1816 uint32_t byte_error = 0, cum_error = 0, error;
1817 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1818 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1819 uint16_t parity_offset;
1821 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1822 "Number of segments differ in original (%u) and filled (%u) op",
1823 nb_dst_segments, m->nb_segs);
1825 /* Validate each mbuf segment length */
1826 for (i = 0; i < nb_dst_segments; ++i) {
1827 /* Apply offset to the first mbuf segment */
1828 uint16_t offset = (i == 0) ? op->offset : 0;
1829 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1830 total_data_size += orig_op->segments[i].length;
1832 TEST_ASSERT(orig_op->segments[i].length <
1833 (uint32_t)(data_len + 64),
1834 "Length of segment differ in original (%u) and filled (%u) op",
1835 orig_op->segments[i].length, data_len);
1836 harq_orig = (int8_t *) orig_op->segments[i].addr;
1837 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1839 if (!(ldpc_cap_flags &
1840 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1841 ) || (ops_ld->op_flags &
1842 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1843 data_len -= ops_ld->z_c;
1844 parity_offset = data_len;
1846 /* Compute RM out size and number of rows */
1847 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1848 * ops_ld->z_c - ops_ld->n_filler;
1849 uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1851 if (data_len > deRmOutSize)
1852 data_len = deRmOutSize;
1853 if (data_len > orig_op->segments[i].length)
1854 data_len = orig_op->segments[i].length;
1857 * HARQ output can have minor differences
1858 * due to integer representation and related scaling
1860 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1861 if (j == parity_offset) {
1862 /* Special Handling of the filler bits */
1863 for (k = 0; k < ops_ld->n_filler; k++) {
1865 llr_max_pre_scaling) {
1866 printf("HARQ Filler issue %d: %d %d\n",
1874 if (!(ops_ld->op_flags &
1875 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1876 if (ldpc_llr_decimals > 1)
1877 harq_out[jj] = (harq_out[jj] + 1)
1878 >> (ldpc_llr_decimals - 1);
1879 /* Saturated to S7 */
1880 if (harq_orig[j] > llr_max)
1881 harq_orig[j] = llr_max;
1882 if (harq_orig[j] < -llr_max)
1883 harq_orig[j] = -llr_max;
1885 if (harq_orig[j] != harq_out[jj]) {
1886 error = (harq_orig[j] > harq_out[jj]) ?
1887 harq_orig[j] - harq_out[jj] :
1888 harq_out[jj] - harq_orig[j];
1889 abs_harq_origin = harq_orig[j] > 0 ?
1892 /* Residual quantization error */
1893 if ((error > 8 && (abs_harq_origin <
1896 printf("HARQ mismatch %d: exp %d act %d => %d\n",
1898 harq_out[jj], error);
1908 TEST_ASSERT(byte_error <= 1,
1909 "HARQ output mismatch (%d) %d",
1910 byte_error, cum_error);
1912 /* Validate total mbuf pkt length */
1913 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1914 TEST_ASSERT(total_data_size < pkt_len + 64,
1915 "Length of data differ in original (%u) and filled (%u) op",
1916 total_data_size, pkt_len);
1918 return TEST_SUCCESS;
1922 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1923 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1927 struct op_data_entries *hard_data_orig =
1928 &test_vector.entries[DATA_HARD_OUTPUT];
1929 struct op_data_entries *soft_data_orig =
1930 &test_vector.entries[DATA_SOFT_OUTPUT];
1931 struct rte_bbdev_op_turbo_dec *ops_td;
1932 struct rte_bbdev_op_data *hard_output;
1933 struct rte_bbdev_op_data *soft_output;
1934 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
1936 for (i = 0; i < n; ++i) {
1937 ops_td = &ops[i]->turbo_dec;
1938 hard_output = &ops_td->hard_output;
1939 soft_output = &ops_td->soft_output;
1941 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1942 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1943 "Returned iter_count (%d) > expected iter_count (%d)",
1944 ops_td->iter_count, ref_td->iter_count);
1945 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1946 TEST_ASSERT_SUCCESS(ret,
1947 "Checking status and ordering for decoder failed");
1949 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1951 "Hard output buffers (CB=%u) are not equal",
1954 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
1955 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1957 "Soft output buffers (CB=%u) are not equal",
1961 return TEST_SUCCESS;
1964 /* Check Number of code blocks errors */
1966 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
1969 struct op_data_entries *hard_data_orig =
1970 &test_vector.entries[DATA_HARD_OUTPUT];
1971 struct rte_bbdev_op_ldpc_dec *ops_td;
1972 struct rte_bbdev_op_data *hard_output;
1976 for (i = 0; i < n; ++i) {
1977 ops_td = &ops[i]->ldpc_dec;
1978 hard_output = &ops_td->hard_output;
1979 m = hard_output->data;
1980 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
1981 hard_data_orig->segments[0].addr,
1982 hard_data_orig->segments[0].length))
1989 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1990 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1994 struct op_data_entries *hard_data_orig =
1995 &test_vector.entries[DATA_HARD_OUTPUT];
1996 struct op_data_entries *soft_data_orig =
1997 &test_vector.entries[DATA_SOFT_OUTPUT];
1998 struct op_data_entries *harq_data_orig =
1999 &test_vector.entries[DATA_HARQ_OUTPUT];
2000 struct rte_bbdev_op_ldpc_dec *ops_td;
2001 struct rte_bbdev_op_data *hard_output;
2002 struct rte_bbdev_op_data *harq_output;
2003 struct rte_bbdev_op_data *soft_output;
2004 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2006 for (i = 0; i < n; ++i) {
2007 ops_td = &ops[i]->ldpc_dec;
2008 hard_output = &ops_td->hard_output;
2009 harq_output = &ops_td->harq_combined_output;
2010 soft_output = &ops_td->soft_output;
2012 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2013 TEST_ASSERT_SUCCESS(ret,
2014 "Checking status and ordering for decoder failed");
2015 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2016 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2017 "Returned iter_count (%d) > expected iter_count (%d)",
2018 ops_td->iter_count, ref_td->iter_count);
2020 * We can ignore output data when the decoding failed to
2021 * converge or for loop-back cases
2023 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2024 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2026 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2028 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2030 "Hard output buffers (CB=%u) are not equal",
2033 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2034 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2036 "Soft output buffers (CB=%u) are not equal",
2038 if (ref_op->ldpc_dec.op_flags &
2039 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2040 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2041 harq_data_orig, ops_td),
2042 "HARQ output buffers (CB=%u) are not equal",
2045 if (ref_op->ldpc_dec.op_flags &
2046 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2047 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2048 harq_data_orig, ops_td),
2049 "HARQ output buffers (CB=%u) are not equal",
2054 return TEST_SUCCESS;
2059 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2060 struct rte_bbdev_enc_op *ref_op)
2064 struct op_data_entries *hard_data_orig =
2065 &test_vector.entries[DATA_HARD_OUTPUT];
2067 for (i = 0; i < n; ++i) {
2068 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2069 TEST_ASSERT_SUCCESS(ret,
2070 "Checking status and ordering for encoder failed");
2071 TEST_ASSERT_SUCCESS(validate_op_chain(
2072 &ops[i]->turbo_enc.output,
2074 "Output buffers (CB=%u) are not equal",
2078 return TEST_SUCCESS;
2082 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2083 struct rte_bbdev_enc_op *ref_op)
2087 struct op_data_entries *hard_data_orig =
2088 &test_vector.entries[DATA_HARD_OUTPUT];
2090 for (i = 0; i < n; ++i) {
2091 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2092 TEST_ASSERT_SUCCESS(ret,
2093 "Checking status and ordering for encoder failed");
2094 TEST_ASSERT_SUCCESS(validate_op_chain(
2095 &ops[i]->ldpc_enc.output,
2097 "Output buffers (CB=%u) are not equal",
2101 return TEST_SUCCESS;
2105 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2108 struct op_data_entries *entry;
2110 op->turbo_dec = test_vector.turbo_dec;
2111 entry = &test_vector.entries[DATA_INPUT];
2112 for (i = 0; i < entry->nb_segments; ++i)
2113 op->turbo_dec.input.length +=
2114 entry->segments[i].length;
2118 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2121 struct op_data_entries *entry;
2123 op->ldpc_dec = test_vector.ldpc_dec;
2124 entry = &test_vector.entries[DATA_INPUT];
2125 for (i = 0; i < entry->nb_segments; ++i)
2126 op->ldpc_dec.input.length +=
2127 entry->segments[i].length;
2128 if (test_vector.ldpc_dec.op_flags &
2129 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2130 entry = &test_vector.entries[DATA_HARQ_INPUT];
2131 for (i = 0; i < entry->nb_segments; ++i)
2132 op->ldpc_dec.harq_combined_input.length +=
2133 entry->segments[i].length;
2139 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2142 struct op_data_entries *entry;
2144 op->turbo_enc = test_vector.turbo_enc;
2145 entry = &test_vector.entries[DATA_INPUT];
2146 for (i = 0; i < entry->nb_segments; ++i)
2147 op->turbo_enc.input.length +=
2148 entry->segments[i].length;
2152 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2155 struct op_data_entries *entry;
2157 op->ldpc_enc = test_vector.ldpc_enc;
2158 entry = &test_vector.entries[DATA_INPUT];
2159 for (i = 0; i < entry->nb_segments; ++i)
2160 op->ldpc_enc.input.length +=
2161 entry->segments[i].length;
2165 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2168 uint32_t c, r, tb_size = 0;
2170 if (op->turbo_dec.code_block_mode) {
2171 tb_size = op->turbo_dec.tb_params.k_neg;
2173 c = op->turbo_dec.tb_params.c;
2174 r = op->turbo_dec.tb_params.r;
2175 for (i = 0; i < c-r; i++)
2176 tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2177 op->turbo_dec.tb_params.k_neg :
2178 op->turbo_dec.tb_params.k_pos;
2184 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2187 uint32_t c, r, tb_size = 0;
2188 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2190 if (op->ldpc_dec.code_block_mode) {
2191 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2193 c = op->ldpc_dec.tb_params.c;
2194 r = op->ldpc_dec.tb_params.r;
2195 for (i = 0; i < c-r; i++)
2196 tb_size += sys_cols * op->ldpc_dec.z_c
2197 - op->ldpc_dec.n_filler;
2203 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2206 uint32_t c, r, tb_size = 0;
2208 if (op->turbo_enc.code_block_mode) {
2209 tb_size = op->turbo_enc.tb_params.k_neg;
2211 c = op->turbo_enc.tb_params.c;
2212 r = op->turbo_enc.tb_params.r;
2213 for (i = 0; i < c-r; i++)
2214 tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2215 op->turbo_enc.tb_params.k_neg :
2216 op->turbo_enc.tb_params.k_pos;
2222 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2225 uint32_t c, r, tb_size = 0;
2226 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2228 if (op->turbo_enc.code_block_mode) {
2229 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2231 c = op->turbo_enc.tb_params.c;
2232 r = op->turbo_enc.tb_params.r;
2233 for (i = 0; i < c-r; i++)
2234 tb_size += sys_cols * op->ldpc_enc.z_c
2235 - op->ldpc_enc.n_filler;
2242 init_test_op_params(struct test_op_params *op_params,
2243 enum rte_bbdev_op_type op_type, const int expected_status,
2244 const int vector_mask, struct rte_mempool *ops_mp,
2245 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2248 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2249 op_type == RTE_BBDEV_OP_LDPC_DEC)
2250 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2251 &op_params->ref_dec_op, 1);
2253 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2254 &op_params->ref_enc_op, 1);
2256 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2258 op_params->mp = ops_mp;
2259 op_params->burst_sz = burst_sz;
2260 op_params->num_to_process = num_to_process;
2261 op_params->num_lcores = num_lcores;
2262 op_params->vector_mask = vector_mask;
2263 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2264 op_type == RTE_BBDEV_OP_LDPC_DEC)
2265 op_params->ref_dec_op->status = expected_status;
2266 else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2267 || op_type == RTE_BBDEV_OP_LDPC_ENC)
2268 op_params->ref_enc_op->status = expected_status;
2273 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2274 struct test_op_params *op_params)
2276 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2278 struct active_device *ad;
2279 unsigned int burst_sz = get_burst_sz();
2280 enum rte_bbdev_op_type op_type = test_vector.op_type;
2281 const struct rte_bbdev_op_cap *capabilities = NULL;
2283 ad = &active_devs[dev_id];
2285 /* Check if device supports op_type */
2286 if (!is_avail_op(ad, test_vector.op_type))
2287 return TEST_SUCCESS;
2289 struct rte_bbdev_info info;
2290 rte_bbdev_info_get(ad->dev_id, &info);
2291 socket_id = GET_SOCKET(info.socket_id);
2293 f_ret = create_mempools(ad, socket_id, op_type,
2295 if (f_ret != TEST_SUCCESS) {
2296 printf("Couldn't create mempools");
2299 if (op_type == RTE_BBDEV_OP_NONE)
2300 op_type = RTE_BBDEV_OP_TURBO_ENC;
2302 f_ret = init_test_op_params(op_params, test_vector.op_type,
2303 test_vector.expected_status,
2309 if (f_ret != TEST_SUCCESS) {
2310 printf("Couldn't init test op params");
2315 /* Find capabilities */
2316 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2317 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2318 if (cap->type == test_vector.op_type) {
2324 TEST_ASSERT_NOT_NULL(capabilities,
2325 "Couldn't find capabilities");
2327 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2328 create_reference_dec_op(op_params->ref_dec_op);
2329 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2330 create_reference_enc_op(op_params->ref_enc_op);
2331 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2332 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2333 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2334 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2336 for (i = 0; i < ad->nb_queues; ++i) {
2337 f_ret = fill_queue_buffers(op_params,
2339 ad->hard_out_mbuf_pool,
2340 ad->soft_out_mbuf_pool,
2341 ad->harq_in_mbuf_pool,
2342 ad->harq_out_mbuf_pool,
2345 info.drv.min_alignment,
2347 if (f_ret != TEST_SUCCESS) {
2348 printf("Couldn't init queue buffers");
2353 /* Run test case function */
2354 t_ret = test_case_func(ad, op_params);
2356 /* Free active device resources and return */
2357 free_buffers(ad, op_params);
2361 free_buffers(ad, op_params);
2365 /* Run given test function per active device per supported op type
2369 run_test_case(test_case_function *test_case_func)
2374 /* Alloc op_params */
2375 struct test_op_params *op_params = rte_zmalloc(NULL,
2376 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2377 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2378 RTE_ALIGN(sizeof(struct test_op_params),
2379 RTE_CACHE_LINE_SIZE));
2381 /* For each device run test case function */
2382 for (dev = 0; dev < nb_active_devs; ++dev)
2383 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2385 rte_free(op_params);
2391 /* Push back the HARQ output from DDR to host */
2393 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2394 struct rte_bbdev_dec_op **ops,
2398 int save_status, ret;
2399 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2400 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2401 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2402 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2403 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2404 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2405 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2406 for (j = 0; j < n; ++j) {
2407 if ((loopback && mem_out) || hc_out) {
2408 save_status = ops[j]->status;
2409 ops[j]->ldpc_dec.op_flags =
2410 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2411 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2413 ops[j]->ldpc_dec.op_flags +=
2414 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2415 ops[j]->ldpc_dec.harq_combined_input.offset =
2417 ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2418 harq_offset += HARQ_INCR;
2420 ops[j]->ldpc_dec.harq_combined_input.length =
2421 ops[j]->ldpc_dec.harq_combined_output.length;
2422 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2426 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2429 ops[j]->ldpc_dec.op_flags = flags;
2430 ops[j]->status = save_status;
2436 * Push back the HARQ output from HW DDR to Host
2437 * Preload HARQ memory input and adjust HARQ offset
2440 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2441 struct rte_bbdev_dec_op **ops, const uint16_t n,
2446 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2447 struct rte_bbdev_op_data save_hc_in, save_hc_out;
2448 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2449 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2450 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2451 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2452 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2453 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2454 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2455 for (j = 0; j < n; ++j) {
2456 if ((mem_in || hc_in) && preload) {
2457 save_hc_in = ops[j]->ldpc_dec.harq_combined_input;
2458 save_hc_out = ops[j]->ldpc_dec.harq_combined_output;
2459 ops[j]->ldpc_dec.op_flags =
2460 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2461 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2463 ops[j]->ldpc_dec.op_flags +=
2464 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2465 ops[j]->ldpc_dec.harq_combined_output.offset =
2467 ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2468 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2472 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2473 dev_id, queue_id, &ops_deq[j], 1);
2474 ops[j]->ldpc_dec.op_flags = flags;
2475 ops[j]->ldpc_dec.harq_combined_input = save_hc_in;
2476 ops[j]->ldpc_dec.harq_combined_output = save_hc_out;
2478 /* Adjust HARQ offset when we reach external DDR */
2479 if (mem_in || hc_in)
2480 ops[j]->ldpc_dec.harq_combined_input.offset
2482 if (mem_out || hc_out)
2483 ops[j]->ldpc_dec.harq_combined_output.offset
2485 harq_offset += HARQ_INCR;
2490 dequeue_event_callback(uint16_t dev_id,
2491 enum rte_bbdev_event_type event, void *cb_arg,
2496 uint64_t total_time;
2497 uint16_t deq, burst_sz, num_ops;
2498 uint16_t queue_id = *(uint16_t *) ret_param;
2499 struct rte_bbdev_info info;
2501 struct thread_params *tp = cb_arg;
2503 /* Find matching thread params using queue_id */
2504 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2505 if (tp->queue_id == queue_id)
2508 if (i == MAX_QUEUES) {
2509 printf("%s: Queue_id from interrupt details was not found!\n",
2514 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2515 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2517 "Dequeue interrupt handler called for incorrect event!\n");
2521 burst_sz = rte_atomic16_read(&tp->burst_sz);
2522 num_ops = tp->op_params->num_to_process;
2524 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2525 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2527 rte_atomic16_read(&tp->nb_dequeued)],
2529 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2530 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2532 rte_atomic16_read(&tp->nb_dequeued)],
2534 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2535 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2537 rte_atomic16_read(&tp->nb_dequeued)],
2539 else /*RTE_BBDEV_OP_TURBO_ENC*/
2540 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2542 rte_atomic16_read(&tp->nb_dequeued)],
2545 if (deq < burst_sz) {
2547 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2549 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2553 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2554 rte_atomic16_add(&tp->nb_dequeued, deq);
2558 total_time = rte_rdtsc_precise() - tp->start_time;
2560 rte_bbdev_info_get(dev_id, &info);
2564 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2565 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2566 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2567 tp->op_params->vector_mask);
2568 /* get the max of iter_count for all dequeued ops */
2569 for (i = 0; i < num_ops; ++i)
2570 tp->iter_count = RTE_MAX(
2571 tp->dec_ops[i]->turbo_dec.iter_count,
2573 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2574 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2575 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2576 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2577 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2578 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2579 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2580 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2581 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2582 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2583 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2584 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2585 tp->op_params->vector_mask);
2586 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2590 printf("Buffers validation failed\n");
2591 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2594 switch (test_vector.op_type) {
2595 case RTE_BBDEV_OP_TURBO_DEC:
2596 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2598 case RTE_BBDEV_OP_TURBO_ENC:
2599 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2601 case RTE_BBDEV_OP_LDPC_DEC:
2602 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2604 case RTE_BBDEV_OP_LDPC_ENC:
2605 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2607 case RTE_BBDEV_OP_NONE:
2611 printf("Unknown op type: %d\n", test_vector.op_type);
2612 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2616 tp->ops_per_sec += ((double)num_ops) /
2617 ((double)total_time / (double)rte_get_tsc_hz());
2618 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2619 ((double)total_time / (double)rte_get_tsc_hz());
2621 rte_atomic16_add(&tp->nb_dequeued, deq);
2625 throughput_intr_lcore_ldpc_dec(void *arg)
2627 struct thread_params *tp = arg;
2628 unsigned int enqueued;
2629 const uint16_t queue_id = tp->queue_id;
2630 const uint16_t burst_sz = tp->op_params->burst_sz;
2631 const uint16_t num_to_process = tp->op_params->num_to_process;
2632 struct rte_bbdev_dec_op *ops[num_to_process];
2633 struct test_buffers *bufs = NULL;
2634 struct rte_bbdev_info info;
2636 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2637 uint16_t num_to_enq, enq;
2639 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2640 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2641 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2642 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2644 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2645 "BURST_SIZE should be <= %u", MAX_BURST);
2647 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2648 "Failed to enable interrupts for dev: %u, queue_id: %u",
2649 tp->dev_id, queue_id);
2651 rte_bbdev_info_get(tp->dev_id, &info);
2653 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2654 "NUM_OPS cannot exceed %u for this device",
2655 info.drv.queue_size_lim);
2657 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2659 rte_atomic16_clear(&tp->processing_status);
2660 rte_atomic16_clear(&tp->nb_dequeued);
2662 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2665 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2667 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2669 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2670 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2671 bufs->hard_outputs, bufs->soft_outputs,
2672 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2674 /* Set counter to validate the ordering */
2675 for (j = 0; j < num_to_process; ++j)
2676 ops[j]->opaque_data = (void *)(uintptr_t)j;
2678 for (j = 0; j < TEST_REPETITIONS; ++j) {
2679 for (i = 0; i < num_to_process; ++i) {
2682 ops[i]->ldpc_dec.hard_output.data);
2683 if (hc_out || loopback)
2685 ops[i]->ldpc_dec.harq_combined_output.data);
2688 tp->start_time = rte_rdtsc_precise();
2689 for (enqueued = 0; enqueued < num_to_process;) {
2690 num_to_enq = burst_sz;
2692 if (unlikely(num_to_process - enqueued < num_to_enq))
2693 num_to_enq = num_to_process - enqueued;
2697 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2699 queue_id, &ops[enqueued],
2701 } while (unlikely(num_to_enq != enq));
2704 /* Write to thread burst_sz current number of enqueued
2705 * descriptors. It ensures that proper number of
2706 * descriptors will be dequeued in callback
2707 * function - needed for last batch in case where
2708 * the number of operations is not a multiple of
2711 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2713 /* Wait until processing of previous batch is
2716 while (rte_atomic16_read(&tp->nb_dequeued) !=
2720 if (j != TEST_REPETITIONS - 1)
2721 rte_atomic16_clear(&tp->nb_dequeued);
2724 return TEST_SUCCESS;
2728 throughput_intr_lcore_dec(void *arg)
2730 struct thread_params *tp = arg;
2731 unsigned int enqueued;
2732 const uint16_t queue_id = tp->queue_id;
2733 const uint16_t burst_sz = tp->op_params->burst_sz;
2734 const uint16_t num_to_process = tp->op_params->num_to_process;
2735 struct rte_bbdev_dec_op *ops[num_to_process];
2736 struct test_buffers *bufs = NULL;
2737 struct rte_bbdev_info info;
2739 uint16_t num_to_enq, enq;
2741 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2742 "BURST_SIZE should be <= %u", MAX_BURST);
2744 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2745 "Failed to enable interrupts for dev: %u, queue_id: %u",
2746 tp->dev_id, queue_id);
2748 rte_bbdev_info_get(tp->dev_id, &info);
2750 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2751 "NUM_OPS cannot exceed %u for this device",
2752 info.drv.queue_size_lim);
2754 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2756 rte_atomic16_clear(&tp->processing_status);
2757 rte_atomic16_clear(&tp->nb_dequeued);
2759 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2762 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2764 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2766 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2767 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2768 bufs->hard_outputs, bufs->soft_outputs,
2769 tp->op_params->ref_dec_op);
2771 /* Set counter to validate the ordering */
2772 for (j = 0; j < num_to_process; ++j)
2773 ops[j]->opaque_data = (void *)(uintptr_t)j;
2775 for (j = 0; j < TEST_REPETITIONS; ++j) {
2776 for (i = 0; i < num_to_process; ++i)
2777 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2779 tp->start_time = rte_rdtsc_precise();
2780 for (enqueued = 0; enqueued < num_to_process;) {
2781 num_to_enq = burst_sz;
2783 if (unlikely(num_to_process - enqueued < num_to_enq))
2784 num_to_enq = num_to_process - enqueued;
2788 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2789 queue_id, &ops[enqueued],
2791 } while (unlikely(num_to_enq != enq));
2794 /* Write to thread burst_sz current number of enqueued
2795 * descriptors. It ensures that proper number of
2796 * descriptors will be dequeued in callback
2797 * function - needed for last batch in case where
2798 * the number of operations is not a multiple of
2801 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2803 /* Wait until processing of previous batch is
2806 while (rte_atomic16_read(&tp->nb_dequeued) !=
2810 if (j != TEST_REPETITIONS - 1)
2811 rte_atomic16_clear(&tp->nb_dequeued);
2814 return TEST_SUCCESS;
2818 throughput_intr_lcore_enc(void *arg)
2820 struct thread_params *tp = arg;
2821 unsigned int enqueued;
2822 const uint16_t queue_id = tp->queue_id;
2823 const uint16_t burst_sz = tp->op_params->burst_sz;
2824 const uint16_t num_to_process = tp->op_params->num_to_process;
2825 struct rte_bbdev_enc_op *ops[num_to_process];
2826 struct test_buffers *bufs = NULL;
2827 struct rte_bbdev_info info;
2829 uint16_t num_to_enq, enq;
2831 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2832 "BURST_SIZE should be <= %u", MAX_BURST);
2834 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2835 "Failed to enable interrupts for dev: %u, queue_id: %u",
2836 tp->dev_id, queue_id);
2838 rte_bbdev_info_get(tp->dev_id, &info);
2840 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2841 "NUM_OPS cannot exceed %u for this device",
2842 info.drv.queue_size_lim);
2844 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2846 rte_atomic16_clear(&tp->processing_status);
2847 rte_atomic16_clear(&tp->nb_dequeued);
2849 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2852 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2854 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2856 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2857 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2858 bufs->hard_outputs, tp->op_params->ref_enc_op);
2860 /* Set counter to validate the ordering */
2861 for (j = 0; j < num_to_process; ++j)
2862 ops[j]->opaque_data = (void *)(uintptr_t)j;
2864 for (j = 0; j < TEST_REPETITIONS; ++j) {
2865 for (i = 0; i < num_to_process; ++i)
2866 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2868 tp->start_time = rte_rdtsc_precise();
2869 for (enqueued = 0; enqueued < num_to_process;) {
2870 num_to_enq = burst_sz;
2872 if (unlikely(num_to_process - enqueued < num_to_enq))
2873 num_to_enq = num_to_process - enqueued;
2877 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2878 queue_id, &ops[enqueued],
2880 } while (unlikely(enq != num_to_enq));
2883 /* Write to thread burst_sz current number of enqueued
2884 * descriptors. It ensures that proper number of
2885 * descriptors will be dequeued in callback
2886 * function - needed for last batch in case where
2887 * the number of operations is not a multiple of
2890 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2892 /* Wait until processing of previous batch is
2895 while (rte_atomic16_read(&tp->nb_dequeued) !=
2899 if (j != TEST_REPETITIONS - 1)
2900 rte_atomic16_clear(&tp->nb_dequeued);
2903 return TEST_SUCCESS;
2908 throughput_intr_lcore_ldpc_enc(void *arg)
2910 struct thread_params *tp = arg;
2911 unsigned int enqueued;
2912 const uint16_t queue_id = tp->queue_id;
2913 const uint16_t burst_sz = tp->op_params->burst_sz;
2914 const uint16_t num_to_process = tp->op_params->num_to_process;
2915 struct rte_bbdev_enc_op *ops[num_to_process];
2916 struct test_buffers *bufs = NULL;
2917 struct rte_bbdev_info info;
2919 uint16_t num_to_enq, enq;
2921 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2922 "BURST_SIZE should be <= %u", MAX_BURST);
2924 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2925 "Failed to enable interrupts for dev: %u, queue_id: %u",
2926 tp->dev_id, queue_id);
2928 rte_bbdev_info_get(tp->dev_id, &info);
2930 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2931 "NUM_OPS cannot exceed %u for this device",
2932 info.drv.queue_size_lim);
2934 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2936 rte_atomic16_clear(&tp->processing_status);
2937 rte_atomic16_clear(&tp->nb_dequeued);
2939 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2942 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2944 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2946 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2947 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
2948 bufs->inputs, bufs->hard_outputs,
2949 tp->op_params->ref_enc_op);
2951 /* Set counter to validate the ordering */
2952 for (j = 0; j < num_to_process; ++j)
2953 ops[j]->opaque_data = (void *)(uintptr_t)j;
2955 for (j = 0; j < TEST_REPETITIONS; ++j) {
2956 for (i = 0; i < num_to_process; ++i)
2957 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2959 tp->start_time = rte_rdtsc_precise();
2960 for (enqueued = 0; enqueued < num_to_process;) {
2961 num_to_enq = burst_sz;
2963 if (unlikely(num_to_process - enqueued < num_to_enq))
2964 num_to_enq = num_to_process - enqueued;
2968 enq += rte_bbdev_enqueue_ldpc_enc_ops(
2970 queue_id, &ops[enqueued],
2972 } while (unlikely(enq != num_to_enq));
2975 /* Write to thread burst_sz current number of enqueued
2976 * descriptors. It ensures that proper number of
2977 * descriptors will be dequeued in callback
2978 * function - needed for last batch in case where
2979 * the number of operations is not a multiple of
2982 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2984 /* Wait until processing of previous batch is
2987 while (rte_atomic16_read(&tp->nb_dequeued) !=
2991 if (j != TEST_REPETITIONS - 1)
2992 rte_atomic16_clear(&tp->nb_dequeued);
2995 return TEST_SUCCESS;
2999 throughput_pmd_lcore_dec(void *arg)
3001 struct thread_params *tp = arg;
3003 uint64_t total_time = 0, start_time;
3004 const uint16_t queue_id = tp->queue_id;
3005 const uint16_t burst_sz = tp->op_params->burst_sz;
3006 const uint16_t num_ops = tp->op_params->num_to_process;
3007 struct rte_bbdev_dec_op *ops_enq[num_ops];
3008 struct rte_bbdev_dec_op *ops_deq[num_ops];
3009 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3010 struct test_buffers *bufs = NULL;
3012 struct rte_bbdev_info info;
3013 uint16_t num_to_enq;
3015 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3016 "BURST_SIZE should be <= %u", MAX_BURST);
3018 rte_bbdev_info_get(tp->dev_id, &info);
3020 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3021 "NUM_OPS cannot exceed %u for this device",
3022 info.drv.queue_size_lim);
3024 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3026 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3029 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3030 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3032 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3033 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3034 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3036 /* Set counter to validate the ordering */
3037 for (j = 0; j < num_ops; ++j)
3038 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3040 for (i = 0; i < TEST_REPETITIONS; ++i) {
3042 for (j = 0; j < num_ops; ++j)
3043 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3045 start_time = rte_rdtsc_precise();
3047 for (enq = 0, deq = 0; enq < num_ops;) {
3048 num_to_enq = burst_sz;
3050 if (unlikely(num_ops - enq < num_to_enq))
3051 num_to_enq = num_ops - enq;
3053 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3054 queue_id, &ops_enq[enq], num_to_enq);
3056 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3057 queue_id, &ops_deq[deq], enq - deq);
3060 /* dequeue the remaining */
3062 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3063 queue_id, &ops_deq[deq], enq - deq);
3066 total_time += rte_rdtsc_precise() - start_time;
3070 /* get the max of iter_count for all dequeued ops */
3071 for (i = 0; i < num_ops; ++i) {
3072 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3076 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3077 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3078 tp->op_params->vector_mask);
3079 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3082 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3084 double tb_len_bits = calc_dec_TB_size(ref_op);
3086 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3087 ((double)total_time / (double)rte_get_tsc_hz());
3088 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3089 1000000.0) / ((double)total_time /
3090 (double)rte_get_tsc_hz());
3092 return TEST_SUCCESS;
3096 bler_pmd_lcore_ldpc_dec(void *arg)
3098 struct thread_params *tp = arg;
3100 uint64_t total_time = 0, start_time;
3101 const uint16_t queue_id = tp->queue_id;
3102 const uint16_t burst_sz = tp->op_params->burst_sz;
3103 const uint16_t num_ops = tp->op_params->num_to_process;
3104 struct rte_bbdev_dec_op *ops_enq[num_ops];
3105 struct rte_bbdev_dec_op *ops_deq[num_ops];
3106 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3107 struct test_buffers *bufs = NULL;
3109 float parity_bler = 0;
3110 struct rte_bbdev_info info;
3111 uint16_t num_to_enq;
3112 bool extDdr = check_bit(ldpc_cap_flags,
3113 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3114 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3115 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3116 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3117 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3119 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3120 "BURST_SIZE should be <= %u", MAX_BURST);
3122 rte_bbdev_info_get(tp->dev_id, &info);
3124 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3125 "NUM_OPS cannot exceed %u for this device",
3126 info.drv.queue_size_lim);
3128 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3130 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3133 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3134 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3136 /* For BLER tests we need to enable early termination */
3137 if (!check_bit(ref_op->ldpc_dec.op_flags,
3138 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3139 ref_op->ldpc_dec.op_flags +=
3140 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3141 ref_op->ldpc_dec.iter_max = get_iter_max();
3142 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3144 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3145 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3146 bufs->hard_outputs, bufs->soft_outputs,
3147 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3148 generate_llr_input(num_ops, bufs->inputs, ref_op);
3150 /* Set counter to validate the ordering */
3151 for (j = 0; j < num_ops; ++j)
3152 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3154 for (i = 0; i < 1; ++i) { /* Could add more iterations */
3155 for (j = 0; j < num_ops; ++j) {
3158 ops_enq[j]->ldpc_dec.hard_output.data);
3159 if (hc_out || loopback)
3161 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3164 bool preload = i == (TEST_REPETITIONS - 1);
3165 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3168 start_time = rte_rdtsc_precise();
3170 for (enq = 0, deq = 0; enq < num_ops;) {
3171 num_to_enq = burst_sz;
3173 if (unlikely(num_ops - enq < num_to_enq))
3174 num_to_enq = num_ops - enq;
3176 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3177 queue_id, &ops_enq[enq], num_to_enq);
3179 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3180 queue_id, &ops_deq[deq], enq - deq);
3183 /* dequeue the remaining */
3185 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3186 queue_id, &ops_deq[deq], enq - deq);
3189 total_time += rte_rdtsc_precise() - start_time;
3193 tp->iter_average = 0;
3194 /* get the max of iter_count for all dequeued ops */
3195 for (i = 0; i < num_ops; ++i) {
3196 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3198 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3199 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3203 parity_bler /= num_ops; /* This one is based on SYND */
3204 tp->iter_average /= num_ops;
3205 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3207 if (test_vector.op_type != RTE_BBDEV_OP_NONE
3211 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3212 tp->op_params->vector_mask);
3213 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3216 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3218 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3219 tp->ops_per_sec = ((double)num_ops * 1) /
3220 ((double)total_time / (double)rte_get_tsc_hz());
3221 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3222 1000000.0) / ((double)total_time /
3223 (double)rte_get_tsc_hz());
3225 return TEST_SUCCESS;
3229 throughput_pmd_lcore_ldpc_dec(void *arg)
3231 struct thread_params *tp = arg;
3233 uint64_t total_time = 0, start_time;
3234 const uint16_t queue_id = tp->queue_id;
3235 const uint16_t burst_sz = tp->op_params->burst_sz;
3236 const uint16_t num_ops = tp->op_params->num_to_process;
3237 struct rte_bbdev_dec_op *ops_enq[num_ops];
3238 struct rte_bbdev_dec_op *ops_deq[num_ops];
3239 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3240 struct test_buffers *bufs = NULL;
3242 struct rte_bbdev_info info;
3243 uint16_t num_to_enq;
3244 bool extDdr = check_bit(ldpc_cap_flags,
3245 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3246 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3247 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3248 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3249 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3251 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3252 "BURST_SIZE should be <= %u", MAX_BURST);
3254 rte_bbdev_info_get(tp->dev_id, &info);
3256 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3257 "NUM_OPS cannot exceed %u for this device",
3258 info.drv.queue_size_lim);
3260 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3262 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3265 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3266 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3268 /* For throughput tests we need to disable early termination */
3269 if (check_bit(ref_op->ldpc_dec.op_flags,
3270 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3271 ref_op->ldpc_dec.op_flags -=
3272 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3273 ref_op->ldpc_dec.iter_max = get_iter_max();
3274 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3276 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3277 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3278 bufs->hard_outputs, bufs->soft_outputs,
3279 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3281 /* Set counter to validate the ordering */
3282 for (j = 0; j < num_ops; ++j)
3283 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3285 for (i = 0; i < TEST_REPETITIONS; ++i) {
3286 for (j = 0; j < num_ops; ++j) {
3289 ops_enq[j]->ldpc_dec.hard_output.data);
3290 if (hc_out || loopback)
3292 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3295 bool preload = i == (TEST_REPETITIONS - 1);
3296 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3299 start_time = rte_rdtsc_precise();
3301 for (enq = 0, deq = 0; enq < num_ops;) {
3302 num_to_enq = burst_sz;
3304 if (unlikely(num_ops - enq < num_to_enq))
3305 num_to_enq = num_ops - enq;
3307 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3308 queue_id, &ops_enq[enq], num_to_enq);
3310 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3311 queue_id, &ops_deq[deq], enq - deq);
3314 /* dequeue the remaining */
3316 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3317 queue_id, &ops_deq[deq], enq - deq);
3320 total_time += rte_rdtsc_precise() - start_time;
3324 /* get the max of iter_count for all dequeued ops */
3325 for (i = 0; i < num_ops; ++i) {
3326 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3330 /* Read loopback is not thread safe */
3331 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3334 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3335 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3336 tp->op_params->vector_mask);
3337 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3340 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3342 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3344 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3345 ((double)total_time / (double)rte_get_tsc_hz());
3346 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3347 1000000.0) / ((double)total_time /
3348 (double)rte_get_tsc_hz());
3350 return TEST_SUCCESS;
3354 throughput_pmd_lcore_enc(void *arg)
3356 struct thread_params *tp = arg;
3358 uint64_t total_time = 0, start_time;
3359 const uint16_t queue_id = tp->queue_id;
3360 const uint16_t burst_sz = tp->op_params->burst_sz;
3361 const uint16_t num_ops = tp->op_params->num_to_process;
3362 struct rte_bbdev_enc_op *ops_enq[num_ops];
3363 struct rte_bbdev_enc_op *ops_deq[num_ops];
3364 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3365 struct test_buffers *bufs = NULL;
3367 struct rte_bbdev_info info;
3368 uint16_t num_to_enq;
3370 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3371 "BURST_SIZE should be <= %u", MAX_BURST);
3373 rte_bbdev_info_get(tp->dev_id, &info);
3375 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3376 "NUM_OPS cannot exceed %u for this device",
3377 info.drv.queue_size_lim);
3379 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3381 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3384 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3386 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3388 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3389 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3390 bufs->hard_outputs, ref_op);
3392 /* Set counter to validate the ordering */
3393 for (j = 0; j < num_ops; ++j)
3394 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3396 for (i = 0; i < TEST_REPETITIONS; ++i) {
3398 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3399 for (j = 0; j < num_ops; ++j)
3400 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3402 start_time = rte_rdtsc_precise();
3404 for (enq = 0, deq = 0; enq < num_ops;) {
3405 num_to_enq = burst_sz;
3407 if (unlikely(num_ops - enq < num_to_enq))
3408 num_to_enq = num_ops - enq;
3410 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3411 queue_id, &ops_enq[enq], num_to_enq);
3413 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3414 queue_id, &ops_deq[deq], enq - deq);
3417 /* dequeue the remaining */
3419 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3420 queue_id, &ops_deq[deq], enq - deq);
3423 total_time += rte_rdtsc_precise() - start_time;
3426 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3427 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3428 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3431 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3433 double tb_len_bits = calc_enc_TB_size(ref_op);
3435 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3436 ((double)total_time / (double)rte_get_tsc_hz());
3437 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3438 / 1000000.0) / ((double)total_time /
3439 (double)rte_get_tsc_hz());
3441 return TEST_SUCCESS;
3445 throughput_pmd_lcore_ldpc_enc(void *arg)
3447 struct thread_params *tp = arg;
3449 uint64_t total_time = 0, start_time;
3450 const uint16_t queue_id = tp->queue_id;
3451 const uint16_t burst_sz = tp->op_params->burst_sz;
3452 const uint16_t num_ops = tp->op_params->num_to_process;
3453 struct rte_bbdev_enc_op *ops_enq[num_ops];
3454 struct rte_bbdev_enc_op *ops_deq[num_ops];
3455 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3456 struct test_buffers *bufs = NULL;
3458 struct rte_bbdev_info info;
3459 uint16_t num_to_enq;
3461 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3462 "BURST_SIZE should be <= %u", MAX_BURST);
3464 rte_bbdev_info_get(tp->dev_id, &info);
3466 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3467 "NUM_OPS cannot exceed %u for this device",
3468 info.drv.queue_size_lim);
3470 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3472 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3475 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3477 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3479 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3480 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3481 bufs->hard_outputs, ref_op);
3483 /* Set counter to validate the ordering */
3484 for (j = 0; j < num_ops; ++j)
3485 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3487 for (i = 0; i < TEST_REPETITIONS; ++i) {
3489 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3490 for (j = 0; j < num_ops; ++j)
3491 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3493 start_time = rte_rdtsc_precise();
3495 for (enq = 0, deq = 0; enq < num_ops;) {
3496 num_to_enq = burst_sz;
3498 if (unlikely(num_ops - enq < num_to_enq))
3499 num_to_enq = num_ops - enq;
3501 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3502 queue_id, &ops_enq[enq], num_to_enq);
3504 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3505 queue_id, &ops_deq[deq], enq - deq);
3508 /* dequeue the remaining */
3510 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3511 queue_id, &ops_deq[deq], enq - deq);
3514 total_time += rte_rdtsc_precise() - start_time;
3517 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3518 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3519 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3522 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3524 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3526 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3527 ((double)total_time / (double)rte_get_tsc_hz());
3528 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3529 / 1000000.0) / ((double)total_time /
3530 (double)rte_get_tsc_hz());
3532 return TEST_SUCCESS;
3536 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3538 unsigned int iter = 0;
3539 double total_mops = 0, total_mbps = 0;
3541 for (iter = 0; iter < used_cores; iter++) {
3543 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3544 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3545 t_params[iter].mbps);
3546 total_mops += t_params[iter].ops_per_sec;
3547 total_mbps += t_params[iter].mbps;
3550 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3551 used_cores, total_mops, total_mbps);
3554 /* Aggregate the performance results over the number of cores used */
3556 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3558 unsigned int core_idx = 0;
3559 double total_mops = 0, total_mbps = 0;
3560 uint8_t iter_count = 0;
3562 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3564 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3565 t_params[core_idx].lcore_id,
3566 t_params[core_idx].ops_per_sec,
3567 t_params[core_idx].mbps,
3568 t_params[core_idx].iter_count);
3569 total_mops += t_params[core_idx].ops_per_sec;
3570 total_mbps += t_params[core_idx].mbps;
3571 iter_count = RTE_MAX(iter_count,
3572 t_params[core_idx].iter_count);
3575 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3576 used_cores, total_mops, total_mbps, iter_count);
3579 /* Aggregate the performance results over the number of cores used */
3581 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3583 unsigned int core_idx = 0;
3584 double total_mbps = 0, total_bler = 0, total_iter = 0;
3585 double snr = get_snr();
3587 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3588 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3589 t_params[core_idx].lcore_id,
3590 t_params[core_idx].bler * 100,
3591 t_params[core_idx].iter_average,
3592 t_params[core_idx].mbps,
3593 get_vector_filename());
3594 total_mbps += t_params[core_idx].mbps;
3595 total_bler += t_params[core_idx].bler;
3596 total_iter += t_params[core_idx].iter_average;
3598 total_bler /= used_cores;
3599 total_iter /= used_cores;
3601 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3602 snr, total_bler * 100, total_iter, get_iter_max(),
3603 total_mbps, get_vector_filename());
3607 * Test function that determines BLER wireless performance
3610 bler_test(struct active_device *ad,
3611 struct test_op_params *op_params)
3614 unsigned int lcore_id, used_cores = 0;
3615 struct thread_params *t_params;
3616 struct rte_bbdev_info info;
3617 lcore_function_t *bler_function;
3618 uint16_t num_lcores;
3619 const char *op_type_str;
3621 rte_bbdev_info_get(ad->dev_id, &info);
3623 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3624 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3625 test_vector.op_type);
3627 printf("+ ------------------------------------------------------- +\n");
3628 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3629 info.dev_name, ad->nb_queues, op_params->burst_sz,
3630 op_params->num_to_process, op_params->num_lcores,
3632 intr_enabled ? "Interrupt mode" : "PMD mode",
3633 (double)rte_get_tsc_hz() / 1000000000.0);
3635 /* Set number of lcores */
3636 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3638 : op_params->num_lcores;
3640 /* Allocate memory for thread parameters structure */
3641 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3642 RTE_CACHE_LINE_SIZE);
3643 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3644 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3645 RTE_CACHE_LINE_SIZE));
3647 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3648 bler_function = bler_pmd_lcore_ldpc_dec;
3650 return TEST_SKIPPED;
3652 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3654 /* Master core is set at first entry */
3655 t_params[0].dev_id = ad->dev_id;
3656 t_params[0].lcore_id = rte_lcore_id();
3657 t_params[0].op_params = op_params;
3658 t_params[0].queue_id = ad->queue_ids[used_cores++];
3659 t_params[0].iter_count = 0;
3661 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3662 if (used_cores >= num_lcores)
3665 t_params[used_cores].dev_id = ad->dev_id;
3666 t_params[used_cores].lcore_id = lcore_id;
3667 t_params[used_cores].op_params = op_params;
3668 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3669 t_params[used_cores].iter_count = 0;
3671 rte_eal_remote_launch(bler_function,
3672 &t_params[used_cores++], lcore_id);
3675 rte_atomic16_set(&op_params->sync, SYNC_START);
3676 ret = bler_function(&t_params[0]);
3678 /* Master core is always used */
3679 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3680 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3682 print_dec_bler(t_params, num_lcores);
3684 /* Return if test failed */
3690 /* Function to print something here*/
3696 * Test function that determines how long an enqueue + dequeue of a burst
3697 * takes on available lcores.
3700 throughput_test(struct active_device *ad,
3701 struct test_op_params *op_params)
3704 unsigned int lcore_id, used_cores = 0;
3705 struct thread_params *t_params, *tp;
3706 struct rte_bbdev_info info;
3707 lcore_function_t *throughput_function;
3708 uint16_t num_lcores;
3709 const char *op_type_str;
3711 rte_bbdev_info_get(ad->dev_id, &info);
3713 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3714 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3715 test_vector.op_type);
3717 printf("+ ------------------------------------------------------- +\n");
3718 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3719 info.dev_name, ad->nb_queues, op_params->burst_sz,
3720 op_params->num_to_process, op_params->num_lcores,
3722 intr_enabled ? "Interrupt mode" : "PMD mode",
3723 (double)rte_get_tsc_hz() / 1000000000.0);
3725 /* Set number of lcores */
3726 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3728 : op_params->num_lcores;
3730 /* Allocate memory for thread parameters structure */
3731 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3732 RTE_CACHE_LINE_SIZE);
3733 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3734 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3735 RTE_CACHE_LINE_SIZE));
3738 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3739 throughput_function = throughput_intr_lcore_dec;
3740 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3741 throughput_function = throughput_intr_lcore_ldpc_dec;
3742 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3743 throughput_function = throughput_intr_lcore_enc;
3744 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3745 throughput_function = throughput_intr_lcore_ldpc_enc;
3747 throughput_function = throughput_intr_lcore_enc;
3749 /* Dequeue interrupt callback registration */
3750 ret = rte_bbdev_callback_register(ad->dev_id,
3751 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3758 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3759 throughput_function = throughput_pmd_lcore_dec;
3760 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3761 throughput_function = throughput_pmd_lcore_ldpc_dec;
3762 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3763 throughput_function = throughput_pmd_lcore_enc;
3764 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3765 throughput_function = throughput_pmd_lcore_ldpc_enc;
3767 throughput_function = throughput_pmd_lcore_enc;
3770 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3772 /* Master core is set at first entry */
3773 t_params[0].dev_id = ad->dev_id;
3774 t_params[0].lcore_id = rte_lcore_id();
3775 t_params[0].op_params = op_params;
3776 t_params[0].queue_id = ad->queue_ids[used_cores++];
3777 t_params[0].iter_count = 0;
3779 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3780 if (used_cores >= num_lcores)
3783 t_params[used_cores].dev_id = ad->dev_id;
3784 t_params[used_cores].lcore_id = lcore_id;
3785 t_params[used_cores].op_params = op_params;
3786 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3787 t_params[used_cores].iter_count = 0;
3789 rte_eal_remote_launch(throughput_function,
3790 &t_params[used_cores++], lcore_id);
3793 rte_atomic16_set(&op_params->sync, SYNC_START);
3794 ret = throughput_function(&t_params[0]);
3796 /* Master core is always used */
3797 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3798 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3800 /* Return if test failed */
3806 /* Print throughput if interrupts are disabled and test passed */
3807 if (!intr_enabled) {
3808 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3809 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3810 print_dec_throughput(t_params, num_lcores);
3812 print_enc_throughput(t_params, num_lcores);
3817 /* In interrupt TC we need to wait for the interrupt callback to deqeue
3818 * all pending operations. Skip waiting for queues which reported an
3819 * error using processing_status variable.
3820 * Wait for master lcore operations.
3823 while ((rte_atomic16_read(&tp->nb_dequeued) <
3824 op_params->num_to_process) &&
3825 (rte_atomic16_read(&tp->processing_status) !=
3829 tp->ops_per_sec /= TEST_REPETITIONS;
3830 tp->mbps /= TEST_REPETITIONS;
3831 ret |= (int)rte_atomic16_read(&tp->processing_status);
3833 /* Wait for slave lcores operations */
3834 for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3835 tp = &t_params[used_cores];
3837 while ((rte_atomic16_read(&tp->nb_dequeued) <
3838 op_params->num_to_process) &&
3839 (rte_atomic16_read(&tp->processing_status) !=
3843 tp->ops_per_sec /= TEST_REPETITIONS;
3844 tp->mbps /= TEST_REPETITIONS;
3845 ret |= (int)rte_atomic16_read(&tp->processing_status);
3848 /* Print throughput if test passed */
3850 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3851 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3852 print_dec_throughput(t_params, num_lcores);
3853 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3854 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3855 print_enc_throughput(t_params, num_lcores);
3863 latency_test_dec(struct rte_mempool *mempool,
3864 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3865 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3866 const uint16_t num_to_process, uint16_t burst_sz,
3867 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3869 int ret = TEST_SUCCESS;
3870 uint16_t i, j, dequeued;
3871 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3872 uint64_t start_time = 0, last_time = 0;
3874 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3875 uint16_t enq = 0, deq = 0;
3876 bool first_time = true;
3879 if (unlikely(num_to_process - dequeued < burst_sz))
3880 burst_sz = num_to_process - dequeued;
3882 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3883 TEST_ASSERT_SUCCESS(ret,
3884 "rte_bbdev_dec_op_alloc_bulk() failed");
3885 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3886 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3892 /* Set counter to validate the ordering */
3893 for (j = 0; j < burst_sz; ++j)
3894 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3896 start_time = rte_rdtsc_precise();
3898 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
3900 TEST_ASSERT(enq == burst_sz,
3901 "Error enqueueing burst, expected %u, got %u",
3906 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3907 &ops_deq[deq], burst_sz - deq);
3908 if (likely(first_time && (deq > 0))) {
3909 last_time = rte_rdtsc_precise() - start_time;
3912 } while (unlikely(burst_sz != deq));
3914 *max_time = RTE_MAX(*max_time, last_time);
3915 *min_time = RTE_MIN(*min_time, last_time);
3916 *total_time += last_time;
3918 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3919 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
3921 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3924 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3932 latency_test_ldpc_dec(struct rte_mempool *mempool,
3933 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3934 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3935 const uint16_t num_to_process, uint16_t burst_sz,
3936 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3938 int ret = TEST_SUCCESS;
3939 uint16_t i, j, dequeued;
3940 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3941 uint64_t start_time = 0, last_time = 0;
3942 bool extDdr = ldpc_cap_flags &
3943 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
3945 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3946 uint16_t enq = 0, deq = 0;
3947 bool first_time = true;
3950 if (unlikely(num_to_process - dequeued < burst_sz))
3951 burst_sz = num_to_process - dequeued;
3953 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3954 TEST_ASSERT_SUCCESS(ret,
3955 "rte_bbdev_dec_op_alloc_bulk() failed");
3957 /* For latency tests we need to disable early termination */
3958 if (check_bit(ref_op->ldpc_dec.op_flags,
3959 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3960 ref_op->ldpc_dec.op_flags -=
3961 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3962 ref_op->ldpc_dec.iter_max = get_iter_max();
3963 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3965 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3966 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
3975 preload_harq_ddr(dev_id, queue_id, ops_enq,
3978 /* Set counter to validate the ordering */
3979 for (j = 0; j < burst_sz; ++j)
3980 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3982 start_time = rte_rdtsc_precise();
3984 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
3985 &ops_enq[enq], burst_sz);
3986 TEST_ASSERT(enq == burst_sz,
3987 "Error enqueueing burst, expected %u, got %u",
3992 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
3993 &ops_deq[deq], burst_sz - deq);
3994 if (likely(first_time && (deq > 0))) {
3995 last_time = rte_rdtsc_precise() - start_time;
3998 } while (unlikely(burst_sz != deq));
4000 *max_time = RTE_MAX(*max_time, last_time);
4001 *min_time = RTE_MIN(*min_time, last_time);
4002 *total_time += last_time;
4005 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4007 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4008 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4010 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4013 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4020 latency_test_enc(struct rte_mempool *mempool,
4021 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4022 uint16_t dev_id, uint16_t queue_id,
4023 const uint16_t num_to_process, uint16_t burst_sz,
4024 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4026 int ret = TEST_SUCCESS;
4027 uint16_t i, j, dequeued;
4028 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4029 uint64_t start_time = 0, last_time = 0;
4031 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4032 uint16_t enq = 0, deq = 0;
4033 bool first_time = true;
4036 if (unlikely(num_to_process - dequeued < burst_sz))
4037 burst_sz = num_to_process - dequeued;
4039 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4040 TEST_ASSERT_SUCCESS(ret,
4041 "rte_bbdev_enc_op_alloc_bulk() failed");
4042 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4043 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4048 /* Set counter to validate the ordering */
4049 for (j = 0; j < burst_sz; ++j)
4050 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4052 start_time = rte_rdtsc_precise();
4054 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4056 TEST_ASSERT(enq == burst_sz,
4057 "Error enqueueing burst, expected %u, got %u",
4062 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4063 &ops_deq[deq], burst_sz - deq);
4064 if (likely(first_time && (deq > 0))) {
4065 last_time += rte_rdtsc_precise() - start_time;
4068 } while (unlikely(burst_sz != deq));
4070 *max_time = RTE_MAX(*max_time, last_time);
4071 *min_time = RTE_MIN(*min_time, last_time);
4072 *total_time += last_time;
4074 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4075 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4076 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4079 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4087 latency_test_ldpc_enc(struct rte_mempool *mempool,
4088 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4089 uint16_t dev_id, uint16_t queue_id,
4090 const uint16_t num_to_process, uint16_t burst_sz,
4091 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4093 int ret = TEST_SUCCESS;
4094 uint16_t i, j, dequeued;
4095 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4096 uint64_t start_time = 0, last_time = 0;
4098 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4099 uint16_t enq = 0, deq = 0;
4100 bool first_time = true;
4103 if (unlikely(num_to_process - dequeued < burst_sz))
4104 burst_sz = num_to_process - dequeued;
4106 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4107 TEST_ASSERT_SUCCESS(ret,
4108 "rte_bbdev_enc_op_alloc_bulk() failed");
4109 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4110 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4115 /* Set counter to validate the ordering */
4116 for (j = 0; j < burst_sz; ++j)
4117 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4119 start_time = rte_rdtsc_precise();
4121 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4122 &ops_enq[enq], burst_sz);
4123 TEST_ASSERT(enq == burst_sz,
4124 "Error enqueueing burst, expected %u, got %u",
4129 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4130 &ops_deq[deq], burst_sz - deq);
4131 if (likely(first_time && (deq > 0))) {
4132 last_time += rte_rdtsc_precise() - start_time;
4135 } while (unlikely(burst_sz != deq));
4137 *max_time = RTE_MAX(*max_time, last_time);
4138 *min_time = RTE_MIN(*min_time, last_time);
4139 *total_time += last_time;
4141 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4142 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4143 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4146 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4154 latency_test(struct active_device *ad,
4155 struct test_op_params *op_params)
4158 uint16_t burst_sz = op_params->burst_sz;
4159 const uint16_t num_to_process = op_params->num_to_process;
4160 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4161 const uint16_t queue_id = ad->queue_ids[0];
4162 struct test_buffers *bufs = NULL;
4163 struct rte_bbdev_info info;
4164 uint64_t total_time, min_time, max_time;
4165 const char *op_type_str;
4167 total_time = max_time = 0;
4168 min_time = UINT64_MAX;
4170 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4171 "BURST_SIZE should be <= %u", MAX_BURST);
4173 rte_bbdev_info_get(ad->dev_id, &info);
4174 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4176 op_type_str = rte_bbdev_op_type_str(op_type);
4177 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4179 printf("+ ------------------------------------------------------- +\n");
4180 printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4181 info.dev_name, burst_sz, num_to_process, op_type_str);
4183 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4184 iter = latency_test_dec(op_params->mp, bufs,
4185 op_params->ref_dec_op, op_params->vector_mask,
4186 ad->dev_id, queue_id, num_to_process,
4187 burst_sz, &total_time, &min_time, &max_time);
4188 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4189 iter = latency_test_enc(op_params->mp, bufs,
4190 op_params->ref_enc_op, ad->dev_id, queue_id,
4191 num_to_process, burst_sz, &total_time,
4192 &min_time, &max_time);
4193 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4194 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4195 op_params->ref_enc_op, ad->dev_id, queue_id,
4196 num_to_process, burst_sz, &total_time,
4197 &min_time, &max_time);
4198 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4199 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4200 op_params->ref_dec_op, op_params->vector_mask,
4201 ad->dev_id, queue_id, num_to_process,
4202 burst_sz, &total_time, &min_time, &max_time);
4204 iter = latency_test_enc(op_params->mp, bufs,
4205 op_params->ref_enc_op,
4206 ad->dev_id, queue_id,
4207 num_to_process, burst_sz, &total_time,
4208 &min_time, &max_time);
4213 printf("Operation latency:\n"
4214 "\tavg: %lg cycles, %lg us\n"
4215 "\tmin: %lg cycles, %lg us\n"
4216 "\tmax: %lg cycles, %lg us\n",
4217 (double)total_time / (double)iter,
4218 (double)(total_time * 1000000) / (double)iter /
4219 (double)rte_get_tsc_hz(), (double)min_time,
4220 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4221 (double)max_time, (double)(max_time * 1000000) /
4222 (double)rte_get_tsc_hz());
4224 return TEST_SUCCESS;
4227 #ifdef RTE_BBDEV_OFFLOAD_COST
4229 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4230 struct rte_bbdev_stats *stats)
4232 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4233 struct rte_bbdev_stats *q_stats;
4235 if (queue_id >= dev->data->num_queues)
4238 q_stats = &dev->data->queues[queue_id].queue_stats;
4240 stats->enqueued_count = q_stats->enqueued_count;
4241 stats->dequeued_count = q_stats->dequeued_count;
4242 stats->enqueue_err_count = q_stats->enqueue_err_count;
4243 stats->dequeue_err_count = q_stats->dequeue_err_count;
4244 stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4250 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4251 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4252 uint16_t queue_id, const uint16_t num_to_process,
4253 uint16_t burst_sz, struct test_time_stats *time_st)
4255 int i, dequeued, ret;
4256 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4257 uint64_t enq_start_time, deq_start_time;
4258 uint64_t enq_sw_last_time, deq_last_time;
4259 struct rte_bbdev_stats stats;
4261 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4262 uint16_t enq = 0, deq = 0;
4264 if (unlikely(num_to_process - dequeued < burst_sz))
4265 burst_sz = num_to_process - dequeued;
4267 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4268 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4269 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4275 /* Start time meas for enqueue function offload latency */
4276 enq_start_time = rte_rdtsc_precise();
4278 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4279 &ops_enq[enq], burst_sz - enq);
4280 } while (unlikely(burst_sz != enq));
4282 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4283 TEST_ASSERT_SUCCESS(ret,
4284 "Failed to get stats for queue (%u) of device (%u)",
4287 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4288 stats.acc_offload_cycles;
4289 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4291 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4293 time_st->enq_sw_total_time += enq_sw_last_time;
4295 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4296 stats.acc_offload_cycles);
4297 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4298 stats.acc_offload_cycles);
4299 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4301 /* give time for device to process ops */
4304 /* Start time meas for dequeue function offload latency */
4305 deq_start_time = rte_rdtsc_precise();
4306 /* Dequeue one operation */
4308 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4310 } while (unlikely(deq != 1));
4312 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4313 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4315 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4317 time_st->deq_total_time += deq_last_time;
4319 /* Dequeue remaining operations if needed*/
4320 while (burst_sz != deq)
4321 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4322 &ops_deq[deq], burst_sz - deq);
4324 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4332 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4333 struct test_buffers *bufs,
4334 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4335 uint16_t queue_id, const uint16_t num_to_process,
4336 uint16_t burst_sz, struct test_time_stats *time_st)
4338 int i, dequeued, ret;
4339 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4340 uint64_t enq_start_time, deq_start_time;
4341 uint64_t enq_sw_last_time, deq_last_time;
4342 struct rte_bbdev_stats stats;
4343 bool extDdr = ldpc_cap_flags &
4344 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4346 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4347 uint16_t enq = 0, deq = 0;
4349 if (unlikely(num_to_process - dequeued < burst_sz))
4350 burst_sz = num_to_process - dequeued;
4352 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4353 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4354 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4363 preload_harq_ddr(dev_id, queue_id, ops_enq,
4366 /* Start time meas for enqueue function offload latency */
4367 enq_start_time = rte_rdtsc_precise();
4369 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4370 &ops_enq[enq], burst_sz - enq);
4371 } while (unlikely(burst_sz != enq));
4373 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4374 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4375 TEST_ASSERT_SUCCESS(ret,
4376 "Failed to get stats for queue (%u) of device (%u)",
4379 enq_sw_last_time -= stats.acc_offload_cycles;
4380 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4382 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4384 time_st->enq_sw_total_time += enq_sw_last_time;
4386 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4387 stats.acc_offload_cycles);
4388 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4389 stats.acc_offload_cycles);
4390 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4392 /* give time for device to process ops */
4395 /* Start time meas for dequeue function offload latency */
4396 deq_start_time = rte_rdtsc_precise();
4397 /* Dequeue one operation */
4399 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4401 } while (unlikely(deq != 1));
4403 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4404 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4406 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4408 time_st->deq_total_time += deq_last_time;
4410 /* Dequeue remaining operations if needed*/
4411 while (burst_sz != deq)
4412 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4413 &ops_deq[deq], burst_sz - deq);
4416 /* Read loopback is not thread safe */
4417 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4420 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4428 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4429 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4430 uint16_t queue_id, const uint16_t num_to_process,
4431 uint16_t burst_sz, struct test_time_stats *time_st)
4433 int i, dequeued, ret;
4434 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4435 uint64_t enq_start_time, deq_start_time;
4436 uint64_t enq_sw_last_time, deq_last_time;
4437 struct rte_bbdev_stats stats;
4439 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4440 uint16_t enq = 0, deq = 0;
4442 if (unlikely(num_to_process - dequeued < burst_sz))
4443 burst_sz = num_to_process - dequeued;
4445 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4446 TEST_ASSERT_SUCCESS(ret,
4447 "rte_bbdev_enc_op_alloc_bulk() failed");
4448 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4449 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4454 /* Start time meas for enqueue function offload latency */
4455 enq_start_time = rte_rdtsc_precise();
4457 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4458 &ops_enq[enq], burst_sz - enq);
4459 } while (unlikely(burst_sz != enq));
4461 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4463 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4464 TEST_ASSERT_SUCCESS(ret,
4465 "Failed to get stats for queue (%u) of device (%u)",
4467 enq_sw_last_time -= stats.acc_offload_cycles;
4468 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4470 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4472 time_st->enq_sw_total_time += enq_sw_last_time;
4474 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4475 stats.acc_offload_cycles);
4476 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4477 stats.acc_offload_cycles);
4478 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4480 /* give time for device to process ops */
4483 /* Start time meas for dequeue function offload latency */
4484 deq_start_time = rte_rdtsc_precise();
4485 /* Dequeue one operation */
4487 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4489 } while (unlikely(deq != 1));
4491 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4492 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4494 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4496 time_st->deq_total_time += deq_last_time;
4498 while (burst_sz != deq)
4499 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4500 &ops_deq[deq], burst_sz - deq);
4502 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4510 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4511 struct test_buffers *bufs,
4512 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4513 uint16_t queue_id, const uint16_t num_to_process,
4514 uint16_t burst_sz, struct test_time_stats *time_st)
4516 int i, dequeued, ret;
4517 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4518 uint64_t enq_start_time, deq_start_time;
4519 uint64_t enq_sw_last_time, deq_last_time;
4520 struct rte_bbdev_stats stats;
4522 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4523 uint16_t enq = 0, deq = 0;
4525 if (unlikely(num_to_process - dequeued < burst_sz))
4526 burst_sz = num_to_process - dequeued;
4528 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4529 TEST_ASSERT_SUCCESS(ret,
4530 "rte_bbdev_enc_op_alloc_bulk() failed");
4531 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4532 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4537 /* Start time meas for enqueue function offload latency */
4538 enq_start_time = rte_rdtsc_precise();
4540 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4541 &ops_enq[enq], burst_sz - enq);
4542 } while (unlikely(burst_sz != enq));
4544 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4545 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4546 TEST_ASSERT_SUCCESS(ret,
4547 "Failed to get stats for queue (%u) of device (%u)",
4550 enq_sw_last_time -= stats.acc_offload_cycles;
4551 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4553 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4555 time_st->enq_sw_total_time += enq_sw_last_time;
4557 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4558 stats.acc_offload_cycles);
4559 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4560 stats.acc_offload_cycles);
4561 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4563 /* give time for device to process ops */
4566 /* Start time meas for dequeue function offload latency */
4567 deq_start_time = rte_rdtsc_precise();
4568 /* Dequeue one operation */
4570 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4572 } while (unlikely(deq != 1));
4574 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4575 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4577 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4579 time_st->deq_total_time += deq_last_time;
4581 while (burst_sz != deq)
4582 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4583 &ops_deq[deq], burst_sz - deq);
4585 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4594 offload_cost_test(struct active_device *ad,
4595 struct test_op_params *op_params)
4597 #ifndef RTE_BBDEV_OFFLOAD_COST
4599 RTE_SET_USED(op_params);
4600 printf("Offload latency test is disabled.\n");
4601 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4602 return TEST_SKIPPED;
4605 uint16_t burst_sz = op_params->burst_sz;
4606 const uint16_t num_to_process = op_params->num_to_process;
4607 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4608 const uint16_t queue_id = ad->queue_ids[0];
4609 struct test_buffers *bufs = NULL;
4610 struct rte_bbdev_info info;
4611 const char *op_type_str;
4612 struct test_time_stats time_st;
4614 memset(&time_st, 0, sizeof(struct test_time_stats));
4615 time_st.enq_sw_min_time = UINT64_MAX;
4616 time_st.enq_acc_min_time = UINT64_MAX;
4617 time_st.deq_min_time = UINT64_MAX;
4619 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4620 "BURST_SIZE should be <= %u", MAX_BURST);
4622 rte_bbdev_info_get(ad->dev_id, &info);
4623 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4625 op_type_str = rte_bbdev_op_type_str(op_type);
4626 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4628 printf("+ ------------------------------------------------------- +\n");
4629 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4630 info.dev_name, burst_sz, num_to_process, op_type_str);
4632 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4633 iter = offload_latency_test_dec(op_params->mp, bufs,
4634 op_params->ref_dec_op, ad->dev_id, queue_id,
4635 num_to_process, burst_sz, &time_st);
4636 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4637 iter = offload_latency_test_enc(op_params->mp, bufs,
4638 op_params->ref_enc_op, ad->dev_id, queue_id,
4639 num_to_process, burst_sz, &time_st);
4640 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4641 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4642 op_params->ref_enc_op, ad->dev_id, queue_id,
4643 num_to_process, burst_sz, &time_st);
4644 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4645 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4646 op_params->ref_dec_op, ad->dev_id, queue_id,
4647 num_to_process, burst_sz, &time_st);
4649 iter = offload_latency_test_enc(op_params->mp, bufs,
4650 op_params->ref_enc_op, ad->dev_id, queue_id,
4651 num_to_process, burst_sz, &time_st);
4656 printf("Enqueue driver offload cost latency:\n"
4657 "\tavg: %lg cycles, %lg us\n"
4658 "\tmin: %lg cycles, %lg us\n"
4659 "\tmax: %lg cycles, %lg us\n"
4660 "Enqueue accelerator offload cost latency:\n"
4661 "\tavg: %lg cycles, %lg us\n"
4662 "\tmin: %lg cycles, %lg us\n"
4663 "\tmax: %lg cycles, %lg us\n",
4664 (double)time_st.enq_sw_total_time / (double)iter,
4665 (double)(time_st.enq_sw_total_time * 1000000) /
4666 (double)iter / (double)rte_get_tsc_hz(),
4667 (double)time_st.enq_sw_min_time,
4668 (double)(time_st.enq_sw_min_time * 1000000) /
4669 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4670 (double)(time_st.enq_sw_max_time * 1000000) /
4671 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4673 (double)(time_st.enq_acc_total_time * 1000000) /
4674 (double)iter / (double)rte_get_tsc_hz(),
4675 (double)time_st.enq_acc_min_time,
4676 (double)(time_st.enq_acc_min_time * 1000000) /
4677 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4678 (double)(time_st.enq_acc_max_time * 1000000) /
4681 printf("Dequeue offload cost latency - one op:\n"
4682 "\tavg: %lg cycles, %lg us\n"
4683 "\tmin: %lg cycles, %lg us\n"
4684 "\tmax: %lg cycles, %lg us\n",
4685 (double)time_st.deq_total_time / (double)iter,
4686 (double)(time_st.deq_total_time * 1000000) /
4687 (double)iter / (double)rte_get_tsc_hz(),
4688 (double)time_st.deq_min_time,
4689 (double)(time_st.deq_min_time * 1000000) /
4690 rte_get_tsc_hz(), (double)time_st.deq_max_time,
4691 (double)(time_st.deq_max_time * 1000000) /
4694 return TEST_SUCCESS;
4698 #ifdef RTE_BBDEV_OFFLOAD_COST
4700 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4701 const uint16_t num_to_process, uint16_t burst_sz,
4702 uint64_t *deq_total_time, uint64_t *deq_min_time,
4703 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4706 struct rte_bbdev_dec_op *ops[MAX_BURST];
4707 uint64_t deq_start_time, deq_last_time;
4709 /* Test deq offload latency from an empty queue */
4711 for (i = 0, deq_total = 0; deq_total < num_to_process;
4712 ++i, deq_total += burst_sz) {
4713 deq_start_time = rte_rdtsc_precise();
4715 if (unlikely(num_to_process - deq_total < burst_sz))
4716 burst_sz = num_to_process - deq_total;
4717 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4718 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4721 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4724 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4725 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4726 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4727 *deq_total_time += deq_last_time;
4734 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4735 const uint16_t num_to_process, uint16_t burst_sz,
4736 uint64_t *deq_total_time, uint64_t *deq_min_time,
4737 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4740 struct rte_bbdev_enc_op *ops[MAX_BURST];
4741 uint64_t deq_start_time, deq_last_time;
4743 /* Test deq offload latency from an empty queue */
4744 for (i = 0, deq_total = 0; deq_total < num_to_process;
4745 ++i, deq_total += burst_sz) {
4746 deq_start_time = rte_rdtsc_precise();
4748 if (unlikely(num_to_process - deq_total < burst_sz))
4749 burst_sz = num_to_process - deq_total;
4750 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4751 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4754 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4757 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4758 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4759 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4760 *deq_total_time += deq_last_time;
4769 offload_latency_empty_q_test(struct active_device *ad,
4770 struct test_op_params *op_params)
4772 #ifndef RTE_BBDEV_OFFLOAD_COST
4774 RTE_SET_USED(op_params);
4775 printf("Offload latency empty dequeue test is disabled.\n");
4776 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4777 return TEST_SKIPPED;
4780 uint64_t deq_total_time, deq_min_time, deq_max_time;
4781 uint16_t burst_sz = op_params->burst_sz;
4782 const uint16_t num_to_process = op_params->num_to_process;
4783 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4784 const uint16_t queue_id = ad->queue_ids[0];
4785 struct rte_bbdev_info info;
4786 const char *op_type_str;
4788 deq_total_time = deq_max_time = 0;
4789 deq_min_time = UINT64_MAX;
4791 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4792 "BURST_SIZE should be <= %u", MAX_BURST);
4794 rte_bbdev_info_get(ad->dev_id, &info);
4796 op_type_str = rte_bbdev_op_type_str(op_type);
4797 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4799 printf("+ ------------------------------------------------------- +\n");
4800 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4801 info.dev_name, burst_sz, num_to_process, op_type_str);
4803 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4804 op_type == RTE_BBDEV_OP_LDPC_DEC)
4805 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4806 num_to_process, burst_sz, &deq_total_time,
4807 &deq_min_time, &deq_max_time, op_type);
4809 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4810 num_to_process, burst_sz, &deq_total_time,
4811 &deq_min_time, &deq_max_time, op_type);
4816 printf("Empty dequeue offload:\n"
4817 "\tavg: %lg cycles, %lg us\n"
4818 "\tmin: %lg cycles, %lg us\n"
4819 "\tmax: %lg cycles, %lg us\n",
4820 (double)deq_total_time / (double)iter,
4821 (double)(deq_total_time * 1000000) / (double)iter /
4822 (double)rte_get_tsc_hz(), (double)deq_min_time,
4823 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4824 (double)deq_max_time, (double)(deq_max_time * 1000000) /
4827 return TEST_SUCCESS;
4834 return run_test_case(bler_test);
4840 return run_test_case(throughput_test);
4844 offload_cost_tc(void)
4846 return run_test_case(offload_cost_test);
4850 offload_latency_empty_q_tc(void)
4852 return run_test_case(offload_latency_empty_q_test);
4858 return run_test_case(latency_test);
4864 return run_test_case(throughput_test);
4867 static struct unit_test_suite bbdev_bler_testsuite = {
4868 .suite_name = "BBdev BLER Tests",
4869 .setup = testsuite_setup,
4870 .teardown = testsuite_teardown,
4871 .unit_test_cases = {
4872 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
4873 TEST_CASES_END() /**< NULL terminate unit test array */
4877 static struct unit_test_suite bbdev_throughput_testsuite = {
4878 .suite_name = "BBdev Throughput Tests",
4879 .setup = testsuite_setup,
4880 .teardown = testsuite_teardown,
4881 .unit_test_cases = {
4882 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
4883 TEST_CASES_END() /**< NULL terminate unit test array */
4887 static struct unit_test_suite bbdev_validation_testsuite = {
4888 .suite_name = "BBdev Validation Tests",
4889 .setup = testsuite_setup,
4890 .teardown = testsuite_teardown,
4891 .unit_test_cases = {
4892 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4893 TEST_CASES_END() /**< NULL terminate unit test array */
4897 static struct unit_test_suite bbdev_latency_testsuite = {
4898 .suite_name = "BBdev Latency Tests",
4899 .setup = testsuite_setup,
4900 .teardown = testsuite_teardown,
4901 .unit_test_cases = {
4902 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4903 TEST_CASES_END() /**< NULL terminate unit test array */
4907 static struct unit_test_suite bbdev_offload_cost_testsuite = {
4908 .suite_name = "BBdev Offload Cost Tests",
4909 .setup = testsuite_setup,
4910 .teardown = testsuite_teardown,
4911 .unit_test_cases = {
4912 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
4913 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
4914 TEST_CASES_END() /**< NULL terminate unit test array */
4918 static struct unit_test_suite bbdev_interrupt_testsuite = {
4919 .suite_name = "BBdev Interrupt Tests",
4920 .setup = interrupt_testsuite_setup,
4921 .teardown = testsuite_teardown,
4922 .unit_test_cases = {
4923 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
4924 TEST_CASES_END() /**< NULL terminate unit test array */
4928 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
4929 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
4930 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
4931 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
4932 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
4933 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);