1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
19 #include <rte_interrupts.h>
22 #include "test_bbdev_vector.h"
24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
26 #define MAX_QUEUES RTE_MAX_LCORE
27 #define TEST_REPETITIONS 100
28 #define WAIT_OFFLOAD_US 1000
30 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
31 #include <fpga_lte_fec.h>
32 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
33 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
34 #define VF_UL_4G_QUEUE_VALUE 4
35 #define VF_DL_4G_QUEUE_VALUE 4
36 #define UL_4G_BANDWIDTH 3
37 #define DL_4G_BANDWIDTH 3
38 #define UL_4G_LOAD_BALANCE 128
39 #define DL_4G_LOAD_BALANCE 128
40 #define FLR_4G_TIMEOUT 610
43 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
44 #include <rte_pmd_fpga_5gnr_fec.h>
45 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
46 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
47 #define VF_UL_5G_QUEUE_VALUE 4
48 #define VF_DL_5G_QUEUE_VALUE 4
49 #define UL_5G_BANDWIDTH 3
50 #define DL_5G_BANDWIDTH 3
51 #define UL_5G_LOAD_BALANCE 128
52 #define DL_5G_LOAD_BALANCE 128
53 #define FLR_5G_TIMEOUT 610
56 #ifdef RTE_BASEBAND_ACC100
57 #include <rte_acc100_cfg.h>
58 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf")
59 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf")
60 #define ACC100_QMGR_NUM_AQS 16
61 #define ACC100_QMGR_NUM_QGS 2
62 #define ACC100_QMGR_AQ_DEPTH 5
63 #define ACC100_QMGR_INVALID_IDX -1
64 #define ACC100_QMGR_RR 1
65 #define ACC100_QOS_GBR 0
68 #define OPS_CACHE_SIZE 256U
69 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
73 #define INVALID_OPAQUE -1
75 #define INVALID_QUEUE_ID -1
76 /* Increment for next code block in external HARQ memory */
77 #define HARQ_INCR 32768
78 /* Headroom for filler LLRs insertion in HARQ buffer */
79 #define FILLER_HEADROOM 1024
80 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
81 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
82 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
83 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
84 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
85 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
86 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
87 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
88 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
90 static struct test_bbdev_vector test_vector;
92 /* Switch between PMD and Interrupt for throughput TC */
93 static bool intr_enabled;
95 /* LLR arithmetic representation for numerical conversion */
96 static int ldpc_llr_decimals;
97 static int ldpc_llr_size;
98 /* Keep track of the LDPC decoder device capability flag */
99 static uint32_t ldpc_cap_flags;
101 /* Represents tested active devices */
102 static struct active_device {
103 const char *driver_name;
105 uint16_t supported_ops;
106 uint16_t queue_ids[MAX_QUEUES];
108 struct rte_mempool *ops_mempool;
109 struct rte_mempool *in_mbuf_pool;
110 struct rte_mempool *hard_out_mbuf_pool;
111 struct rte_mempool *soft_out_mbuf_pool;
112 struct rte_mempool *harq_in_mbuf_pool;
113 struct rte_mempool *harq_out_mbuf_pool;
114 } active_devs[RTE_BBDEV_MAX_DEVS];
116 static uint8_t nb_active_devs;
118 /* Data buffers used by BBDEV ops */
119 struct test_buffers {
120 struct rte_bbdev_op_data *inputs;
121 struct rte_bbdev_op_data *hard_outputs;
122 struct rte_bbdev_op_data *soft_outputs;
123 struct rte_bbdev_op_data *harq_inputs;
124 struct rte_bbdev_op_data *harq_outputs;
127 /* Operation parameters specific for given test case */
128 struct test_op_params {
129 struct rte_mempool *mp;
130 struct rte_bbdev_dec_op *ref_dec_op;
131 struct rte_bbdev_enc_op *ref_enc_op;
133 uint16_t num_to_process;
137 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
140 /* Contains per lcore params */
141 struct thread_params {
151 uint16_t nb_dequeued;
152 int16_t processing_status;
154 struct test_op_params *op_params;
155 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
156 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
159 #ifdef RTE_BBDEV_OFFLOAD_COST
160 /* Stores time statistics */
161 struct test_time_stats {
162 /* Stores software enqueue total working time */
163 uint64_t enq_sw_total_time;
164 /* Stores minimum value of software enqueue working time */
165 uint64_t enq_sw_min_time;
166 /* Stores maximum value of software enqueue working time */
167 uint64_t enq_sw_max_time;
168 /* Stores turbo enqueue total working time */
169 uint64_t enq_acc_total_time;
170 /* Stores minimum value of accelerator enqueue working time */
171 uint64_t enq_acc_min_time;
172 /* Stores maximum value of accelerator enqueue working time */
173 uint64_t enq_acc_max_time;
174 /* Stores dequeue total working time */
175 uint64_t deq_total_time;
176 /* Stores minimum value of dequeue working time */
177 uint64_t deq_min_time;
178 /* Stores maximum value of dequeue working time */
179 uint64_t deq_max_time;
183 typedef int (test_case_function)(struct active_device *ad,
184 struct test_op_params *op_params);
187 mbuf_reset(struct rte_mbuf *m)
197 /* Read flag value 0/1 from bitmap */
199 check_bit(uint32_t bitmap, uint32_t bitmask)
201 return bitmap & bitmask;
205 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
207 ad->supported_ops |= (1 << op_type);
211 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
213 return ad->supported_ops & (1 << op_type);
217 flags_match(uint32_t flags_req, uint32_t flags_present)
219 return (flags_req & flags_present) == flags_req;
223 clear_soft_out_cap(uint32_t *op_flags)
225 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
226 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
227 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
230 /* This API is to convert all the test vector op data entries
231 * to big endian format. It is used when the device supports
232 * the input in the big endian format.
235 convert_op_data_to_be(void)
237 struct op_data_entries *op;
238 enum op_data_type type;
239 uint8_t nb_segs, *rem_data, temp;
241 int complete, rem, i, j;
243 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
244 nb_segs = test_vector.entries[type].nb_segments;
245 op = &test_vector.entries[type];
247 /* Invert byte endianness for all the segments */
248 for (i = 0; i < nb_segs; ++i) {
249 len = op->segments[i].length;
250 data = op->segments[i].addr;
252 /* Swap complete u32 bytes */
254 for (j = 0; j < complete; j++)
255 data[j] = rte_bswap32(data[j]);
257 /* Swap any remaining bytes */
259 rem_data = (uint8_t *)&data[j];
260 for (j = 0; j < rem/2; j++) {
262 rem_data[j] = rem_data[rem - j - 1];
263 rem_data[rem - j - 1] = temp;
270 check_dev_cap(const struct rte_bbdev_info *dev_info)
273 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
274 nb_harq_inputs, nb_harq_outputs;
275 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
276 uint8_t dev_data_endianness = dev_info->drv.data_endianness;
278 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
279 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
280 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
281 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
282 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
284 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
285 if (op_cap->type != test_vector.op_type)
288 if (dev_data_endianness == RTE_BIG_ENDIAN)
289 convert_op_data_to_be();
291 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
292 const struct rte_bbdev_op_cap_turbo_dec *cap =
293 &op_cap->cap.turbo_dec;
294 /* Ignore lack of soft output capability, just skip
295 * checking if soft output is valid.
297 if ((test_vector.turbo_dec.op_flags &
298 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
299 !(cap->capability_flags &
300 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
302 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
305 &test_vector.turbo_dec.op_flags);
308 if (!flags_match(test_vector.turbo_dec.op_flags,
309 cap->capability_flags))
311 if (nb_inputs > cap->num_buffers_src) {
312 printf("Too many inputs defined: %u, max: %u\n",
313 nb_inputs, cap->num_buffers_src);
316 if (nb_soft_outputs > cap->num_buffers_soft_out &&
317 (test_vector.turbo_dec.op_flags &
318 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
320 "Too many soft outputs defined: %u, max: %u\n",
322 cap->num_buffers_soft_out);
325 if (nb_hard_outputs > cap->num_buffers_hard_out) {
327 "Too many hard outputs defined: %u, max: %u\n",
329 cap->num_buffers_hard_out);
332 if (intr_enabled && !(cap->capability_flags &
333 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
335 "Dequeue interrupts are not supported!\n");
340 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
341 const struct rte_bbdev_op_cap_turbo_enc *cap =
342 &op_cap->cap.turbo_enc;
344 if (!flags_match(test_vector.turbo_enc.op_flags,
345 cap->capability_flags))
347 if (nb_inputs > cap->num_buffers_src) {
348 printf("Too many inputs defined: %u, max: %u\n",
349 nb_inputs, cap->num_buffers_src);
352 if (nb_hard_outputs > cap->num_buffers_dst) {
354 "Too many hard outputs defined: %u, max: %u\n",
355 nb_hard_outputs, cap->num_buffers_dst);
358 if (intr_enabled && !(cap->capability_flags &
359 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
361 "Dequeue interrupts are not supported!\n");
366 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
367 const struct rte_bbdev_op_cap_ldpc_enc *cap =
368 &op_cap->cap.ldpc_enc;
370 if (!flags_match(test_vector.ldpc_enc.op_flags,
371 cap->capability_flags)){
372 printf("Flag Mismatch\n");
375 if (nb_inputs > cap->num_buffers_src) {
376 printf("Too many inputs defined: %u, max: %u\n",
377 nb_inputs, cap->num_buffers_src);
380 if (nb_hard_outputs > cap->num_buffers_dst) {
382 "Too many hard outputs defined: %u, max: %u\n",
383 nb_hard_outputs, cap->num_buffers_dst);
386 if (intr_enabled && !(cap->capability_flags &
387 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
389 "Dequeue interrupts are not supported!\n");
394 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
395 const struct rte_bbdev_op_cap_ldpc_dec *cap =
396 &op_cap->cap.ldpc_dec;
398 if (!flags_match(test_vector.ldpc_dec.op_flags,
399 cap->capability_flags)){
400 printf("Flag Mismatch\n");
403 if (nb_inputs > cap->num_buffers_src) {
404 printf("Too many inputs defined: %u, max: %u\n",
405 nb_inputs, cap->num_buffers_src);
408 if (nb_hard_outputs > cap->num_buffers_hard_out) {
410 "Too many hard outputs defined: %u, max: %u\n",
412 cap->num_buffers_hard_out);
415 if (nb_harq_inputs > cap->num_buffers_hard_out) {
417 "Too many HARQ inputs defined: %u, max: %u\n",
419 cap->num_buffers_hard_out);
422 if (nb_harq_outputs > cap->num_buffers_hard_out) {
424 "Too many HARQ outputs defined: %u, max: %u\n",
426 cap->num_buffers_hard_out);
429 if (intr_enabled && !(cap->capability_flags &
430 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
432 "Dequeue interrupts are not supported!\n");
435 if (intr_enabled && (test_vector.ldpc_dec.op_flags &
436 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
437 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
438 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
440 printf("Skip loop-back with interrupt\n");
447 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
448 return TEST_SUCCESS; /* Special case for NULL device */
453 /* calculates optimal mempool size not smaller than the val */
455 optimal_mempool_size(unsigned int val)
457 return rte_align32pow2(val + 1) - 1;
460 /* allocates mbuf mempool for inputs and outputs */
461 static struct rte_mempool *
462 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
463 int socket_id, unsigned int mbuf_pool_size,
464 const char *op_type_str)
467 uint32_t max_seg_sz = 0;
468 char pool_name[RTE_MEMPOOL_NAMESIZE];
470 /* find max input segment size */
471 for (i = 0; i < entries->nb_segments; ++i)
472 if (entries->segments[i].length > max_seg_sz)
473 max_seg_sz = entries->segments[i].length;
475 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
477 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
478 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
480 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
484 create_mempools(struct active_device *ad, int socket_id,
485 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
487 struct rte_mempool *mp;
488 unsigned int ops_pool_size, mbuf_pool_size = 0;
489 char pool_name[RTE_MEMPOOL_NAMESIZE];
490 const char *op_type_str;
491 enum rte_bbdev_op_type op_type = org_op_type;
493 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
494 struct op_data_entries *hard_out =
495 &test_vector.entries[DATA_HARD_OUTPUT];
496 struct op_data_entries *soft_out =
497 &test_vector.entries[DATA_SOFT_OUTPUT];
498 struct op_data_entries *harq_in =
499 &test_vector.entries[DATA_HARQ_INPUT];
500 struct op_data_entries *harq_out =
501 &test_vector.entries[DATA_HARQ_OUTPUT];
503 /* allocate ops mempool */
504 ops_pool_size = optimal_mempool_size(RTE_MAX(
505 /* Ops used plus 1 reference op */
506 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
507 /* Minimal cache size plus 1 reference op */
508 (unsigned int)(1.5 * rte_lcore_count() *
509 OPS_CACHE_SIZE + 1)),
512 if (org_op_type == RTE_BBDEV_OP_NONE)
513 op_type = RTE_BBDEV_OP_TURBO_ENC;
515 op_type_str = rte_bbdev_op_type_str(op_type);
516 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
518 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
520 mp = rte_bbdev_op_pool_create(pool_name, op_type,
521 ops_pool_size, OPS_CACHE_SIZE, socket_id);
522 TEST_ASSERT_NOT_NULL(mp,
523 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
527 ad->ops_mempool = mp;
529 /* Do not create inputs and outputs mbufs for BaseBand Null Device */
530 if (org_op_type == RTE_BBDEV_OP_NONE)
534 if (in->nb_segments > 0) {
535 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
537 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
538 mbuf_pool_size, "in");
539 TEST_ASSERT_NOT_NULL(mp,
540 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
544 ad->in_mbuf_pool = mp;
548 if (hard_out->nb_segments > 0) {
549 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
550 hard_out->nb_segments);
551 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
554 TEST_ASSERT_NOT_NULL(mp,
555 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
559 ad->hard_out_mbuf_pool = mp;
563 if (soft_out->nb_segments > 0) {
564 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
565 soft_out->nb_segments);
566 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
569 TEST_ASSERT_NOT_NULL(mp,
570 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
574 ad->soft_out_mbuf_pool = mp;
578 if (harq_in->nb_segments > 0) {
579 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
580 harq_in->nb_segments);
581 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
584 TEST_ASSERT_NOT_NULL(mp,
585 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
589 ad->harq_in_mbuf_pool = mp;
593 if (harq_out->nb_segments > 0) {
594 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
595 harq_out->nb_segments);
596 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
599 TEST_ASSERT_NOT_NULL(mp,
600 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
604 ad->harq_out_mbuf_pool = mp;
611 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
612 struct test_bbdev_vector *vector)
615 unsigned int queue_id;
616 struct rte_bbdev_queue_conf qconf;
617 struct active_device *ad = &active_devs[nb_active_devs];
618 unsigned int nb_queues;
619 enum rte_bbdev_op_type op_type = vector->op_type;
621 /* Configure fpga lte fec with PF & VF values
622 * if '-i' flag is set and using fpga device
624 #ifdef RTE_BASEBAND_FPGA_LTE_FEC
625 if ((get_init_device() == true) &&
626 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
627 struct rte_fpga_lte_fec_conf conf;
630 printf("Configure FPGA LTE FEC Driver %s with default values\n",
631 info->drv.driver_name);
633 /* clear default configuration before initialization */
634 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf));
637 * true if PF is used for data plane
640 conf.pf_mode_en = true;
642 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
643 /* Number of UL queues per VF (fpga supports 8 VFs) */
644 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
645 /* Number of DL queues per VF (fpga supports 8 VFs) */
646 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
649 /* UL bandwidth. Needed for schedule algorithm */
650 conf.ul_bandwidth = UL_4G_BANDWIDTH;
652 conf.dl_bandwidth = DL_4G_BANDWIDTH;
654 /* UL & DL load Balance Factor to 64 */
655 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
656 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
658 /**< FLR timeout value */
659 conf.flr_time_out = FLR_4G_TIMEOUT;
661 /* setup FPGA PF with configuration information */
662 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf);
663 TEST_ASSERT_SUCCESS(ret,
664 "Failed to configure 4G FPGA PF for bbdev %s",
668 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC
669 if ((get_init_device() == true) &&
670 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
671 struct rte_fpga_5gnr_fec_conf conf;
674 printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
675 info->drv.driver_name);
677 /* clear default configuration before initialization */
678 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
681 * true if PF is used for data plane
684 conf.pf_mode_en = true;
686 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
687 /* Number of UL queues per VF (fpga supports 8 VFs) */
688 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
689 /* Number of DL queues per VF (fpga supports 8 VFs) */
690 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
693 /* UL bandwidth. Needed for schedule algorithm */
694 conf.ul_bandwidth = UL_5G_BANDWIDTH;
696 conf.dl_bandwidth = DL_5G_BANDWIDTH;
698 /* UL & DL load Balance Factor to 64 */
699 conf.ul_load_balance = UL_5G_LOAD_BALANCE;
700 conf.dl_load_balance = DL_5G_LOAD_BALANCE;
702 /**< FLR timeout value */
703 conf.flr_time_out = FLR_5G_TIMEOUT;
705 /* setup FPGA PF with configuration information */
706 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf);
707 TEST_ASSERT_SUCCESS(ret,
708 "Failed to configure 5G FPGA PF for bbdev %s",
712 #ifdef RTE_BASEBAND_ACC100
713 if ((get_init_device() == true) &&
714 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) {
715 struct rte_acc100_conf conf;
718 printf("Configure ACC100 FEC Driver %s with default values\n",
719 info->drv.driver_name);
721 /* clear default configuration before initialization */
722 memset(&conf, 0, sizeof(struct rte_acc100_conf));
724 /* Always set in PF mode for built-in configuration */
725 conf.pf_mode_en = true;
726 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) {
727 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
728 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
729 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR;
730 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
731 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR;
732 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR;
733 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
734 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
735 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR;
736 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
737 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR;
738 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR;
741 conf.input_pos_llr_1_bit = true;
742 conf.output_pos_llr_1_bit = true;
743 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */
745 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
746 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
747 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
748 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
749 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS;
750 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
751 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
752 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
753 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
754 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
755 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
756 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
757 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS;
758 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX;
759 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS;
760 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH;
762 /* setup PF with configuration information */
763 ret = rte_acc100_configure(info->dev_name, &conf);
764 TEST_ASSERT_SUCCESS(ret,
765 "Failed to configure ACC100 PF for bbdev %s",
769 /* Let's refresh this now this is configured */
770 rte_bbdev_info_get(dev_id, info);
771 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
772 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
775 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
777 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
778 dev_id, nb_queues, info->socket_id, ret);
782 /* configure interrupts if needed */
784 ret = rte_bbdev_intr_enable(dev_id);
786 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
792 /* setup device queues */
793 qconf.socket = info->socket_id;
794 qconf.queue_size = info->drv.default_queue_conf.queue_size;
796 qconf.deferred_start = 0;
797 qconf.op_type = op_type;
799 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
800 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
803 "Allocated all queues (id=%u) at prio%u on dev%u\n",
804 queue_id, qconf.priority, dev_id);
806 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
810 printf("All queues on dev %u allocated: %u\n",
814 ad->queue_ids[queue_id] = queue_id;
816 TEST_ASSERT(queue_id != 0,
817 "ERROR Failed to configure any queues on dev %u",
819 ad->nb_queues = queue_id;
821 set_avail_op(ad, op_type);
827 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
828 struct test_bbdev_vector *vector)
832 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
833 active_devs[nb_active_devs].dev_id = dev_id;
835 ret = add_bbdev_dev(dev_id, info, vector);
836 if (ret == TEST_SUCCESS)
842 populate_active_devices(void)
846 uint8_t nb_devs_added = 0;
847 struct rte_bbdev_info info;
849 RTE_BBDEV_FOREACH(dev_id) {
850 rte_bbdev_info_get(dev_id, &info);
852 if (check_dev_cap(&info)) {
854 "Device %d (%s) does not support specified capabilities\n",
855 dev_id, info.dev_name);
859 ret = add_active_device(dev_id, &info, &test_vector);
861 printf("Adding active bbdev %s skipped\n",
868 return nb_devs_added;
872 read_test_vector(void)
876 memset(&test_vector, 0, sizeof(test_vector));
877 printf("Test vector file = %s\n", get_vector_filename());
878 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
879 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
880 get_vector_filename());
886 testsuite_setup(void)
888 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
890 if (populate_active_devices() == 0) {
891 printf("No suitable devices found!\n");
899 interrupt_testsuite_setup(void)
901 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
903 /* Enable interrupts */
906 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
907 if (populate_active_devices() == 0 ||
908 test_vector.op_type == RTE_BBDEV_OP_NONE) {
909 intr_enabled = false;
910 printf("No suitable devices found!\n");
918 testsuite_teardown(void)
922 /* Unconfigure devices */
923 RTE_BBDEV_FOREACH(dev_id)
924 rte_bbdev_close(dev_id);
926 /* Clear active devices structs. */
927 memset(active_devs, 0, sizeof(active_devs));
930 /* Disable interrupts */
931 intr_enabled = false;
939 for (i = 0; i < nb_active_devs; i++) {
940 dev_id = active_devs[i].dev_id;
941 /* reset bbdev stats */
942 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
943 "Failed to reset stats of bbdev %u", dev_id);
944 /* start the device */
945 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
946 "Failed to start bbdev %u", dev_id);
956 struct rte_bbdev_stats stats;
958 for (i = 0; i < nb_active_devs; i++) {
959 dev_id = active_devs[i].dev_id;
960 /* read stats and print */
961 rte_bbdev_stats_get(dev_id, &stats);
962 /* Stop the device */
963 rte_bbdev_stop(dev_id);
968 init_op_data_objs(struct rte_bbdev_op_data *bufs,
969 struct op_data_entries *ref_entries,
970 struct rte_mempool *mbuf_pool, const uint16_t n,
971 enum op_data_type op_type, uint16_t min_alignment)
975 bool large_input = false;
977 for (i = 0; i < n; ++i) {
979 struct op_data_buf *seg = &ref_entries->segments[0];
980 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
981 TEST_ASSERT_NOT_NULL(m_head,
982 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
983 op_type, n * ref_entries->nb_segments,
986 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
988 * Special case when DPDK mbuf cannot handle
989 * the required input size
991 printf("Warning: Larger input size than DPDK mbuf %d\n",
995 bufs[i].data = m_head;
999 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
1000 if ((op_type == DATA_INPUT) && large_input) {
1001 /* Allocate a fake overused mbuf */
1002 data = rte_malloc(NULL, seg->length, 0);
1003 TEST_ASSERT_NOT_NULL(data,
1004 "rte malloc failed with %u bytes",
1006 memcpy(data, seg->addr, seg->length);
1007 m_head->buf_addr = data;
1008 m_head->buf_iova = rte_malloc_virt2iova(data);
1009 m_head->data_off = 0;
1010 m_head->data_len = seg->length;
1012 data = rte_pktmbuf_append(m_head, seg->length);
1013 TEST_ASSERT_NOT_NULL(data,
1014 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
1015 seg->length, op_type);
1017 TEST_ASSERT(data == RTE_PTR_ALIGN(
1018 data, min_alignment),
1019 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
1020 data, min_alignment);
1021 rte_memcpy(data, seg->addr, seg->length);
1024 bufs[i].length += seg->length;
1026 for (j = 1; j < ref_entries->nb_segments; ++j) {
1027 struct rte_mbuf *m_tail =
1028 rte_pktmbuf_alloc(mbuf_pool);
1029 TEST_ASSERT_NOT_NULL(m_tail,
1030 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1032 n * ref_entries->nb_segments,
1036 data = rte_pktmbuf_append(m_tail, seg->length);
1037 TEST_ASSERT_NOT_NULL(data,
1038 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
1039 seg->length, op_type);
1041 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
1043 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
1044 data, min_alignment);
1045 rte_memcpy(data, seg->addr, seg->length);
1046 bufs[i].length += seg->length;
1048 ret = rte_pktmbuf_chain(m_head, m_tail);
1049 TEST_ASSERT_SUCCESS(ret,
1050 "Couldn't chain mbufs from %d data type mbuf pool",
1055 /* allocate chained-mbuf for output buffer */
1056 for (j = 1; j < ref_entries->nb_segments; ++j) {
1057 struct rte_mbuf *m_tail =
1058 rte_pktmbuf_alloc(mbuf_pool);
1059 TEST_ASSERT_NOT_NULL(m_tail,
1060 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
1062 n * ref_entries->nb_segments,
1065 ret = rte_pktmbuf_chain(m_head, m_tail);
1066 TEST_ASSERT_SUCCESS(ret,
1067 "Couldn't chain mbufs from %d data type mbuf pool",
1077 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
1082 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
1083 if (*buffers == NULL) {
1084 printf("WARNING: Failed to allocate op_data on socket %d\n",
1086 /* try to allocate memory on other detected sockets */
1087 for (i = 0; i < socket; i++) {
1088 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
1089 if (*buffers != NULL)
1094 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
1098 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
1099 const uint16_t n, const int8_t max_llr_modulus)
1101 uint16_t i, byte_idx;
1103 for (i = 0; i < n; ++i) {
1104 struct rte_mbuf *m = input_ops[i].data;
1106 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1107 input_ops[i].offset);
1108 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1110 llr[byte_idx] = round((double)max_llr_modulus *
1111 llr[byte_idx] / INT8_MAX);
1119 * We may have to insert filler bits
1120 * when they are required by the HARQ assumption
1123 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
1124 const uint16_t n, struct test_op_params *op_params)
1126 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
1128 if (input_ops == NULL)
1130 /* No need to add filler if not required by device */
1131 if (!(ldpc_cap_flags &
1132 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
1134 /* No need to add filler for loopback operation */
1135 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1138 uint16_t i, j, parity_offset;
1139 for (i = 0; i < n; ++i) {
1140 struct rte_mbuf *m = input_ops[i].data;
1141 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1142 input_ops[i].offset);
1143 parity_offset = (dec.basegraph == 1 ? 20 : 8)
1144 * dec.z_c - dec.n_filler;
1145 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
1146 m->data_len = new_hin_size;
1147 input_ops[i].length = new_hin_size;
1148 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
1150 llr[j] = llr[j - dec.n_filler];
1151 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1152 for (j = 0; j < dec.n_filler; j++)
1153 llr[parity_offset + j] = llr_max_pre_scaling;
1158 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
1159 const uint16_t n, const int8_t llr_size,
1160 const int8_t llr_decimals)
1162 if (input_ops == NULL)
1165 uint16_t i, byte_idx;
1167 int16_t llr_max, llr_min, llr_tmp;
1168 llr_max = (1 << (llr_size - 1)) - 1;
1170 for (i = 0; i < n; ++i) {
1171 struct rte_mbuf *m = input_ops[i].data;
1173 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
1174 input_ops[i].offset);
1175 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1178 llr_tmp = llr[byte_idx];
1179 if (llr_decimals == 4)
1181 else if (llr_decimals == 2)
1183 else if (llr_decimals == 0)
1185 llr_tmp = RTE_MIN(llr_max,
1186 RTE_MAX(llr_min, llr_tmp));
1187 llr[byte_idx] = (int8_t) llr_tmp;
1198 fill_queue_buffers(struct test_op_params *op_params,
1199 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1200 struct rte_mempool *soft_out_mp,
1201 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1203 const struct rte_bbdev_op_cap *capabilities,
1204 uint16_t min_alignment, const int socket_id)
1207 enum op_data_type type;
1208 const uint16_t n = op_params->num_to_process;
1210 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1218 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1219 &op_params->q_bufs[socket_id][queue_id].inputs,
1220 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1221 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1222 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1223 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1226 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1227 struct op_data_entries *ref_entries =
1228 &test_vector.entries[type];
1229 if (ref_entries->nb_segments == 0)
1232 ret = allocate_buffers_on_socket(queue_ops[type],
1233 n * sizeof(struct rte_bbdev_op_data),
1235 TEST_ASSERT_SUCCESS(ret,
1236 "Couldn't allocate memory for rte_bbdev_op_data structs");
1238 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1239 mbuf_pools[type], n, type, min_alignment);
1240 TEST_ASSERT_SUCCESS(ret,
1241 "Couldn't init rte_bbdev_op_data structs");
1244 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1245 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1246 capabilities->cap.turbo_dec.max_llr_modulus);
1248 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1249 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1250 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1251 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1252 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1253 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1254 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1255 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1256 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1257 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1258 if (!loopback && !llr_comp)
1259 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1260 ldpc_llr_size, ldpc_llr_decimals);
1261 if (!loopback && !harq_comp)
1262 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1263 ldpc_llr_size, ldpc_llr_decimals);
1265 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1273 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1277 rte_mempool_free(ad->ops_mempool);
1278 rte_mempool_free(ad->in_mbuf_pool);
1279 rte_mempool_free(ad->hard_out_mbuf_pool);
1280 rte_mempool_free(ad->soft_out_mbuf_pool);
1281 rte_mempool_free(ad->harq_in_mbuf_pool);
1282 rte_mempool_free(ad->harq_out_mbuf_pool);
1284 for (i = 0; i < rte_lcore_count(); ++i) {
1285 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1286 rte_free(op_params->q_bufs[j][i].inputs);
1287 rte_free(op_params->q_bufs[j][i].hard_outputs);
1288 rte_free(op_params->q_bufs[j][i].soft_outputs);
1289 rte_free(op_params->q_bufs[j][i].harq_inputs);
1290 rte_free(op_params->q_bufs[j][i].harq_outputs);
1296 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1297 unsigned int start_idx,
1298 struct rte_bbdev_op_data *inputs,
1299 struct rte_bbdev_op_data *hard_outputs,
1300 struct rte_bbdev_op_data *soft_outputs,
1301 struct rte_bbdev_dec_op *ref_op)
1304 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1306 for (i = 0; i < n; ++i) {
1307 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1308 ops[i]->turbo_dec.tb_params.ea =
1309 turbo_dec->tb_params.ea;
1310 ops[i]->turbo_dec.tb_params.eb =
1311 turbo_dec->tb_params.eb;
1312 ops[i]->turbo_dec.tb_params.k_pos =
1313 turbo_dec->tb_params.k_pos;
1314 ops[i]->turbo_dec.tb_params.k_neg =
1315 turbo_dec->tb_params.k_neg;
1316 ops[i]->turbo_dec.tb_params.c =
1317 turbo_dec->tb_params.c;
1318 ops[i]->turbo_dec.tb_params.c_neg =
1319 turbo_dec->tb_params.c_neg;
1320 ops[i]->turbo_dec.tb_params.cab =
1321 turbo_dec->tb_params.cab;
1322 ops[i]->turbo_dec.tb_params.r =
1323 turbo_dec->tb_params.r;
1325 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1326 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1329 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1330 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1331 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1332 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1333 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1334 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1335 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1337 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1338 ops[i]->turbo_dec.input = inputs[start_idx + i];
1339 if (soft_outputs != NULL)
1340 ops[i]->turbo_dec.soft_output =
1341 soft_outputs[start_idx + i];
1346 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1347 unsigned int start_idx,
1348 struct rte_bbdev_op_data *inputs,
1349 struct rte_bbdev_op_data *outputs,
1350 struct rte_bbdev_enc_op *ref_op)
1353 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1354 for (i = 0; i < n; ++i) {
1355 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1356 ops[i]->turbo_enc.tb_params.ea =
1357 turbo_enc->tb_params.ea;
1358 ops[i]->turbo_enc.tb_params.eb =
1359 turbo_enc->tb_params.eb;
1360 ops[i]->turbo_enc.tb_params.k_pos =
1361 turbo_enc->tb_params.k_pos;
1362 ops[i]->turbo_enc.tb_params.k_neg =
1363 turbo_enc->tb_params.k_neg;
1364 ops[i]->turbo_enc.tb_params.c =
1365 turbo_enc->tb_params.c;
1366 ops[i]->turbo_enc.tb_params.c_neg =
1367 turbo_enc->tb_params.c_neg;
1368 ops[i]->turbo_enc.tb_params.cab =
1369 turbo_enc->tb_params.cab;
1370 ops[i]->turbo_enc.tb_params.ncb_pos =
1371 turbo_enc->tb_params.ncb_pos;
1372 ops[i]->turbo_enc.tb_params.ncb_neg =
1373 turbo_enc->tb_params.ncb_neg;
1374 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1376 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1377 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1378 ops[i]->turbo_enc.cb_params.ncb =
1379 turbo_enc->cb_params.ncb;
1381 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1382 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1383 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1385 ops[i]->turbo_enc.output = outputs[start_idx + i];
1386 ops[i]->turbo_enc.input = inputs[start_idx + i];
1391 /* Returns a random number drawn from a normal distribution
1392 * with mean of 0 and variance of 1
1393 * Marsaglia algorithm
1398 double S, Z, U1, U2, u, v, fac;
1401 U1 = (double)rand() / RAND_MAX;
1402 U2 = (double)rand() / RAND_MAX;
1406 } while (S >= 1 || S == 0);
1407 fac = sqrt(-2. * log(S) / S);
1408 Z = (n % 2) ? u * fac : v * fac;
1412 static inline double
1413 maxstar(double A, double B)
1415 if (fabs(A - B) > 5)
1416 return RTE_MAX(A, B);
1418 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1422 * Generate Qm LLRS for Qm==8
1423 * Modulation, AWGN and LLR estimation from max log development
1426 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1431 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1432 /* 5.1.4 of TS38.211 */
1433 const double symbols_I[256] = {
1434 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1435 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1436 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1437 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1438 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1439 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1440 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1441 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1442 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1443 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1444 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1445 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1446 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1447 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1448 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1449 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1450 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1451 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1452 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1453 -13, -13, -15, -15, -13, -13, -15, -15};
1454 const double symbols_Q[256] = {
1455 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1456 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1457 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1458 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1459 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1460 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1461 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1462 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1463 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1464 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1465 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1466 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1467 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1468 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1469 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1470 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1471 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1472 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1473 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1474 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1475 /* Average constellation point energy */
1477 for (k = 0; k < qm; k++)
1478 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1479 /* 5.1.4 of TS38.211 */
1480 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1481 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1482 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1483 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1485 I += sqrt(N0 / 2) * randn(0);
1486 Q += sqrt(N0 / 2) * randn(1);
1488 * Calculate the log of the probability that each of
1489 * the constellation points was transmitted
1491 for (m = 0; m < qam; m++)
1492 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1493 + pow(Q - symbols_Q[m], 2.0)) / N0;
1494 /* Calculate an LLR for each of the k_64QAM bits in the set */
1495 for (k = 0; k < qm; k++) {
1498 /* For each constellation point */
1499 for (m = 0; m < qam; m++) {
1500 if ((m >> (qm - k - 1)) & 1)
1501 p1 = maxstar(p1, log_syml_prob[m]);
1503 p0 = maxstar(p0, log_syml_prob[m]);
1505 /* Calculate the LLR */
1507 llr_ *= (1 << ldpc_llr_decimals);
1511 if (llr_ < -llr_max)
1513 llrs[qm * i + k] = (int8_t) llr_;
1519 * Generate Qm LLRS for Qm==6
1520 * Modulation, AWGN and LLR estimation from max log development
1523 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1528 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1529 /* 5.1.4 of TS38.211 */
1530 const double symbols_I[64] = {
1531 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1532 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1533 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1534 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1535 -5, -5, -7, -7, -5, -5, -7, -7};
1536 const double symbols_Q[64] = {
1537 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1538 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1539 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1540 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1541 -3, -1, -3, -1, -5, -7, -5, -7};
1542 /* Average constellation point energy */
1544 for (k = 0; k < qm; k++)
1545 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1546 /* 5.1.4 of TS38.211 */
1547 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1548 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1550 I += sqrt(N0 / 2) * randn(0);
1551 Q += sqrt(N0 / 2) * randn(1);
1553 * Calculate the log of the probability that each of
1554 * the constellation points was transmitted
1556 for (m = 0; m < qam; m++)
1557 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1558 + pow(Q - symbols_Q[m], 2.0)) / N0;
1559 /* Calculate an LLR for each of the k_64QAM bits in the set */
1560 for (k = 0; k < qm; k++) {
1563 /* For each constellation point */
1564 for (m = 0; m < qam; m++) {
1565 if ((m >> (qm - k - 1)) & 1)
1566 p1 = maxstar(p1, log_syml_prob[m]);
1568 p0 = maxstar(p0, log_syml_prob[m]);
1570 /* Calculate the LLR */
1572 llr_ *= (1 << ldpc_llr_decimals);
1576 if (llr_ < -llr_max)
1578 llrs[qm * i + k] = (int8_t) llr_;
1583 * Generate Qm LLRS for Qm==4
1584 * Modulation, AWGN and LLR estimation from max log development
1587 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1592 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1593 /* 5.1.4 of TS38.211 */
1594 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1595 -1, -1, -3, -3, -1, -1, -3, -3};
1596 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1597 1, 3, 1, 3, -1, -3, -1, -3};
1598 /* Average constellation point energy */
1600 for (k = 0; k < qm; k++)
1601 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1602 /* 5.1.4 of TS38.211 */
1603 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1604 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1606 I += sqrt(N0 / 2) * randn(0);
1607 Q += sqrt(N0 / 2) * randn(1);
1609 * Calculate the log of the probability that each of
1610 * the constellation points was transmitted
1612 for (m = 0; m < qam; m++)
1613 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1614 + pow(Q - symbols_Q[m], 2.0)) / N0;
1615 /* Calculate an LLR for each of the k_64QAM bits in the set */
1616 for (k = 0; k < qm; k++) {
1619 /* For each constellation point */
1620 for (m = 0; m < qam; m++) {
1621 if ((m >> (qm - k - 1)) & 1)
1622 p1 = maxstar(p1, log_syml_prob[m]);
1624 p0 = maxstar(p0, log_syml_prob[m]);
1626 /* Calculate the LLR */
1628 llr_ *= (1 << ldpc_llr_decimals);
1632 if (llr_ < -llr_max)
1634 llrs[qm * i + k] = (int8_t) llr_;
1639 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1642 double coeff = 2.0 * sqrt(N0);
1644 /* Ignore in vectors rare quasi null LLRs not to be saturated */
1645 if (llrs[j] < 8 && llrs[j] > -8)
1648 /* Note don't change sign here */
1650 b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1652 b = b1 * (1 << ldpc_llr_decimals);
1658 llrs[j] = (int8_t) b;
1661 /* Generate LLR for a given SNR */
1663 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1664 struct rte_bbdev_dec_op *ref_op)
1668 uint32_t i, j, e, range;
1671 e = ref_op->ldpc_dec.cb_params.e;
1672 qm = ref_op->ldpc_dec.q_m;
1673 llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1675 N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1677 for (i = 0; i < n; ++i) {
1679 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1681 for (j = 0; j < range; ++j)
1682 gen_qm8_llr(llrs, j, N0, llr_max);
1683 } else if (qm == 6) {
1684 for (j = 0; j < range; ++j)
1685 gen_qm6_llr(llrs, j, N0, llr_max);
1686 } else if (qm == 4) {
1687 for (j = 0; j < range; ++j)
1688 gen_qm4_llr(llrs, j, N0, llr_max);
1690 for (j = 0; j < e; ++j)
1691 gen_qm2_llr(llrs, j, N0, llr_max);
1697 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1698 unsigned int start_idx,
1699 struct rte_bbdev_op_data *inputs,
1700 struct rte_bbdev_op_data *hard_outputs,
1701 struct rte_bbdev_op_data *soft_outputs,
1702 struct rte_bbdev_op_data *harq_inputs,
1703 struct rte_bbdev_op_data *harq_outputs,
1704 struct rte_bbdev_dec_op *ref_op)
1707 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1709 for (i = 0; i < n; ++i) {
1710 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1711 ops[i]->ldpc_dec.tb_params.ea =
1712 ldpc_dec->tb_params.ea;
1713 ops[i]->ldpc_dec.tb_params.eb =
1714 ldpc_dec->tb_params.eb;
1715 ops[i]->ldpc_dec.tb_params.c =
1716 ldpc_dec->tb_params.c;
1717 ops[i]->ldpc_dec.tb_params.cab =
1718 ldpc_dec->tb_params.cab;
1719 ops[i]->ldpc_dec.tb_params.r =
1720 ldpc_dec->tb_params.r;
1722 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1725 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1726 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1727 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1728 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1729 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1730 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1731 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1732 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1733 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1735 if (hard_outputs != NULL)
1736 ops[i]->ldpc_dec.hard_output =
1737 hard_outputs[start_idx + i];
1739 ops[i]->ldpc_dec.input =
1740 inputs[start_idx + i];
1741 if (soft_outputs != NULL)
1742 ops[i]->ldpc_dec.soft_output =
1743 soft_outputs[start_idx + i];
1744 if (harq_inputs != NULL)
1745 ops[i]->ldpc_dec.harq_combined_input =
1746 harq_inputs[start_idx + i];
1747 if (harq_outputs != NULL)
1748 ops[i]->ldpc_dec.harq_combined_output =
1749 harq_outputs[start_idx + i];
1755 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1756 unsigned int start_idx,
1757 struct rte_bbdev_op_data *inputs,
1758 struct rte_bbdev_op_data *outputs,
1759 struct rte_bbdev_enc_op *ref_op)
1762 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1763 for (i = 0; i < n; ++i) {
1764 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1765 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1766 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1767 ops[i]->ldpc_enc.tb_params.cab =
1768 ldpc_enc->tb_params.cab;
1769 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1770 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1772 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1774 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1775 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1776 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1777 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1778 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1779 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1780 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1781 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1782 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1783 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1788 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1789 unsigned int order_idx, const int expected_status)
1791 int status = op->status;
1792 /* ignore parity mismatch false alarms for long iterations */
1793 if (get_iter_max() >= 10) {
1794 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1795 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1796 printf("WARNING: Ignore Syndrome Check mismatch\n");
1797 status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1799 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1800 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1801 printf("WARNING: Ignore Syndrome Check mismatch\n");
1802 status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1806 TEST_ASSERT(status == expected_status,
1807 "op_status (%d) != expected_status (%d)",
1808 op->status, expected_status);
1810 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1811 "Ordering error, expected %p, got %p",
1812 (void *)(uintptr_t)order_idx, op->opaque_data);
1814 return TEST_SUCCESS;
1818 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1819 unsigned int order_idx, const int expected_status)
1821 TEST_ASSERT(op->status == expected_status,
1822 "op_status (%d) != expected_status (%d)",
1823 op->status, expected_status);
1825 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1826 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1827 "Ordering error, expected %p, got %p",
1828 (void *)(uintptr_t)order_idx, op->opaque_data);
1830 return TEST_SUCCESS;
1834 validate_op_chain(struct rte_bbdev_op_data *op,
1835 struct op_data_entries *orig_op)
1838 struct rte_mbuf *m = op->data;
1839 uint8_t nb_dst_segments = orig_op->nb_segments;
1840 uint32_t total_data_size = 0;
1842 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1843 "Number of segments differ in original (%u) and filled (%u) op",
1844 nb_dst_segments, m->nb_segs);
1846 /* Validate each mbuf segment length */
1847 for (i = 0; i < nb_dst_segments; ++i) {
1848 /* Apply offset to the first mbuf segment */
1849 uint16_t offset = (i == 0) ? op->offset : 0;
1850 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1851 total_data_size += orig_op->segments[i].length;
1853 TEST_ASSERT(orig_op->segments[i].length == data_len,
1854 "Length of segment differ in original (%u) and filled (%u) op",
1855 orig_op->segments[i].length, data_len);
1856 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1857 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1859 "Output buffers (CB=%u) are not equal", i);
1863 /* Validate total mbuf pkt length */
1864 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1865 TEST_ASSERT(total_data_size == pkt_len,
1866 "Length of data differ in original (%u) and filled (%u) op",
1867 total_data_size, pkt_len);
1869 return TEST_SUCCESS;
1873 * Compute K0 for a given configuration for HARQ output length computation
1874 * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1876 static inline uint16_t
1877 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1881 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1884 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1885 else if (rv_index == 2)
1886 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1888 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1890 /* LBRM case - includes a division by N */
1892 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1894 else if (rv_index == 2)
1895 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1898 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1902 /* HARQ output length including the Filler bits */
1903 static inline uint16_t
1904 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1907 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1908 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1909 /* Compute RM out size and number of rows */
1910 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1911 * ops_ld->z_c - ops_ld->n_filler;
1912 uint16_t deRmOutSize = RTE_MIN(
1913 k0 + ops_ld->cb_params.e +
1914 ((k0 > parity_offset) ?
1915 0 : ops_ld->n_filler),
1917 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1919 uint16_t harq_output_len = numRows * ops_ld->z_c;
1920 return harq_output_len;
1924 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1925 struct op_data_entries *orig_op,
1926 struct rte_bbdev_op_ldpc_dec *ops_ld)
1930 struct rte_mbuf *m = op->data;
1931 uint8_t nb_dst_segments = orig_op->nb_segments;
1932 uint32_t total_data_size = 0;
1933 int8_t *harq_orig, *harq_out, abs_harq_origin;
1934 uint32_t byte_error = 0, cum_error = 0, error;
1935 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1936 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1937 uint16_t parity_offset;
1939 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1940 "Number of segments differ in original (%u) and filled (%u) op",
1941 nb_dst_segments, m->nb_segs);
1943 /* Validate each mbuf segment length */
1944 for (i = 0; i < nb_dst_segments; ++i) {
1945 /* Apply offset to the first mbuf segment */
1946 uint16_t offset = (i == 0) ? op->offset : 0;
1947 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1948 total_data_size += orig_op->segments[i].length;
1950 TEST_ASSERT(orig_op->segments[i].length <
1951 (uint32_t)(data_len + 64),
1952 "Length of segment differ in original (%u) and filled (%u) op",
1953 orig_op->segments[i].length, data_len);
1954 harq_orig = (int8_t *) orig_op->segments[i].addr;
1955 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1957 if (!(ldpc_cap_flags &
1958 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1959 ) || (ops_ld->op_flags &
1960 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1961 data_len -= ops_ld->z_c;
1962 parity_offset = data_len;
1964 /* Compute RM out size and number of rows */
1965 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1966 * ops_ld->z_c - ops_ld->n_filler;
1967 uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1969 if (data_len > deRmOutSize)
1970 data_len = deRmOutSize;
1971 if (data_len > orig_op->segments[i].length)
1972 data_len = orig_op->segments[i].length;
1975 * HARQ output can have minor differences
1976 * due to integer representation and related scaling
1978 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1979 if (j == parity_offset) {
1980 /* Special Handling of the filler bits */
1981 for (k = 0; k < ops_ld->n_filler; k++) {
1983 llr_max_pre_scaling) {
1984 printf("HARQ Filler issue %d: %d %d\n",
1992 if (!(ops_ld->op_flags &
1993 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1994 if (ldpc_llr_decimals > 1)
1995 harq_out[jj] = (harq_out[jj] + 1)
1996 >> (ldpc_llr_decimals - 1);
1997 /* Saturated to S7 */
1998 if (harq_orig[j] > llr_max)
1999 harq_orig[j] = llr_max;
2000 if (harq_orig[j] < -llr_max)
2001 harq_orig[j] = -llr_max;
2003 if (harq_orig[j] != harq_out[jj]) {
2004 error = (harq_orig[j] > harq_out[jj]) ?
2005 harq_orig[j] - harq_out[jj] :
2006 harq_out[jj] - harq_orig[j];
2007 abs_harq_origin = harq_orig[j] > 0 ?
2010 /* Residual quantization error */
2011 if ((error > 8 && (abs_harq_origin <
2014 printf("HARQ mismatch %d: exp %d act %d => %d\n",
2016 harq_out[jj], error);
2026 TEST_ASSERT(byte_error <= 1,
2027 "HARQ output mismatch (%d) %d",
2028 byte_error, cum_error);
2030 /* Validate total mbuf pkt length */
2031 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
2032 TEST_ASSERT(total_data_size < pkt_len + 64,
2033 "Length of data differ in original (%u) and filled (%u) op",
2034 total_data_size, pkt_len);
2036 return TEST_SUCCESS;
2040 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2041 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2045 struct op_data_entries *hard_data_orig =
2046 &test_vector.entries[DATA_HARD_OUTPUT];
2047 struct op_data_entries *soft_data_orig =
2048 &test_vector.entries[DATA_SOFT_OUTPUT];
2049 struct rte_bbdev_op_turbo_dec *ops_td;
2050 struct rte_bbdev_op_data *hard_output;
2051 struct rte_bbdev_op_data *soft_output;
2052 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
2054 for (i = 0; i < n; ++i) {
2055 ops_td = &ops[i]->turbo_dec;
2056 hard_output = &ops_td->hard_output;
2057 soft_output = &ops_td->soft_output;
2059 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2060 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2061 "Returned iter_count (%d) > expected iter_count (%d)",
2062 ops_td->iter_count, ref_td->iter_count);
2063 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2064 TEST_ASSERT_SUCCESS(ret,
2065 "Checking status and ordering for decoder failed");
2067 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2069 "Hard output buffers (CB=%u) are not equal",
2072 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
2073 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2075 "Soft output buffers (CB=%u) are not equal",
2079 return TEST_SUCCESS;
2082 /* Check Number of code blocks errors */
2084 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
2087 struct op_data_entries *hard_data_orig =
2088 &test_vector.entries[DATA_HARD_OUTPUT];
2089 struct rte_bbdev_op_ldpc_dec *ops_td;
2090 struct rte_bbdev_op_data *hard_output;
2094 for (i = 0; i < n; ++i) {
2095 ops_td = &ops[i]->ldpc_dec;
2096 hard_output = &ops_td->hard_output;
2097 m = hard_output->data;
2098 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
2099 hard_data_orig->segments[0].addr,
2100 hard_data_orig->segments[0].length))
2107 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
2108 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
2112 struct op_data_entries *hard_data_orig =
2113 &test_vector.entries[DATA_HARD_OUTPUT];
2114 struct op_data_entries *soft_data_orig =
2115 &test_vector.entries[DATA_SOFT_OUTPUT];
2116 struct op_data_entries *harq_data_orig =
2117 &test_vector.entries[DATA_HARQ_OUTPUT];
2118 struct rte_bbdev_op_ldpc_dec *ops_td;
2119 struct rte_bbdev_op_data *hard_output;
2120 struct rte_bbdev_op_data *harq_output;
2121 struct rte_bbdev_op_data *soft_output;
2122 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
2124 for (i = 0; i < n; ++i) {
2125 ops_td = &ops[i]->ldpc_dec;
2126 hard_output = &ops_td->hard_output;
2127 harq_output = &ops_td->harq_combined_output;
2128 soft_output = &ops_td->soft_output;
2130 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
2131 TEST_ASSERT_SUCCESS(ret,
2132 "Checking status and ordering for decoder failed");
2133 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
2134 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
2135 "Returned iter_count (%d) > expected iter_count (%d)",
2136 ops_td->iter_count, ref_td->iter_count);
2138 * We can ignore output data when the decoding failed to
2139 * converge or for loop-back cases
2141 if (!check_bit(ops[i]->ldpc_dec.op_flags,
2142 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
2144 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
2146 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
2148 "Hard output buffers (CB=%u) are not equal",
2151 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
2152 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
2154 "Soft output buffers (CB=%u) are not equal",
2156 if (ref_op->ldpc_dec.op_flags &
2157 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
2158 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2159 harq_data_orig, ops_td),
2160 "HARQ output buffers (CB=%u) are not equal",
2163 if (ref_op->ldpc_dec.op_flags &
2164 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
2165 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
2166 harq_data_orig, ops_td),
2167 "HARQ output buffers (CB=%u) are not equal",
2172 return TEST_SUCCESS;
2177 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2178 struct rte_bbdev_enc_op *ref_op)
2182 struct op_data_entries *hard_data_orig =
2183 &test_vector.entries[DATA_HARD_OUTPUT];
2185 for (i = 0; i < n; ++i) {
2186 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2187 TEST_ASSERT_SUCCESS(ret,
2188 "Checking status and ordering for encoder failed");
2189 TEST_ASSERT_SUCCESS(validate_op_chain(
2190 &ops[i]->turbo_enc.output,
2192 "Output buffers (CB=%u) are not equal",
2196 return TEST_SUCCESS;
2200 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2201 struct rte_bbdev_enc_op *ref_op)
2205 struct op_data_entries *hard_data_orig =
2206 &test_vector.entries[DATA_HARD_OUTPUT];
2208 for (i = 0; i < n; ++i) {
2209 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2210 TEST_ASSERT_SUCCESS(ret,
2211 "Checking status and ordering for encoder failed");
2212 TEST_ASSERT_SUCCESS(validate_op_chain(
2213 &ops[i]->ldpc_enc.output,
2215 "Output buffers (CB=%u) are not equal",
2219 return TEST_SUCCESS;
2223 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2226 struct op_data_entries *entry;
2228 op->turbo_dec = test_vector.turbo_dec;
2229 entry = &test_vector.entries[DATA_INPUT];
2230 for (i = 0; i < entry->nb_segments; ++i)
2231 op->turbo_dec.input.length +=
2232 entry->segments[i].length;
2236 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2239 struct op_data_entries *entry;
2241 op->ldpc_dec = test_vector.ldpc_dec;
2242 entry = &test_vector.entries[DATA_INPUT];
2243 for (i = 0; i < entry->nb_segments; ++i)
2244 op->ldpc_dec.input.length +=
2245 entry->segments[i].length;
2246 if (test_vector.ldpc_dec.op_flags &
2247 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2248 entry = &test_vector.entries[DATA_HARQ_INPUT];
2249 for (i = 0; i < entry->nb_segments; ++i)
2250 op->ldpc_dec.harq_combined_input.length +=
2251 entry->segments[i].length;
2257 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2260 struct op_data_entries *entry;
2262 op->turbo_enc = test_vector.turbo_enc;
2263 entry = &test_vector.entries[DATA_INPUT];
2264 for (i = 0; i < entry->nb_segments; ++i)
2265 op->turbo_enc.input.length +=
2266 entry->segments[i].length;
2270 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2273 struct op_data_entries *entry;
2275 op->ldpc_enc = test_vector.ldpc_enc;
2276 entry = &test_vector.entries[DATA_INPUT];
2277 for (i = 0; i < entry->nb_segments; ++i)
2278 op->ldpc_enc.input.length +=
2279 entry->segments[i].length;
2283 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2286 uint32_t c, r, tb_size = 0;
2288 if (op->turbo_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2289 tb_size = op->turbo_dec.tb_params.k_neg;
2291 c = op->turbo_dec.tb_params.c;
2292 r = op->turbo_dec.tb_params.r;
2293 for (i = 0; i < c-r; i++)
2294 tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2295 op->turbo_dec.tb_params.k_neg :
2296 op->turbo_dec.tb_params.k_pos;
2302 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2305 uint32_t c, r, tb_size = 0;
2306 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2308 if (op->ldpc_dec.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2309 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2311 c = op->ldpc_dec.tb_params.c;
2312 r = op->ldpc_dec.tb_params.r;
2313 for (i = 0; i < c-r; i++)
2314 tb_size += sys_cols * op->ldpc_dec.z_c
2315 - op->ldpc_dec.n_filler;
2321 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2324 uint32_t c, r, tb_size = 0;
2326 if (op->turbo_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2327 tb_size = op->turbo_enc.tb_params.k_neg;
2329 c = op->turbo_enc.tb_params.c;
2330 r = op->turbo_enc.tb_params.r;
2331 for (i = 0; i < c-r; i++)
2332 tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2333 op->turbo_enc.tb_params.k_neg :
2334 op->turbo_enc.tb_params.k_pos;
2340 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2343 uint32_t c, r, tb_size = 0;
2344 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2346 if (op->ldpc_enc.code_block_mode == RTE_BBDEV_CODE_BLOCK) {
2347 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2349 c = op->turbo_enc.tb_params.c;
2350 r = op->turbo_enc.tb_params.r;
2351 for (i = 0; i < c-r; i++)
2352 tb_size += sys_cols * op->ldpc_enc.z_c
2353 - op->ldpc_enc.n_filler;
2360 init_test_op_params(struct test_op_params *op_params,
2361 enum rte_bbdev_op_type op_type, const int expected_status,
2362 const int vector_mask, struct rte_mempool *ops_mp,
2363 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2366 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2367 op_type == RTE_BBDEV_OP_LDPC_DEC)
2368 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2369 &op_params->ref_dec_op, 1);
2371 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2372 &op_params->ref_enc_op, 1);
2374 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2376 op_params->mp = ops_mp;
2377 op_params->burst_sz = burst_sz;
2378 op_params->num_to_process = num_to_process;
2379 op_params->num_lcores = num_lcores;
2380 op_params->vector_mask = vector_mask;
2381 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2382 op_type == RTE_BBDEV_OP_LDPC_DEC)
2383 op_params->ref_dec_op->status = expected_status;
2384 else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2385 || op_type == RTE_BBDEV_OP_LDPC_ENC)
2386 op_params->ref_enc_op->status = expected_status;
2391 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2392 struct test_op_params *op_params)
2394 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2396 struct active_device *ad;
2397 unsigned int burst_sz = get_burst_sz();
2398 enum rte_bbdev_op_type op_type = test_vector.op_type;
2399 const struct rte_bbdev_op_cap *capabilities = NULL;
2401 ad = &active_devs[dev_id];
2403 /* Check if device supports op_type */
2404 if (!is_avail_op(ad, test_vector.op_type))
2405 return TEST_SUCCESS;
2407 struct rte_bbdev_info info;
2408 rte_bbdev_info_get(ad->dev_id, &info);
2409 socket_id = GET_SOCKET(info.socket_id);
2411 f_ret = create_mempools(ad, socket_id, op_type,
2413 if (f_ret != TEST_SUCCESS) {
2414 printf("Couldn't create mempools");
2417 if (op_type == RTE_BBDEV_OP_NONE)
2418 op_type = RTE_BBDEV_OP_TURBO_ENC;
2420 f_ret = init_test_op_params(op_params, test_vector.op_type,
2421 test_vector.expected_status,
2427 if (f_ret != TEST_SUCCESS) {
2428 printf("Couldn't init test op params");
2433 /* Find capabilities */
2434 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2435 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2436 if (cap->type == test_vector.op_type) {
2442 TEST_ASSERT_NOT_NULL(capabilities,
2443 "Couldn't find capabilities");
2445 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2446 create_reference_dec_op(op_params->ref_dec_op);
2447 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2448 create_reference_enc_op(op_params->ref_enc_op);
2449 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2450 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2451 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2452 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2454 for (i = 0; i < ad->nb_queues; ++i) {
2455 f_ret = fill_queue_buffers(op_params,
2457 ad->hard_out_mbuf_pool,
2458 ad->soft_out_mbuf_pool,
2459 ad->harq_in_mbuf_pool,
2460 ad->harq_out_mbuf_pool,
2463 info.drv.min_alignment,
2465 if (f_ret != TEST_SUCCESS) {
2466 printf("Couldn't init queue buffers");
2471 /* Run test case function */
2472 t_ret = test_case_func(ad, op_params);
2474 /* Free active device resources and return */
2475 free_buffers(ad, op_params);
2479 free_buffers(ad, op_params);
2483 /* Run given test function per active device per supported op type
2487 run_test_case(test_case_function *test_case_func)
2492 /* Alloc op_params */
2493 struct test_op_params *op_params = rte_zmalloc(NULL,
2494 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2495 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2496 RTE_ALIGN(sizeof(struct test_op_params),
2497 RTE_CACHE_LINE_SIZE));
2499 /* For each device run test case function */
2500 for (dev = 0; dev < nb_active_devs; ++dev)
2501 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2503 rte_free(op_params);
2509 /* Push back the HARQ output from DDR to host */
2511 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2512 struct rte_bbdev_dec_op **ops,
2516 int save_status, ret;
2517 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2518 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2519 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2520 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2521 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2522 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2523 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2524 for (j = 0; j < n; ++j) {
2525 if ((loopback && mem_out) || hc_out) {
2526 save_status = ops[j]->status;
2527 ops[j]->ldpc_dec.op_flags =
2528 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2529 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2531 ops[j]->ldpc_dec.op_flags +=
2532 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2533 ops[j]->ldpc_dec.harq_combined_input.offset =
2535 ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2536 harq_offset += HARQ_INCR;
2538 ops[j]->ldpc_dec.harq_combined_input.length =
2539 ops[j]->ldpc_dec.harq_combined_output.length;
2540 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2544 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2547 ops[j]->ldpc_dec.op_flags = flags;
2548 ops[j]->status = save_status;
2554 * Push back the HARQ output from HW DDR to Host
2555 * Preload HARQ memory input and adjust HARQ offset
2558 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2559 struct rte_bbdev_dec_op **ops, const uint16_t n,
2564 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2565 struct rte_bbdev_op_data save_hc_in[MAX_OPS], save_hc_out[MAX_OPS];
2566 struct rte_bbdev_dec_op *ops_deq[MAX_OPS];
2567 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2568 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2569 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2570 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2571 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2572 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2573 if ((mem_in || hc_in) && preload) {
2574 for (j = 0; j < n; ++j) {
2575 save_hc_in[j] = ops[j]->ldpc_dec.harq_combined_input;
2576 save_hc_out[j] = ops[j]->ldpc_dec.harq_combined_output;
2577 ops[j]->ldpc_dec.op_flags =
2578 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2579 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2581 ops[j]->ldpc_dec.op_flags +=
2582 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2583 ops[j]->ldpc_dec.harq_combined_output.offset =
2585 ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2586 harq_offset += HARQ_INCR;
2588 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, &ops[0], n);
2591 deq += rte_bbdev_dequeue_ldpc_dec_ops(
2592 dev_id, queue_id, &ops_deq[deq],
2594 /* Restore the operations */
2595 for (j = 0; j < n; ++j) {
2596 ops[j]->ldpc_dec.op_flags = flags;
2597 ops[j]->ldpc_dec.harq_combined_input = save_hc_in[j];
2598 ops[j]->ldpc_dec.harq_combined_output = save_hc_out[j];
2601 harq_offset = (uint32_t) queue_id * HARQ_INCR * MAX_OPS;
2602 for (j = 0; j < n; ++j) {
2603 /* Adjust HARQ offset when we reach external DDR */
2604 if (mem_in || hc_in)
2605 ops[j]->ldpc_dec.harq_combined_input.offset
2607 if (mem_out || hc_out)
2608 ops[j]->ldpc_dec.harq_combined_output.offset
2610 harq_offset += HARQ_INCR;
2615 dequeue_event_callback(uint16_t dev_id,
2616 enum rte_bbdev_event_type event, void *cb_arg,
2621 uint64_t total_time;
2622 uint16_t deq, burst_sz, num_ops;
2623 uint16_t queue_id = *(uint16_t *) ret_param;
2624 struct rte_bbdev_info info;
2626 struct thread_params *tp = cb_arg;
2628 /* Find matching thread params using queue_id */
2629 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2630 if (tp->queue_id == queue_id)
2633 if (i == MAX_QUEUES) {
2634 printf("%s: Queue_id from interrupt details was not found!\n",
2639 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2640 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
2642 "Dequeue interrupt handler called for incorrect event!\n");
2646 burst_sz = __atomic_load_n(&tp->burst_sz, __ATOMIC_RELAXED);
2647 num_ops = tp->op_params->num_to_process;
2649 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2650 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2652 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)],
2654 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2655 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2657 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)],
2659 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2660 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2662 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)],
2664 else /*RTE_BBDEV_OP_TURBO_ENC*/
2665 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2667 __atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED)],
2670 if (deq < burst_sz) {
2672 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2674 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
2678 if (__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) + deq < num_ops) {
2679 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED);
2683 total_time = rte_rdtsc_precise() - tp->start_time;
2685 rte_bbdev_info_get(dev_id, &info);
2689 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2690 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2691 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2692 tp->op_params->vector_mask);
2693 /* get the max of iter_count for all dequeued ops */
2694 for (i = 0; i < num_ops; ++i)
2695 tp->iter_count = RTE_MAX(
2696 tp->dec_ops[i]->turbo_dec.iter_count,
2698 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2699 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2700 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2701 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2702 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2703 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2704 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2705 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2706 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2707 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2708 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2709 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2710 tp->op_params->vector_mask);
2711 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2715 printf("Buffers validation failed\n");
2716 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
2719 switch (test_vector.op_type) {
2720 case RTE_BBDEV_OP_TURBO_DEC:
2721 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2723 case RTE_BBDEV_OP_TURBO_ENC:
2724 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2726 case RTE_BBDEV_OP_LDPC_DEC:
2727 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2729 case RTE_BBDEV_OP_LDPC_ENC:
2730 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2732 case RTE_BBDEV_OP_NONE:
2736 printf("Unknown op type: %d\n", test_vector.op_type);
2737 __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
2741 tp->ops_per_sec += ((double)num_ops) /
2742 ((double)total_time / (double)rte_get_tsc_hz());
2743 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2744 ((double)total_time / (double)rte_get_tsc_hz());
2746 __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED);
2750 throughput_intr_lcore_ldpc_dec(void *arg)
2752 struct thread_params *tp = arg;
2753 unsigned int enqueued;
2754 const uint16_t queue_id = tp->queue_id;
2755 const uint16_t burst_sz = tp->op_params->burst_sz;
2756 const uint16_t num_to_process = tp->op_params->num_to_process;
2757 struct rte_bbdev_dec_op *ops[num_to_process];
2758 struct test_buffers *bufs = NULL;
2759 struct rte_bbdev_info info;
2761 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2762 uint16_t num_to_enq, enq;
2764 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2765 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2766 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2767 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2769 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2770 "BURST_SIZE should be <= %u", MAX_BURST);
2772 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2773 "Failed to enable interrupts for dev: %u, queue_id: %u",
2774 tp->dev_id, queue_id);
2776 rte_bbdev_info_get(tp->dev_id, &info);
2778 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2779 "NUM_OPS cannot exceed %u for this device",
2780 info.drv.queue_size_lim);
2782 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2784 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
2785 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2787 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
2789 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2791 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2793 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2794 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2795 bufs->hard_outputs, bufs->soft_outputs,
2796 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2798 /* Set counter to validate the ordering */
2799 for (j = 0; j < num_to_process; ++j)
2800 ops[j]->opaque_data = (void *)(uintptr_t)j;
2802 for (j = 0; j < TEST_REPETITIONS; ++j) {
2803 for (i = 0; i < num_to_process; ++i) {
2806 ops[i]->ldpc_dec.hard_output.data);
2807 if (hc_out || loopback)
2809 ops[i]->ldpc_dec.harq_combined_output.data);
2812 tp->start_time = rte_rdtsc_precise();
2813 for (enqueued = 0; enqueued < num_to_process;) {
2814 num_to_enq = burst_sz;
2816 if (unlikely(num_to_process - enqueued < num_to_enq))
2817 num_to_enq = num_to_process - enqueued;
2821 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2823 queue_id, &ops[enqueued],
2825 } while (unlikely(num_to_enq != enq));
2828 /* Write to thread burst_sz current number of enqueued
2829 * descriptors. It ensures that proper number of
2830 * descriptors will be dequeued in callback
2831 * function - needed for last batch in case where
2832 * the number of operations is not a multiple of
2835 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
2837 /* Wait until processing of previous batch is
2840 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
2842 if (j != TEST_REPETITIONS - 1)
2843 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2846 return TEST_SUCCESS;
2850 throughput_intr_lcore_dec(void *arg)
2852 struct thread_params *tp = arg;
2853 unsigned int enqueued;
2854 const uint16_t queue_id = tp->queue_id;
2855 const uint16_t burst_sz = tp->op_params->burst_sz;
2856 const uint16_t num_to_process = tp->op_params->num_to_process;
2857 struct rte_bbdev_dec_op *ops[num_to_process];
2858 struct test_buffers *bufs = NULL;
2859 struct rte_bbdev_info info;
2861 uint16_t num_to_enq, enq;
2863 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2864 "BURST_SIZE should be <= %u", MAX_BURST);
2866 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2867 "Failed to enable interrupts for dev: %u, queue_id: %u",
2868 tp->dev_id, queue_id);
2870 rte_bbdev_info_get(tp->dev_id, &info);
2872 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2873 "NUM_OPS cannot exceed %u for this device",
2874 info.drv.queue_size_lim);
2876 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2878 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
2879 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2881 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
2883 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2885 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2887 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2888 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2889 bufs->hard_outputs, bufs->soft_outputs,
2890 tp->op_params->ref_dec_op);
2892 /* Set counter to validate the ordering */
2893 for (j = 0; j < num_to_process; ++j)
2894 ops[j]->opaque_data = (void *)(uintptr_t)j;
2896 for (j = 0; j < TEST_REPETITIONS; ++j) {
2897 for (i = 0; i < num_to_process; ++i)
2898 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2900 tp->start_time = rte_rdtsc_precise();
2901 for (enqueued = 0; enqueued < num_to_process;) {
2902 num_to_enq = burst_sz;
2904 if (unlikely(num_to_process - enqueued < num_to_enq))
2905 num_to_enq = num_to_process - enqueued;
2909 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2910 queue_id, &ops[enqueued],
2912 } while (unlikely(num_to_enq != enq));
2915 /* Write to thread burst_sz current number of enqueued
2916 * descriptors. It ensures that proper number of
2917 * descriptors will be dequeued in callback
2918 * function - needed for last batch in case where
2919 * the number of operations is not a multiple of
2922 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
2924 /* Wait until processing of previous batch is
2927 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
2929 if (j != TEST_REPETITIONS - 1)
2930 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2933 return TEST_SUCCESS;
2937 throughput_intr_lcore_enc(void *arg)
2939 struct thread_params *tp = arg;
2940 unsigned int enqueued;
2941 const uint16_t queue_id = tp->queue_id;
2942 const uint16_t burst_sz = tp->op_params->burst_sz;
2943 const uint16_t num_to_process = tp->op_params->num_to_process;
2944 struct rte_bbdev_enc_op *ops[num_to_process];
2945 struct test_buffers *bufs = NULL;
2946 struct rte_bbdev_info info;
2948 uint16_t num_to_enq, enq;
2950 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2951 "BURST_SIZE should be <= %u", MAX_BURST);
2953 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2954 "Failed to enable interrupts for dev: %u, queue_id: %u",
2955 tp->dev_id, queue_id);
2957 rte_bbdev_info_get(tp->dev_id, &info);
2959 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2960 "NUM_OPS cannot exceed %u for this device",
2961 info.drv.queue_size_lim);
2963 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2965 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
2966 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
2968 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
2970 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2972 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2974 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2975 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2976 bufs->hard_outputs, tp->op_params->ref_enc_op);
2978 /* Set counter to validate the ordering */
2979 for (j = 0; j < num_to_process; ++j)
2980 ops[j]->opaque_data = (void *)(uintptr_t)j;
2982 for (j = 0; j < TEST_REPETITIONS; ++j) {
2983 for (i = 0; i < num_to_process; ++i)
2984 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2986 tp->start_time = rte_rdtsc_precise();
2987 for (enqueued = 0; enqueued < num_to_process;) {
2988 num_to_enq = burst_sz;
2990 if (unlikely(num_to_process - enqueued < num_to_enq))
2991 num_to_enq = num_to_process - enqueued;
2995 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2996 queue_id, &ops[enqueued],
2998 } while (unlikely(enq != num_to_enq));
3001 /* Write to thread burst_sz current number of enqueued
3002 * descriptors. It ensures that proper number of
3003 * descriptors will be dequeued in callback
3004 * function - needed for last batch in case where
3005 * the number of operations is not a multiple of
3008 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
3010 /* Wait until processing of previous batch is
3013 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
3015 if (j != TEST_REPETITIONS - 1)
3016 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
3019 return TEST_SUCCESS;
3024 throughput_intr_lcore_ldpc_enc(void *arg)
3026 struct thread_params *tp = arg;
3027 unsigned int enqueued;
3028 const uint16_t queue_id = tp->queue_id;
3029 const uint16_t burst_sz = tp->op_params->burst_sz;
3030 const uint16_t num_to_process = tp->op_params->num_to_process;
3031 struct rte_bbdev_enc_op *ops[num_to_process];
3032 struct test_buffers *bufs = NULL;
3033 struct rte_bbdev_info info;
3035 uint16_t num_to_enq, enq;
3037 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3038 "BURST_SIZE should be <= %u", MAX_BURST);
3040 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
3041 "Failed to enable interrupts for dev: %u, queue_id: %u",
3042 tp->dev_id, queue_id);
3044 rte_bbdev_info_get(tp->dev_id, &info);
3046 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
3047 "NUM_OPS cannot exceed %u for this device",
3048 info.drv.queue_size_lim);
3050 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3052 __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
3053 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
3055 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3057 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
3059 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3061 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3062 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
3063 bufs->inputs, bufs->hard_outputs,
3064 tp->op_params->ref_enc_op);
3066 /* Set counter to validate the ordering */
3067 for (j = 0; j < num_to_process; ++j)
3068 ops[j]->opaque_data = (void *)(uintptr_t)j;
3070 for (j = 0; j < TEST_REPETITIONS; ++j) {
3071 for (i = 0; i < num_to_process; ++i)
3072 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
3074 tp->start_time = rte_rdtsc_precise();
3075 for (enqueued = 0; enqueued < num_to_process;) {
3076 num_to_enq = burst_sz;
3078 if (unlikely(num_to_process - enqueued < num_to_enq))
3079 num_to_enq = num_to_process - enqueued;
3083 enq += rte_bbdev_enqueue_ldpc_enc_ops(
3085 queue_id, &ops[enqueued],
3087 } while (unlikely(enq != num_to_enq));
3090 /* Write to thread burst_sz current number of enqueued
3091 * descriptors. It ensures that proper number of
3092 * descriptors will be dequeued in callback
3093 * function - needed for last batch in case where
3094 * the number of operations is not a multiple of
3097 __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
3099 /* Wait until processing of previous batch is
3102 rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
3104 if (j != TEST_REPETITIONS - 1)
3105 __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
3108 return TEST_SUCCESS;
3112 throughput_pmd_lcore_dec(void *arg)
3114 struct thread_params *tp = arg;
3116 uint64_t total_time = 0, start_time;
3117 const uint16_t queue_id = tp->queue_id;
3118 const uint16_t burst_sz = tp->op_params->burst_sz;
3119 const uint16_t num_ops = tp->op_params->num_to_process;
3120 struct rte_bbdev_dec_op *ops_enq[num_ops];
3121 struct rte_bbdev_dec_op *ops_deq[num_ops];
3122 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3123 struct test_buffers *bufs = NULL;
3125 struct rte_bbdev_info info;
3126 uint16_t num_to_enq;
3128 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3129 "BURST_SIZE should be <= %u", MAX_BURST);
3131 rte_bbdev_info_get(tp->dev_id, &info);
3133 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3134 "NUM_OPS cannot exceed %u for this device",
3135 info.drv.queue_size_lim);
3137 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3139 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3141 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3142 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3144 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3145 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3146 bufs->hard_outputs, bufs->soft_outputs, ref_op);
3148 /* Set counter to validate the ordering */
3149 for (j = 0; j < num_ops; ++j)
3150 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3152 for (i = 0; i < TEST_REPETITIONS; ++i) {
3154 for (j = 0; j < num_ops; ++j)
3155 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
3157 start_time = rte_rdtsc_precise();
3159 for (enq = 0, deq = 0; enq < num_ops;) {
3160 num_to_enq = burst_sz;
3162 if (unlikely(num_ops - enq < num_to_enq))
3163 num_to_enq = num_ops - enq;
3165 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
3166 queue_id, &ops_enq[enq], num_to_enq);
3168 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3169 queue_id, &ops_deq[deq], enq - deq);
3172 /* dequeue the remaining */
3174 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3175 queue_id, &ops_deq[deq], enq - deq);
3178 total_time += rte_rdtsc_precise() - start_time;
3182 /* get the max of iter_count for all dequeued ops */
3183 for (i = 0; i < num_ops; ++i) {
3184 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3188 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3189 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3190 tp->op_params->vector_mask);
3191 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3194 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3196 double tb_len_bits = calc_dec_TB_size(ref_op);
3198 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3199 ((double)total_time / (double)rte_get_tsc_hz());
3200 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3201 1000000.0) / ((double)total_time /
3202 (double)rte_get_tsc_hz());
3204 return TEST_SUCCESS;
3208 bler_pmd_lcore_ldpc_dec(void *arg)
3210 struct thread_params *tp = arg;
3212 uint64_t total_time = 0, start_time;
3213 const uint16_t queue_id = tp->queue_id;
3214 const uint16_t burst_sz = tp->op_params->burst_sz;
3215 const uint16_t num_ops = tp->op_params->num_to_process;
3216 struct rte_bbdev_dec_op *ops_enq[num_ops];
3217 struct rte_bbdev_dec_op *ops_deq[num_ops];
3218 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3219 struct test_buffers *bufs = NULL;
3221 float parity_bler = 0;
3222 struct rte_bbdev_info info;
3223 uint16_t num_to_enq;
3224 bool extDdr = check_bit(ldpc_cap_flags,
3225 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3226 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3227 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3228 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3229 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3231 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3232 "BURST_SIZE should be <= %u", MAX_BURST);
3234 rte_bbdev_info_get(tp->dev_id, &info);
3236 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3237 "NUM_OPS cannot exceed %u for this device",
3238 info.drv.queue_size_lim);
3240 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3242 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3244 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3245 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3247 /* For BLER tests we need to enable early termination */
3248 if (!check_bit(ref_op->ldpc_dec.op_flags,
3249 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3250 ref_op->ldpc_dec.op_flags +=
3251 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3252 ref_op->ldpc_dec.iter_max = get_iter_max();
3253 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3255 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3256 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3257 bufs->hard_outputs, bufs->soft_outputs,
3258 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3259 generate_llr_input(num_ops, bufs->inputs, ref_op);
3261 /* Set counter to validate the ordering */
3262 for (j = 0; j < num_ops; ++j)
3263 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3265 for (i = 0; i < 1; ++i) { /* Could add more iterations */
3266 for (j = 0; j < num_ops; ++j) {
3269 ops_enq[j]->ldpc_dec.hard_output.data);
3270 if (hc_out || loopback)
3272 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3275 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3277 start_time = rte_rdtsc_precise();
3279 for (enq = 0, deq = 0; enq < num_ops;) {
3280 num_to_enq = burst_sz;
3282 if (unlikely(num_ops - enq < num_to_enq))
3283 num_to_enq = num_ops - enq;
3285 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3286 queue_id, &ops_enq[enq], num_to_enq);
3288 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3289 queue_id, &ops_deq[deq], enq - deq);
3292 /* dequeue the remaining */
3294 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3295 queue_id, &ops_deq[deq], enq - deq);
3298 total_time += rte_rdtsc_precise() - start_time;
3302 tp->iter_average = 0;
3303 /* get the max of iter_count for all dequeued ops */
3304 for (i = 0; i < num_ops; ++i) {
3305 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3307 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3308 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3312 parity_bler /= num_ops; /* This one is based on SYND */
3313 tp->iter_average /= num_ops;
3314 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3316 if (test_vector.op_type != RTE_BBDEV_OP_NONE
3320 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3321 tp->op_params->vector_mask);
3322 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3325 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3327 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3328 tp->ops_per_sec = ((double)num_ops * 1) /
3329 ((double)total_time / (double)rte_get_tsc_hz());
3330 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3331 1000000.0) / ((double)total_time /
3332 (double)rte_get_tsc_hz());
3334 return TEST_SUCCESS;
3338 throughput_pmd_lcore_ldpc_dec(void *arg)
3340 struct thread_params *tp = arg;
3342 uint64_t total_time = 0, start_time;
3343 const uint16_t queue_id = tp->queue_id;
3344 const uint16_t burst_sz = tp->op_params->burst_sz;
3345 const uint16_t num_ops = tp->op_params->num_to_process;
3346 struct rte_bbdev_dec_op *ops_enq[num_ops];
3347 struct rte_bbdev_dec_op *ops_deq[num_ops];
3348 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3349 struct test_buffers *bufs = NULL;
3351 struct rte_bbdev_info info;
3352 uint16_t num_to_enq;
3353 bool extDdr = check_bit(ldpc_cap_flags,
3354 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3355 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3356 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3357 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3358 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3360 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3361 "BURST_SIZE should be <= %u", MAX_BURST);
3363 rte_bbdev_info_get(tp->dev_id, &info);
3365 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3366 "NUM_OPS cannot exceed %u for this device",
3367 info.drv.queue_size_lim);
3369 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3371 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3373 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3374 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3376 /* For throughput tests we need to disable early termination */
3377 if (check_bit(ref_op->ldpc_dec.op_flags,
3378 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3379 ref_op->ldpc_dec.op_flags -=
3380 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3381 ref_op->ldpc_dec.iter_max = get_iter_max();
3382 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3384 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3385 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3386 bufs->hard_outputs, bufs->soft_outputs,
3387 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3389 /* Set counter to validate the ordering */
3390 for (j = 0; j < num_ops; ++j)
3391 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3393 for (i = 0; i < TEST_REPETITIONS; ++i) {
3394 for (j = 0; j < num_ops; ++j) {
3397 ops_enq[j]->ldpc_dec.hard_output.data);
3398 if (hc_out || loopback)
3400 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3403 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3405 start_time = rte_rdtsc_precise();
3407 for (enq = 0, deq = 0; enq < num_ops;) {
3408 num_to_enq = burst_sz;
3410 if (unlikely(num_ops - enq < num_to_enq))
3411 num_to_enq = num_ops - enq;
3413 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3414 queue_id, &ops_enq[enq], num_to_enq);
3416 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3417 queue_id, &ops_deq[deq], enq - deq);
3420 /* dequeue the remaining */
3422 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3423 queue_id, &ops_deq[deq], enq - deq);
3426 total_time += rte_rdtsc_precise() - start_time;
3430 /* get the max of iter_count for all dequeued ops */
3431 for (i = 0; i < num_ops; ++i) {
3432 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3436 /* Read loopback is not thread safe */
3437 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3440 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3441 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3442 tp->op_params->vector_mask);
3443 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3446 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3448 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3450 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3451 ((double)total_time / (double)rte_get_tsc_hz());
3452 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3453 1000000.0) / ((double)total_time /
3454 (double)rte_get_tsc_hz());
3456 return TEST_SUCCESS;
3460 throughput_pmd_lcore_enc(void *arg)
3462 struct thread_params *tp = arg;
3464 uint64_t total_time = 0, start_time;
3465 const uint16_t queue_id = tp->queue_id;
3466 const uint16_t burst_sz = tp->op_params->burst_sz;
3467 const uint16_t num_ops = tp->op_params->num_to_process;
3468 struct rte_bbdev_enc_op *ops_enq[num_ops];
3469 struct rte_bbdev_enc_op *ops_deq[num_ops];
3470 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3471 struct test_buffers *bufs = NULL;
3473 struct rte_bbdev_info info;
3474 uint16_t num_to_enq;
3476 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3477 "BURST_SIZE should be <= %u", MAX_BURST);
3479 rte_bbdev_info_get(tp->dev_id, &info);
3481 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3482 "NUM_OPS cannot exceed %u for this device",
3483 info.drv.queue_size_lim);
3485 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3487 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3489 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3491 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3493 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3494 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3495 bufs->hard_outputs, ref_op);
3497 /* Set counter to validate the ordering */
3498 for (j = 0; j < num_ops; ++j)
3499 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3501 for (i = 0; i < TEST_REPETITIONS; ++i) {
3503 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3504 for (j = 0; j < num_ops; ++j)
3505 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3507 start_time = rte_rdtsc_precise();
3509 for (enq = 0, deq = 0; enq < num_ops;) {
3510 num_to_enq = burst_sz;
3512 if (unlikely(num_ops - enq < num_to_enq))
3513 num_to_enq = num_ops - enq;
3515 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3516 queue_id, &ops_enq[enq], num_to_enq);
3518 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3519 queue_id, &ops_deq[deq], enq - deq);
3522 /* dequeue the remaining */
3524 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3525 queue_id, &ops_deq[deq], enq - deq);
3528 total_time += rte_rdtsc_precise() - start_time;
3531 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3532 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3533 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3536 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3538 double tb_len_bits = calc_enc_TB_size(ref_op);
3540 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3541 ((double)total_time / (double)rte_get_tsc_hz());
3542 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3543 / 1000000.0) / ((double)total_time /
3544 (double)rte_get_tsc_hz());
3546 return TEST_SUCCESS;
3550 throughput_pmd_lcore_ldpc_enc(void *arg)
3552 struct thread_params *tp = arg;
3554 uint64_t total_time = 0, start_time;
3555 const uint16_t queue_id = tp->queue_id;
3556 const uint16_t burst_sz = tp->op_params->burst_sz;
3557 const uint16_t num_ops = tp->op_params->num_to_process;
3558 struct rte_bbdev_enc_op *ops_enq[num_ops];
3559 struct rte_bbdev_enc_op *ops_deq[num_ops];
3560 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3561 struct test_buffers *bufs = NULL;
3563 struct rte_bbdev_info info;
3564 uint16_t num_to_enq;
3566 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3567 "BURST_SIZE should be <= %u", MAX_BURST);
3569 rte_bbdev_info_get(tp->dev_id, &info);
3571 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3572 "NUM_OPS cannot exceed %u for this device",
3573 info.drv.queue_size_lim);
3575 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3577 rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3579 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3581 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3583 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3584 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3585 bufs->hard_outputs, ref_op);
3587 /* Set counter to validate the ordering */
3588 for (j = 0; j < num_ops; ++j)
3589 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3591 for (i = 0; i < TEST_REPETITIONS; ++i) {
3593 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3594 for (j = 0; j < num_ops; ++j)
3595 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3597 start_time = rte_rdtsc_precise();
3599 for (enq = 0, deq = 0; enq < num_ops;) {
3600 num_to_enq = burst_sz;
3602 if (unlikely(num_ops - enq < num_to_enq))
3603 num_to_enq = num_ops - enq;
3605 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3606 queue_id, &ops_enq[enq], num_to_enq);
3608 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3609 queue_id, &ops_deq[deq], enq - deq);
3612 /* dequeue the remaining */
3614 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3615 queue_id, &ops_deq[deq], enq - deq);
3618 total_time += rte_rdtsc_precise() - start_time;
3621 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3622 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3623 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3626 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3628 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3630 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3631 ((double)total_time / (double)rte_get_tsc_hz());
3632 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3633 / 1000000.0) / ((double)total_time /
3634 (double)rte_get_tsc_hz());
3636 return TEST_SUCCESS;
3640 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3642 unsigned int iter = 0;
3643 double total_mops = 0, total_mbps = 0;
3645 for (iter = 0; iter < used_cores; iter++) {
3647 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3648 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3649 t_params[iter].mbps);
3650 total_mops += t_params[iter].ops_per_sec;
3651 total_mbps += t_params[iter].mbps;
3654 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3655 used_cores, total_mops, total_mbps);
3658 /* Aggregate the performance results over the number of cores used */
3660 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3662 unsigned int core_idx = 0;
3663 double total_mops = 0, total_mbps = 0;
3664 uint8_t iter_count = 0;
3666 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3668 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3669 t_params[core_idx].lcore_id,
3670 t_params[core_idx].ops_per_sec,
3671 t_params[core_idx].mbps,
3672 t_params[core_idx].iter_count);
3673 total_mops += t_params[core_idx].ops_per_sec;
3674 total_mbps += t_params[core_idx].mbps;
3675 iter_count = RTE_MAX(iter_count,
3676 t_params[core_idx].iter_count);
3679 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3680 used_cores, total_mops, total_mbps, iter_count);
3683 /* Aggregate the performance results over the number of cores used */
3685 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3687 unsigned int core_idx = 0;
3688 double total_mbps = 0, total_bler = 0, total_iter = 0;
3689 double snr = get_snr();
3691 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3692 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3693 t_params[core_idx].lcore_id,
3694 t_params[core_idx].bler * 100,
3695 t_params[core_idx].iter_average,
3696 t_params[core_idx].mbps,
3697 get_vector_filename());
3698 total_mbps += t_params[core_idx].mbps;
3699 total_bler += t_params[core_idx].bler;
3700 total_iter += t_params[core_idx].iter_average;
3702 total_bler /= used_cores;
3703 total_iter /= used_cores;
3705 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3706 snr, total_bler * 100, total_iter, get_iter_max(),
3707 total_mbps, get_vector_filename());
3711 * Test function that determines BLER wireless performance
3714 bler_test(struct active_device *ad,
3715 struct test_op_params *op_params)
3718 unsigned int lcore_id, used_cores = 0;
3719 struct thread_params *t_params;
3720 struct rte_bbdev_info info;
3721 lcore_function_t *bler_function;
3722 uint16_t num_lcores;
3723 const char *op_type_str;
3725 rte_bbdev_info_get(ad->dev_id, &info);
3727 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3728 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3729 test_vector.op_type);
3731 printf("+ ------------------------------------------------------- +\n");
3732 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3733 info.dev_name, ad->nb_queues, op_params->burst_sz,
3734 op_params->num_to_process, op_params->num_lcores,
3736 intr_enabled ? "Interrupt mode" : "PMD mode",
3737 (double)rte_get_tsc_hz() / 1000000000.0);
3739 /* Set number of lcores */
3740 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3742 : op_params->num_lcores;
3744 /* Allocate memory for thread parameters structure */
3745 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3746 RTE_CACHE_LINE_SIZE);
3747 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3748 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3749 RTE_CACHE_LINE_SIZE));
3751 if ((test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) &&
3752 !check_bit(test_vector.ldpc_dec.op_flags,
3753 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
3754 && !check_bit(test_vector.ldpc_dec.op_flags,
3755 RTE_BBDEV_LDPC_LLR_COMPRESSION))
3756 bler_function = bler_pmd_lcore_ldpc_dec;
3758 return TEST_SKIPPED;
3760 __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED);
3762 /* Main core is set at first entry */
3763 t_params[0].dev_id = ad->dev_id;
3764 t_params[0].lcore_id = rte_lcore_id();
3765 t_params[0].op_params = op_params;
3766 t_params[0].queue_id = ad->queue_ids[used_cores++];
3767 t_params[0].iter_count = 0;
3769 RTE_LCORE_FOREACH_WORKER(lcore_id) {
3770 if (used_cores >= num_lcores)
3773 t_params[used_cores].dev_id = ad->dev_id;
3774 t_params[used_cores].lcore_id = lcore_id;
3775 t_params[used_cores].op_params = op_params;
3776 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3777 t_params[used_cores].iter_count = 0;
3779 rte_eal_remote_launch(bler_function,
3780 &t_params[used_cores++], lcore_id);
3783 __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3784 ret = bler_function(&t_params[0]);
3786 /* Main core is always used */
3787 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3788 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3790 print_dec_bler(t_params, num_lcores);
3792 /* Return if test failed */
3798 /* Function to print something here*/
3804 * Test function that determines how long an enqueue + dequeue of a burst
3805 * takes on available lcores.
3808 throughput_test(struct active_device *ad,
3809 struct test_op_params *op_params)
3812 unsigned int lcore_id, used_cores = 0;
3813 struct thread_params *t_params, *tp;
3814 struct rte_bbdev_info info;
3815 lcore_function_t *throughput_function;
3816 uint16_t num_lcores;
3817 const char *op_type_str;
3819 rte_bbdev_info_get(ad->dev_id, &info);
3821 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3822 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3823 test_vector.op_type);
3825 printf("+ ------------------------------------------------------- +\n");
3826 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3827 info.dev_name, ad->nb_queues, op_params->burst_sz,
3828 op_params->num_to_process, op_params->num_lcores,
3830 intr_enabled ? "Interrupt mode" : "PMD mode",
3831 (double)rte_get_tsc_hz() / 1000000000.0);
3833 /* Set number of lcores */
3834 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3836 : op_params->num_lcores;
3838 /* Allocate memory for thread parameters structure */
3839 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3840 RTE_CACHE_LINE_SIZE);
3841 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3842 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3843 RTE_CACHE_LINE_SIZE));
3846 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3847 throughput_function = throughput_intr_lcore_dec;
3848 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3849 throughput_function = throughput_intr_lcore_ldpc_dec;
3850 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3851 throughput_function = throughput_intr_lcore_enc;
3852 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3853 throughput_function = throughput_intr_lcore_ldpc_enc;
3855 throughput_function = throughput_intr_lcore_enc;
3857 /* Dequeue interrupt callback registration */
3858 ret = rte_bbdev_callback_register(ad->dev_id,
3859 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3866 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3867 throughput_function = throughput_pmd_lcore_dec;
3868 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3869 throughput_function = throughput_pmd_lcore_ldpc_dec;
3870 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3871 throughput_function = throughput_pmd_lcore_enc;
3872 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3873 throughput_function = throughput_pmd_lcore_ldpc_enc;
3875 throughput_function = throughput_pmd_lcore_enc;
3878 __atomic_store_n(&op_params->sync, SYNC_WAIT, __ATOMIC_RELAXED);
3880 /* Main core is set at first entry */
3881 t_params[0].dev_id = ad->dev_id;
3882 t_params[0].lcore_id = rte_lcore_id();
3883 t_params[0].op_params = op_params;
3884 t_params[0].queue_id = ad->queue_ids[used_cores++];
3885 t_params[0].iter_count = 0;
3887 RTE_LCORE_FOREACH_WORKER(lcore_id) {
3888 if (used_cores >= num_lcores)
3891 t_params[used_cores].dev_id = ad->dev_id;
3892 t_params[used_cores].lcore_id = lcore_id;
3893 t_params[used_cores].op_params = op_params;
3894 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3895 t_params[used_cores].iter_count = 0;
3897 rte_eal_remote_launch(throughput_function,
3898 &t_params[used_cores++], lcore_id);
3901 __atomic_store_n(&op_params->sync, SYNC_START, __ATOMIC_RELAXED);
3902 ret = throughput_function(&t_params[0]);
3904 /* Main core is always used */
3905 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3906 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3908 /* Return if test failed */
3914 /* Print throughput if interrupts are disabled and test passed */
3915 if (!intr_enabled) {
3916 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3917 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3918 print_dec_throughput(t_params, num_lcores);
3920 print_enc_throughput(t_params, num_lcores);
3925 /* In interrupt TC we need to wait for the interrupt callback to deqeue
3926 * all pending operations. Skip waiting for queues which reported an
3927 * error using processing_status variable.
3928 * Wait for main lcore operations.
3931 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) <
3932 op_params->num_to_process) &&
3933 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) !=
3937 tp->ops_per_sec /= TEST_REPETITIONS;
3938 tp->mbps /= TEST_REPETITIONS;
3939 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED);
3941 /* Wait for worker lcores operations */
3942 for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3943 tp = &t_params[used_cores];
3945 while ((__atomic_load_n(&tp->nb_dequeued, __ATOMIC_RELAXED) <
3946 op_params->num_to_process) &&
3947 (__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED) !=
3951 tp->ops_per_sec /= TEST_REPETITIONS;
3952 tp->mbps /= TEST_REPETITIONS;
3953 ret |= (int)__atomic_load_n(&tp->processing_status, __ATOMIC_RELAXED);
3956 /* Print throughput if test passed */
3958 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3959 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3960 print_dec_throughput(t_params, num_lcores);
3961 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3962 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3963 print_enc_throughput(t_params, num_lcores);
3971 latency_test_dec(struct rte_mempool *mempool,
3972 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3973 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3974 const uint16_t num_to_process, uint16_t burst_sz,
3975 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3977 int ret = TEST_SUCCESS;
3978 uint16_t i, j, dequeued;
3979 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3980 uint64_t start_time = 0, last_time = 0;
3982 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3983 uint16_t enq = 0, deq = 0;
3984 bool first_time = true;
3987 if (unlikely(num_to_process - dequeued < burst_sz))
3988 burst_sz = num_to_process - dequeued;
3990 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3991 TEST_ASSERT_SUCCESS(ret,
3992 "rte_bbdev_dec_op_alloc_bulk() failed");
3993 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3994 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4000 /* Set counter to validate the ordering */
4001 for (j = 0; j < burst_sz; ++j)
4002 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4004 start_time = rte_rdtsc_precise();
4006 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
4008 TEST_ASSERT(enq == burst_sz,
4009 "Error enqueueing burst, expected %u, got %u",
4014 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4015 &ops_deq[deq], burst_sz - deq);
4016 if (likely(first_time && (deq > 0))) {
4017 last_time = rte_rdtsc_precise() - start_time;
4020 } while (unlikely(burst_sz != deq));
4022 *max_time = RTE_MAX(*max_time, last_time);
4023 *min_time = RTE_MIN(*min_time, last_time);
4024 *total_time += last_time;
4026 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4027 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
4029 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4032 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4039 /* Test case for latency/validation for LDPC Decoder */
4041 latency_test_ldpc_dec(struct rte_mempool *mempool,
4042 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
4043 int vector_mask, uint16_t dev_id, uint16_t queue_id,
4044 const uint16_t num_to_process, uint16_t burst_sz,
4045 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time,
4048 int ret = TEST_SUCCESS;
4049 uint16_t i, j, dequeued;
4050 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4051 uint64_t start_time = 0, last_time = 0;
4052 bool extDdr = ldpc_cap_flags &
4053 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4055 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4056 uint16_t enq = 0, deq = 0;
4057 bool first_time = true;
4060 if (unlikely(num_to_process - dequeued < burst_sz))
4061 burst_sz = num_to_process - dequeued;
4063 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4064 TEST_ASSERT_SUCCESS(ret,
4065 "rte_bbdev_dec_op_alloc_bulk() failed");
4067 /* For latency tests we need to disable early termination */
4068 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags,
4069 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
4070 ref_op->ldpc_dec.op_flags -=
4071 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
4072 ref_op->ldpc_dec.iter_max = get_iter_max();
4073 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
4075 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4076 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4085 preload_harq_ddr(dev_id, queue_id, ops_enq,
4088 /* Set counter to validate the ordering */
4089 for (j = 0; j < burst_sz; ++j)
4090 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4092 start_time = rte_rdtsc_precise();
4094 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4095 &ops_enq[enq], burst_sz);
4096 TEST_ASSERT(enq == burst_sz,
4097 "Error enqueueing burst, expected %u, got %u",
4102 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4103 &ops_deq[deq], burst_sz - deq);
4104 if (likely(first_time && (deq > 0))) {
4105 last_time = rte_rdtsc_precise() - start_time;
4108 } while (unlikely(burst_sz != deq));
4110 *max_time = RTE_MAX(*max_time, last_time);
4111 *min_time = RTE_MIN(*min_time, last_time);
4112 *total_time += last_time;
4115 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4117 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4118 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
4120 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4123 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4130 latency_test_enc(struct rte_mempool *mempool,
4131 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4132 uint16_t dev_id, uint16_t queue_id,
4133 const uint16_t num_to_process, uint16_t burst_sz,
4134 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4136 int ret = TEST_SUCCESS;
4137 uint16_t i, j, dequeued;
4138 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4139 uint64_t start_time = 0, last_time = 0;
4141 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4142 uint16_t enq = 0, deq = 0;
4143 bool first_time = true;
4146 if (unlikely(num_to_process - dequeued < burst_sz))
4147 burst_sz = num_to_process - dequeued;
4149 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4150 TEST_ASSERT_SUCCESS(ret,
4151 "rte_bbdev_enc_op_alloc_bulk() failed");
4152 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4153 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4158 /* Set counter to validate the ordering */
4159 for (j = 0; j < burst_sz; ++j)
4160 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4162 start_time = rte_rdtsc_precise();
4164 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
4166 TEST_ASSERT(enq == burst_sz,
4167 "Error enqueueing burst, expected %u, got %u",
4172 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4173 &ops_deq[deq], burst_sz - deq);
4174 if (likely(first_time && (deq > 0))) {
4175 last_time += rte_rdtsc_precise() - start_time;
4178 } while (unlikely(burst_sz != deq));
4180 *max_time = RTE_MAX(*max_time, last_time);
4181 *min_time = RTE_MIN(*min_time, last_time);
4182 *total_time += last_time;
4184 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4185 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4186 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4189 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4197 latency_test_ldpc_enc(struct rte_mempool *mempool,
4198 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4199 uint16_t dev_id, uint16_t queue_id,
4200 const uint16_t num_to_process, uint16_t burst_sz,
4201 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4203 int ret = TEST_SUCCESS;
4204 uint16_t i, j, dequeued;
4205 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4206 uint64_t start_time = 0, last_time = 0;
4208 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4209 uint16_t enq = 0, deq = 0;
4210 bool first_time = true;
4213 if (unlikely(num_to_process - dequeued < burst_sz))
4214 burst_sz = num_to_process - dequeued;
4216 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4217 TEST_ASSERT_SUCCESS(ret,
4218 "rte_bbdev_enc_op_alloc_bulk() failed");
4219 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4220 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4225 /* Set counter to validate the ordering */
4226 for (j = 0; j < burst_sz; ++j)
4227 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4229 start_time = rte_rdtsc_precise();
4231 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4232 &ops_enq[enq], burst_sz);
4233 TEST_ASSERT(enq == burst_sz,
4234 "Error enqueueing burst, expected %u, got %u",
4239 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4240 &ops_deq[deq], burst_sz - deq);
4241 if (likely(first_time && (deq > 0))) {
4242 last_time += rte_rdtsc_precise() - start_time;
4245 } while (unlikely(burst_sz != deq));
4247 *max_time = RTE_MAX(*max_time, last_time);
4248 *min_time = RTE_MIN(*min_time, last_time);
4249 *total_time += last_time;
4251 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4252 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4253 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4256 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4263 /* Common function for running validation and latency test cases */
4265 validation_latency_test(struct active_device *ad,
4266 struct test_op_params *op_params, bool latency_flag)
4269 uint16_t burst_sz = op_params->burst_sz;
4270 const uint16_t num_to_process = op_params->num_to_process;
4271 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4272 const uint16_t queue_id = ad->queue_ids[0];
4273 struct test_buffers *bufs = NULL;
4274 struct rte_bbdev_info info;
4275 uint64_t total_time, min_time, max_time;
4276 const char *op_type_str;
4278 total_time = max_time = 0;
4279 min_time = UINT64_MAX;
4281 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4282 "BURST_SIZE should be <= %u", MAX_BURST);
4284 rte_bbdev_info_get(ad->dev_id, &info);
4285 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4287 op_type_str = rte_bbdev_op_type_str(op_type);
4288 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4290 printf("+ ------------------------------------------------------- +\n");
4292 printf("== test: latency\ndev:");
4294 printf("== test: validation\ndev:");
4295 printf("%s, burst size: %u, num ops: %u, op type: %s\n",
4296 info.dev_name, burst_sz, num_to_process, op_type_str);
4298 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4299 iter = latency_test_dec(op_params->mp, bufs,
4300 op_params->ref_dec_op, op_params->vector_mask,
4301 ad->dev_id, queue_id, num_to_process,
4302 burst_sz, &total_time, &min_time, &max_time);
4303 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4304 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4305 op_params->ref_enc_op, ad->dev_id, queue_id,
4306 num_to_process, burst_sz, &total_time,
4307 &min_time, &max_time);
4308 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4309 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4310 op_params->ref_dec_op, op_params->vector_mask,
4311 ad->dev_id, queue_id, num_to_process,
4312 burst_sz, &total_time, &min_time, &max_time,
4314 else /* RTE_BBDEV_OP_TURBO_ENC */
4315 iter = latency_test_enc(op_params->mp, bufs,
4316 op_params->ref_enc_op,
4317 ad->dev_id, queue_id,
4318 num_to_process, burst_sz, &total_time,
4319 &min_time, &max_time);
4324 printf("Operation latency:\n"
4325 "\tavg: %lg cycles, %lg us\n"
4326 "\tmin: %lg cycles, %lg us\n"
4327 "\tmax: %lg cycles, %lg us\n",
4328 (double)total_time / (double)iter,
4329 (double)(total_time * 1000000) / (double)iter /
4330 (double)rte_get_tsc_hz(), (double)min_time,
4331 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4332 (double)max_time, (double)(max_time * 1000000) /
4333 (double)rte_get_tsc_hz());
4335 return TEST_SUCCESS;
4339 latency_test(struct active_device *ad, struct test_op_params *op_params)
4341 return validation_latency_test(ad, op_params, true);
4345 validation_test(struct active_device *ad, struct test_op_params *op_params)
4347 return validation_latency_test(ad, op_params, false);
4350 #ifdef RTE_BBDEV_OFFLOAD_COST
4352 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4353 struct rte_bbdev_stats *stats)
4355 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4356 struct rte_bbdev_stats *q_stats;
4358 if (queue_id >= dev->data->num_queues)
4361 q_stats = &dev->data->queues[queue_id].queue_stats;
4363 stats->enqueued_count = q_stats->enqueued_count;
4364 stats->dequeued_count = q_stats->dequeued_count;
4365 stats->enqueue_err_count = q_stats->enqueue_err_count;
4366 stats->dequeue_err_count = q_stats->dequeue_err_count;
4367 stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4373 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4374 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4375 uint16_t queue_id, const uint16_t num_to_process,
4376 uint16_t burst_sz, struct test_time_stats *time_st)
4378 int i, dequeued, ret;
4379 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4380 uint64_t enq_start_time, deq_start_time;
4381 uint64_t enq_sw_last_time, deq_last_time;
4382 struct rte_bbdev_stats stats;
4384 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4385 uint16_t enq = 0, deq = 0;
4387 if (unlikely(num_to_process - dequeued < burst_sz))
4388 burst_sz = num_to_process - dequeued;
4390 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4391 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4392 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4398 /* Start time meas for enqueue function offload latency */
4399 enq_start_time = rte_rdtsc_precise();
4401 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4402 &ops_enq[enq], burst_sz - enq);
4403 } while (unlikely(burst_sz != enq));
4405 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4406 TEST_ASSERT_SUCCESS(ret,
4407 "Failed to get stats for queue (%u) of device (%u)",
4410 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4411 stats.acc_offload_cycles;
4412 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4414 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4416 time_st->enq_sw_total_time += enq_sw_last_time;
4418 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4419 stats.acc_offload_cycles);
4420 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4421 stats.acc_offload_cycles);
4422 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4424 /* give time for device to process ops */
4425 rte_delay_us(WAIT_OFFLOAD_US);
4427 /* Start time meas for dequeue function offload latency */
4428 deq_start_time = rte_rdtsc_precise();
4429 /* Dequeue one operation */
4431 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4432 &ops_deq[deq], enq);
4433 } while (unlikely(deq == 0));
4435 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4436 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4438 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4440 time_st->deq_total_time += deq_last_time;
4442 /* Dequeue remaining operations if needed*/
4443 while (burst_sz != deq)
4444 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4445 &ops_deq[deq], burst_sz - deq);
4447 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4455 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4456 struct test_buffers *bufs,
4457 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4458 uint16_t queue_id, const uint16_t num_to_process,
4459 uint16_t burst_sz, struct test_time_stats *time_st)
4461 int i, dequeued, ret;
4462 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4463 uint64_t enq_start_time, deq_start_time;
4464 uint64_t enq_sw_last_time, deq_last_time;
4465 struct rte_bbdev_stats stats;
4466 bool extDdr = ldpc_cap_flags &
4467 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4469 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4470 uint16_t enq = 0, deq = 0;
4472 if (unlikely(num_to_process - dequeued < burst_sz))
4473 burst_sz = num_to_process - dequeued;
4475 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4476 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4477 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4486 preload_harq_ddr(dev_id, queue_id, ops_enq,
4489 /* Start time meas for enqueue function offload latency */
4490 enq_start_time = rte_rdtsc_precise();
4492 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4493 &ops_enq[enq], burst_sz - enq);
4494 } while (unlikely(burst_sz != enq));
4496 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4497 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4498 TEST_ASSERT_SUCCESS(ret,
4499 "Failed to get stats for queue (%u) of device (%u)",
4502 enq_sw_last_time -= stats.acc_offload_cycles;
4503 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4505 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4507 time_st->enq_sw_total_time += enq_sw_last_time;
4509 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4510 stats.acc_offload_cycles);
4511 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4512 stats.acc_offload_cycles);
4513 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4515 /* give time for device to process ops */
4516 rte_delay_us(WAIT_OFFLOAD_US);
4518 /* Start time meas for dequeue function offload latency */
4519 deq_start_time = rte_rdtsc_precise();
4520 /* Dequeue one operation */
4522 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4523 &ops_deq[deq], enq);
4524 } while (unlikely(deq == 0));
4526 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4527 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4529 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4531 time_st->deq_total_time += deq_last_time;
4533 /* Dequeue remaining operations if needed*/
4534 while (burst_sz != deq)
4535 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4536 &ops_deq[deq], burst_sz - deq);
4539 /* Read loopback is not thread safe */
4540 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4543 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4551 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4552 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4553 uint16_t queue_id, const uint16_t num_to_process,
4554 uint16_t burst_sz, struct test_time_stats *time_st)
4556 int i, dequeued, ret;
4557 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4558 uint64_t enq_start_time, deq_start_time;
4559 uint64_t enq_sw_last_time, deq_last_time;
4560 struct rte_bbdev_stats stats;
4562 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4563 uint16_t enq = 0, deq = 0;
4565 if (unlikely(num_to_process - dequeued < burst_sz))
4566 burst_sz = num_to_process - dequeued;
4568 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4569 TEST_ASSERT_SUCCESS(ret,
4570 "rte_bbdev_enc_op_alloc_bulk() failed");
4571 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4572 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4577 /* Start time meas for enqueue function offload latency */
4578 enq_start_time = rte_rdtsc_precise();
4580 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4581 &ops_enq[enq], burst_sz - enq);
4582 } while (unlikely(burst_sz != enq));
4584 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4586 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4587 TEST_ASSERT_SUCCESS(ret,
4588 "Failed to get stats for queue (%u) of device (%u)",
4590 enq_sw_last_time -= stats.acc_offload_cycles;
4591 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4593 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4595 time_st->enq_sw_total_time += enq_sw_last_time;
4597 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4598 stats.acc_offload_cycles);
4599 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4600 stats.acc_offload_cycles);
4601 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4603 /* give time for device to process ops */
4604 rte_delay_us(WAIT_OFFLOAD_US);
4606 /* Start time meas for dequeue function offload latency */
4607 deq_start_time = rte_rdtsc_precise();
4608 /* Dequeue one operation */
4610 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4611 &ops_deq[deq], enq);
4612 } while (unlikely(deq == 0));
4614 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4615 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4617 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4619 time_st->deq_total_time += deq_last_time;
4621 while (burst_sz != deq)
4622 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4623 &ops_deq[deq], burst_sz - deq);
4625 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4633 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4634 struct test_buffers *bufs,
4635 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4636 uint16_t queue_id, const uint16_t num_to_process,
4637 uint16_t burst_sz, struct test_time_stats *time_st)
4639 int i, dequeued, ret;
4640 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4641 uint64_t enq_start_time, deq_start_time;
4642 uint64_t enq_sw_last_time, deq_last_time;
4643 struct rte_bbdev_stats stats;
4645 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4646 uint16_t enq = 0, deq = 0;
4648 if (unlikely(num_to_process - dequeued < burst_sz))
4649 burst_sz = num_to_process - dequeued;
4651 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4652 TEST_ASSERT_SUCCESS(ret,
4653 "rte_bbdev_enc_op_alloc_bulk() failed");
4654 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4655 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4660 /* Start time meas for enqueue function offload latency */
4661 enq_start_time = rte_rdtsc_precise();
4663 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4664 &ops_enq[enq], burst_sz - enq);
4665 } while (unlikely(burst_sz != enq));
4667 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4668 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4669 TEST_ASSERT_SUCCESS(ret,
4670 "Failed to get stats for queue (%u) of device (%u)",
4673 enq_sw_last_time -= stats.acc_offload_cycles;
4674 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4676 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4678 time_st->enq_sw_total_time += enq_sw_last_time;
4680 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4681 stats.acc_offload_cycles);
4682 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4683 stats.acc_offload_cycles);
4684 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4686 /* give time for device to process ops */
4687 rte_delay_us(WAIT_OFFLOAD_US);
4689 /* Start time meas for dequeue function offload latency */
4690 deq_start_time = rte_rdtsc_precise();
4691 /* Dequeue one operation */
4693 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4694 &ops_deq[deq], enq);
4695 } while (unlikely(deq == 0));
4697 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4698 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4700 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4702 time_st->deq_total_time += deq_last_time;
4704 while (burst_sz != deq)
4705 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4706 &ops_deq[deq], burst_sz - deq);
4708 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4717 offload_cost_test(struct active_device *ad,
4718 struct test_op_params *op_params)
4720 #ifndef RTE_BBDEV_OFFLOAD_COST
4722 RTE_SET_USED(op_params);
4723 printf("Offload latency test is disabled.\n");
4724 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4725 return TEST_SKIPPED;
4728 uint16_t burst_sz = op_params->burst_sz;
4729 const uint16_t num_to_process = op_params->num_to_process;
4730 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4731 const uint16_t queue_id = ad->queue_ids[0];
4732 struct test_buffers *bufs = NULL;
4733 struct rte_bbdev_info info;
4734 const char *op_type_str;
4735 struct test_time_stats time_st;
4737 memset(&time_st, 0, sizeof(struct test_time_stats));
4738 time_st.enq_sw_min_time = UINT64_MAX;
4739 time_st.enq_acc_min_time = UINT64_MAX;
4740 time_st.deq_min_time = UINT64_MAX;
4742 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4743 "BURST_SIZE should be <= %u", MAX_BURST);
4745 rte_bbdev_info_get(ad->dev_id, &info);
4746 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4748 op_type_str = rte_bbdev_op_type_str(op_type);
4749 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4751 printf("+ ------------------------------------------------------- +\n");
4752 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4753 info.dev_name, burst_sz, num_to_process, op_type_str);
4755 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4756 iter = offload_latency_test_dec(op_params->mp, bufs,
4757 op_params->ref_dec_op, ad->dev_id, queue_id,
4758 num_to_process, burst_sz, &time_st);
4759 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4760 iter = offload_latency_test_enc(op_params->mp, bufs,
4761 op_params->ref_enc_op, ad->dev_id, queue_id,
4762 num_to_process, burst_sz, &time_st);
4763 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4764 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4765 op_params->ref_enc_op, ad->dev_id, queue_id,
4766 num_to_process, burst_sz, &time_st);
4767 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4768 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4769 op_params->ref_dec_op, ad->dev_id, queue_id,
4770 num_to_process, burst_sz, &time_st);
4772 iter = offload_latency_test_enc(op_params->mp, bufs,
4773 op_params->ref_enc_op, ad->dev_id, queue_id,
4774 num_to_process, burst_sz, &time_st);
4779 printf("Enqueue driver offload cost latency:\n"
4780 "\tavg: %lg cycles, %lg us\n"
4781 "\tmin: %lg cycles, %lg us\n"
4782 "\tmax: %lg cycles, %lg us\n"
4783 "Enqueue accelerator offload cost latency:\n"
4784 "\tavg: %lg cycles, %lg us\n"
4785 "\tmin: %lg cycles, %lg us\n"
4786 "\tmax: %lg cycles, %lg us\n",
4787 (double)time_st.enq_sw_total_time / (double)iter,
4788 (double)(time_st.enq_sw_total_time * 1000000) /
4789 (double)iter / (double)rte_get_tsc_hz(),
4790 (double)time_st.enq_sw_min_time,
4791 (double)(time_st.enq_sw_min_time * 1000000) /
4792 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4793 (double)(time_st.enq_sw_max_time * 1000000) /
4794 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4796 (double)(time_st.enq_acc_total_time * 1000000) /
4797 (double)iter / (double)rte_get_tsc_hz(),
4798 (double)time_st.enq_acc_min_time,
4799 (double)(time_st.enq_acc_min_time * 1000000) /
4800 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4801 (double)(time_st.enq_acc_max_time * 1000000) /
4804 printf("Dequeue offload cost latency - one op:\n"
4805 "\tavg: %lg cycles, %lg us\n"
4806 "\tmin: %lg cycles, %lg us\n"
4807 "\tmax: %lg cycles, %lg us\n",
4808 (double)time_st.deq_total_time / (double)iter,
4809 (double)(time_st.deq_total_time * 1000000) /
4810 (double)iter / (double)rte_get_tsc_hz(),
4811 (double)time_st.deq_min_time,
4812 (double)(time_st.deq_min_time * 1000000) /
4813 rte_get_tsc_hz(), (double)time_st.deq_max_time,
4814 (double)(time_st.deq_max_time * 1000000) /
4817 struct rte_bbdev_stats stats = {0};
4818 get_bbdev_queue_stats(ad->dev_id, queue_id, &stats);
4819 if (op_type != RTE_BBDEV_OP_LDPC_DEC) {
4820 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process,
4821 "Mismatch in enqueue count %10"PRIu64" %d",
4822 stats.enqueued_count, num_to_process);
4823 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process,
4824 "Mismatch in dequeue count %10"PRIu64" %d",
4825 stats.dequeued_count, num_to_process);
4827 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0,
4828 "Enqueue count Error %10"PRIu64"",
4829 stats.enqueue_err_count);
4830 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0,
4831 "Dequeue count Error (%10"PRIu64"",
4832 stats.dequeue_err_count);
4834 return TEST_SUCCESS;
4838 #ifdef RTE_BBDEV_OFFLOAD_COST
4840 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4841 const uint16_t num_to_process, uint16_t burst_sz,
4842 uint64_t *deq_total_time, uint64_t *deq_min_time,
4843 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4846 struct rte_bbdev_dec_op *ops[MAX_BURST];
4847 uint64_t deq_start_time, deq_last_time;
4849 /* Test deq offload latency from an empty queue */
4851 for (i = 0, deq_total = 0; deq_total < num_to_process;
4852 ++i, deq_total += burst_sz) {
4853 deq_start_time = rte_rdtsc_precise();
4855 if (unlikely(num_to_process - deq_total < burst_sz))
4856 burst_sz = num_to_process - deq_total;
4857 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4858 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4861 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4864 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4865 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4866 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4867 *deq_total_time += deq_last_time;
4874 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4875 const uint16_t num_to_process, uint16_t burst_sz,
4876 uint64_t *deq_total_time, uint64_t *deq_min_time,
4877 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4880 struct rte_bbdev_enc_op *ops[MAX_BURST];
4881 uint64_t deq_start_time, deq_last_time;
4883 /* Test deq offload latency from an empty queue */
4884 for (i = 0, deq_total = 0; deq_total < num_to_process;
4885 ++i, deq_total += burst_sz) {
4886 deq_start_time = rte_rdtsc_precise();
4888 if (unlikely(num_to_process - deq_total < burst_sz))
4889 burst_sz = num_to_process - deq_total;
4890 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4891 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4894 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4897 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4898 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4899 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4900 *deq_total_time += deq_last_time;
4909 offload_latency_empty_q_test(struct active_device *ad,
4910 struct test_op_params *op_params)
4912 #ifndef RTE_BBDEV_OFFLOAD_COST
4914 RTE_SET_USED(op_params);
4915 printf("Offload latency empty dequeue test is disabled.\n");
4916 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4917 return TEST_SKIPPED;
4920 uint64_t deq_total_time, deq_min_time, deq_max_time;
4921 uint16_t burst_sz = op_params->burst_sz;
4922 const uint16_t num_to_process = op_params->num_to_process;
4923 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4924 const uint16_t queue_id = ad->queue_ids[0];
4925 struct rte_bbdev_info info;
4926 const char *op_type_str;
4928 deq_total_time = deq_max_time = 0;
4929 deq_min_time = UINT64_MAX;
4931 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4932 "BURST_SIZE should be <= %u", MAX_BURST);
4934 rte_bbdev_info_get(ad->dev_id, &info);
4936 op_type_str = rte_bbdev_op_type_str(op_type);
4937 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4939 printf("+ ------------------------------------------------------- +\n");
4940 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4941 info.dev_name, burst_sz, num_to_process, op_type_str);
4943 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4944 op_type == RTE_BBDEV_OP_LDPC_DEC)
4945 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4946 num_to_process, burst_sz, &deq_total_time,
4947 &deq_min_time, &deq_max_time, op_type);
4949 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4950 num_to_process, burst_sz, &deq_total_time,
4951 &deq_min_time, &deq_max_time, op_type);
4956 printf("Empty dequeue offload:\n"
4957 "\tavg: %lg cycles, %lg us\n"
4958 "\tmin: %lg cycles, %lg us\n"
4959 "\tmax: %lg cycles, %lg us\n",
4960 (double)deq_total_time / (double)iter,
4961 (double)(deq_total_time * 1000000) / (double)iter /
4962 (double)rte_get_tsc_hz(), (double)deq_min_time,
4963 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4964 (double)deq_max_time, (double)(deq_max_time * 1000000) /
4967 return TEST_SUCCESS;
4974 return run_test_case(bler_test);
4980 return run_test_case(throughput_test);
4984 offload_cost_tc(void)
4986 return run_test_case(offload_cost_test);
4990 offload_latency_empty_q_tc(void)
4992 return run_test_case(offload_latency_empty_q_test);
4998 return run_test_case(latency_test);
5004 return run_test_case(validation_test);
5010 return run_test_case(throughput_test);
5013 static struct unit_test_suite bbdev_bler_testsuite = {
5014 .suite_name = "BBdev BLER Tests",
5015 .setup = testsuite_setup,
5016 .teardown = testsuite_teardown,
5017 .unit_test_cases = {
5018 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
5019 TEST_CASES_END() /**< NULL terminate unit test array */
5023 static struct unit_test_suite bbdev_throughput_testsuite = {
5024 .suite_name = "BBdev Throughput Tests",
5025 .setup = testsuite_setup,
5026 .teardown = testsuite_teardown,
5027 .unit_test_cases = {
5028 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
5029 TEST_CASES_END() /**< NULL terminate unit test array */
5033 static struct unit_test_suite bbdev_validation_testsuite = {
5034 .suite_name = "BBdev Validation Tests",
5035 .setup = testsuite_setup,
5036 .teardown = testsuite_teardown,
5037 .unit_test_cases = {
5038 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc),
5039 TEST_CASES_END() /**< NULL terminate unit test array */
5043 static struct unit_test_suite bbdev_latency_testsuite = {
5044 .suite_name = "BBdev Latency Tests",
5045 .setup = testsuite_setup,
5046 .teardown = testsuite_teardown,
5047 .unit_test_cases = {
5048 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
5049 TEST_CASES_END() /**< NULL terminate unit test array */
5053 static struct unit_test_suite bbdev_offload_cost_testsuite = {
5054 .suite_name = "BBdev Offload Cost Tests",
5055 .setup = testsuite_setup,
5056 .teardown = testsuite_teardown,
5057 .unit_test_cases = {
5058 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
5059 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
5060 TEST_CASES_END() /**< NULL terminate unit test array */
5064 static struct unit_test_suite bbdev_interrupt_testsuite = {
5065 .suite_name = "BBdev Interrupt Tests",
5066 .setup = interrupt_testsuite_setup,
5067 .teardown = testsuite_teardown,
5068 .unit_test_cases = {
5069 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
5070 TEST_CASES_END() /**< NULL terminate unit test array */
5074 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
5075 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
5076 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
5077 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
5078 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
5079 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);