1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
19 #include <rte_interrupts.h>
22 #include "test_bbdev_vector.h"
24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
26 #define MAX_QUEUES RTE_MAX_LCORE
27 #define TEST_REPETITIONS 1000
29 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
30 #include <fpga_lte_fec.h>
31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
33 #define VF_UL_4G_QUEUE_VALUE 4
34 #define VF_DL_4G_QUEUE_VALUE 4
35 #define UL_4G_BANDWIDTH 3
36 #define DL_4G_BANDWIDTH 3
37 #define UL_4G_LOAD_BALANCE 128
38 #define DL_4G_LOAD_BALANCE 128
39 #define FLR_4G_TIMEOUT 610
42 #define OPS_CACHE_SIZE 256U
43 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
47 #define INVALID_OPAQUE -1
49 #define INVALID_QUEUE_ID -1
50 /* Increment for next code block in external HARQ memory */
51 #define HARQ_INCR 32768
52 /* Headroom for filler LLRs insertion in HARQ buffer */
53 #define FILLER_HEADROOM 1024
54 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
55 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
56 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
57 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
58 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
59 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
60 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
61 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
62 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
64 static struct test_bbdev_vector test_vector;
66 /* Switch between PMD and Interrupt for throughput TC */
67 static bool intr_enabled;
69 /* LLR arithmetic representation for numerical conversion */
70 static int ldpc_llr_decimals;
71 static int ldpc_llr_size;
72 /* Keep track of the LDPC decoder device capability flag */
73 static uint32_t ldpc_cap_flags;
75 /* Represents tested active devices */
76 static struct active_device {
77 const char *driver_name;
79 uint16_t supported_ops;
80 uint16_t queue_ids[MAX_QUEUES];
82 struct rte_mempool *ops_mempool;
83 struct rte_mempool *in_mbuf_pool;
84 struct rte_mempool *hard_out_mbuf_pool;
85 struct rte_mempool *soft_out_mbuf_pool;
86 struct rte_mempool *harq_in_mbuf_pool;
87 struct rte_mempool *harq_out_mbuf_pool;
88 } active_devs[RTE_BBDEV_MAX_DEVS];
90 static uint8_t nb_active_devs;
92 /* Data buffers used by BBDEV ops */
94 struct rte_bbdev_op_data *inputs;
95 struct rte_bbdev_op_data *hard_outputs;
96 struct rte_bbdev_op_data *soft_outputs;
97 struct rte_bbdev_op_data *harq_inputs;
98 struct rte_bbdev_op_data *harq_outputs;
101 /* Operation parameters specific for given test case */
102 struct test_op_params {
103 struct rte_mempool *mp;
104 struct rte_bbdev_dec_op *ref_dec_op;
105 struct rte_bbdev_enc_op *ref_enc_op;
107 uint16_t num_to_process;
111 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
114 /* Contains per lcore params */
115 struct thread_params {
125 rte_atomic16_t nb_dequeued;
126 rte_atomic16_t processing_status;
127 rte_atomic16_t burst_sz;
128 struct test_op_params *op_params;
129 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
130 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
133 #ifdef RTE_BBDEV_OFFLOAD_COST
134 /* Stores time statistics */
135 struct test_time_stats {
136 /* Stores software enqueue total working time */
137 uint64_t enq_sw_total_time;
138 /* Stores minimum value of software enqueue working time */
139 uint64_t enq_sw_min_time;
140 /* Stores maximum value of software enqueue working time */
141 uint64_t enq_sw_max_time;
142 /* Stores turbo enqueue total working time */
143 uint64_t enq_acc_total_time;
144 /* Stores minimum value of accelerator enqueue working time */
145 uint64_t enq_acc_min_time;
146 /* Stores maximum value of accelerator enqueue working time */
147 uint64_t enq_acc_max_time;
148 /* Stores dequeue total working time */
149 uint64_t deq_total_time;
150 /* Stores minimum value of dequeue working time */
151 uint64_t deq_min_time;
152 /* Stores maximum value of dequeue working time */
153 uint64_t deq_max_time;
157 typedef int (test_case_function)(struct active_device *ad,
158 struct test_op_params *op_params);
161 mbuf_reset(struct rte_mbuf *m)
171 /* Read flag value 0/1 from bitmap */
173 check_bit(uint32_t bitmap, uint32_t bitmask)
175 return bitmap & bitmask;
179 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
181 ad->supported_ops |= (1 << op_type);
185 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
187 return ad->supported_ops & (1 << op_type);
191 flags_match(uint32_t flags_req, uint32_t flags_present)
193 return (flags_req & flags_present) == flags_req;
197 clear_soft_out_cap(uint32_t *op_flags)
199 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
200 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
201 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
205 check_dev_cap(const struct rte_bbdev_info *dev_info)
208 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
209 nb_harq_inputs, nb_harq_outputs;
210 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
212 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
213 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
214 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
215 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
216 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
218 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
219 if (op_cap->type != test_vector.op_type)
222 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
223 const struct rte_bbdev_op_cap_turbo_dec *cap =
224 &op_cap->cap.turbo_dec;
225 /* Ignore lack of soft output capability, just skip
226 * checking if soft output is valid.
228 if ((test_vector.turbo_dec.op_flags &
229 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
230 !(cap->capability_flags &
231 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
233 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
236 &test_vector.turbo_dec.op_flags);
239 if (!flags_match(test_vector.turbo_dec.op_flags,
240 cap->capability_flags))
242 if (nb_inputs > cap->num_buffers_src) {
243 printf("Too many inputs defined: %u, max: %u\n",
244 nb_inputs, cap->num_buffers_src);
247 if (nb_soft_outputs > cap->num_buffers_soft_out &&
248 (test_vector.turbo_dec.op_flags &
249 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
251 "Too many soft outputs defined: %u, max: %u\n",
253 cap->num_buffers_soft_out);
256 if (nb_hard_outputs > cap->num_buffers_hard_out) {
258 "Too many hard outputs defined: %u, max: %u\n",
260 cap->num_buffers_hard_out);
263 if (intr_enabled && !(cap->capability_flags &
264 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
266 "Dequeue interrupts are not supported!\n");
271 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
272 const struct rte_bbdev_op_cap_turbo_enc *cap =
273 &op_cap->cap.turbo_enc;
275 if (!flags_match(test_vector.turbo_enc.op_flags,
276 cap->capability_flags))
278 if (nb_inputs > cap->num_buffers_src) {
279 printf("Too many inputs defined: %u, max: %u\n",
280 nb_inputs, cap->num_buffers_src);
283 if (nb_hard_outputs > cap->num_buffers_dst) {
285 "Too many hard outputs defined: %u, max: %u\n",
286 nb_hard_outputs, cap->num_buffers_dst);
289 if (intr_enabled && !(cap->capability_flags &
290 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
292 "Dequeue interrupts are not supported!\n");
297 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
298 const struct rte_bbdev_op_cap_ldpc_enc *cap =
299 &op_cap->cap.ldpc_enc;
301 if (!flags_match(test_vector.ldpc_enc.op_flags,
302 cap->capability_flags)){
303 printf("Flag Mismatch\n");
306 if (nb_inputs > cap->num_buffers_src) {
307 printf("Too many inputs defined: %u, max: %u\n",
308 nb_inputs, cap->num_buffers_src);
311 if (nb_hard_outputs > cap->num_buffers_dst) {
313 "Too many hard outputs defined: %u, max: %u\n",
314 nb_hard_outputs, cap->num_buffers_dst);
317 if (intr_enabled && !(cap->capability_flags &
318 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
320 "Dequeue interrupts are not supported!\n");
325 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
326 const struct rte_bbdev_op_cap_ldpc_dec *cap =
327 &op_cap->cap.ldpc_dec;
329 if (!flags_match(test_vector.ldpc_dec.op_flags,
330 cap->capability_flags)){
331 printf("Flag Mismatch\n");
334 if (nb_inputs > cap->num_buffers_src) {
335 printf("Too many inputs defined: %u, max: %u\n",
336 nb_inputs, cap->num_buffers_src);
339 if (nb_hard_outputs > cap->num_buffers_hard_out) {
341 "Too many hard outputs defined: %u, max: %u\n",
343 cap->num_buffers_hard_out);
346 if (nb_harq_inputs > cap->num_buffers_hard_out) {
348 "Too many HARQ inputs defined: %u, max: %u\n",
350 cap->num_buffers_hard_out);
353 if (nb_harq_outputs > cap->num_buffers_hard_out) {
355 "Too many HARQ outputs defined: %u, max: %u\n",
357 cap->num_buffers_hard_out);
360 if (intr_enabled && !(cap->capability_flags &
361 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
363 "Dequeue interrupts are not supported!\n");
366 if (intr_enabled && (test_vector.ldpc_dec.op_flags &
367 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
368 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
369 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
371 printf("Skip loop-back with interrupt\n");
378 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
379 return TEST_SUCCESS; /* Special case for NULL device */
384 /* calculates optimal mempool size not smaller than the val */
386 optimal_mempool_size(unsigned int val)
388 return rte_align32pow2(val + 1) - 1;
391 /* allocates mbuf mempool for inputs and outputs */
392 static struct rte_mempool *
393 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
394 int socket_id, unsigned int mbuf_pool_size,
395 const char *op_type_str)
398 uint32_t max_seg_sz = 0;
399 char pool_name[RTE_MEMPOOL_NAMESIZE];
401 /* find max input segment size */
402 for (i = 0; i < entries->nb_segments; ++i)
403 if (entries->segments[i].length > max_seg_sz)
404 max_seg_sz = entries->segments[i].length;
406 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
408 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
409 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
411 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
415 create_mempools(struct active_device *ad, int socket_id,
416 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
418 struct rte_mempool *mp;
419 unsigned int ops_pool_size, mbuf_pool_size = 0;
420 char pool_name[RTE_MEMPOOL_NAMESIZE];
421 const char *op_type_str;
422 enum rte_bbdev_op_type op_type = org_op_type;
424 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
425 struct op_data_entries *hard_out =
426 &test_vector.entries[DATA_HARD_OUTPUT];
427 struct op_data_entries *soft_out =
428 &test_vector.entries[DATA_SOFT_OUTPUT];
429 struct op_data_entries *harq_in =
430 &test_vector.entries[DATA_HARQ_INPUT];
431 struct op_data_entries *harq_out =
432 &test_vector.entries[DATA_HARQ_OUTPUT];
434 /* allocate ops mempool */
435 ops_pool_size = optimal_mempool_size(RTE_MAX(
436 /* Ops used plus 1 reference op */
437 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
438 /* Minimal cache size plus 1 reference op */
439 (unsigned int)(1.5 * rte_lcore_count() *
440 OPS_CACHE_SIZE + 1)),
443 if (org_op_type == RTE_BBDEV_OP_NONE)
444 op_type = RTE_BBDEV_OP_TURBO_ENC;
446 op_type_str = rte_bbdev_op_type_str(op_type);
447 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
449 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
451 mp = rte_bbdev_op_pool_create(pool_name, op_type,
452 ops_pool_size, OPS_CACHE_SIZE, socket_id);
453 TEST_ASSERT_NOT_NULL(mp,
454 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
458 ad->ops_mempool = mp;
460 /* Do not create inputs and outputs mbufs for BaseBand Null Device */
461 if (org_op_type == RTE_BBDEV_OP_NONE)
465 if (in->nb_segments > 0) {
466 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
468 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
469 mbuf_pool_size, "in");
470 TEST_ASSERT_NOT_NULL(mp,
471 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
475 ad->in_mbuf_pool = mp;
479 if (hard_out->nb_segments > 0) {
480 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
481 hard_out->nb_segments);
482 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
485 TEST_ASSERT_NOT_NULL(mp,
486 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
490 ad->hard_out_mbuf_pool = mp;
494 if (soft_out->nb_segments > 0) {
495 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
496 soft_out->nb_segments);
497 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
500 TEST_ASSERT_NOT_NULL(mp,
501 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
505 ad->soft_out_mbuf_pool = mp;
509 if (harq_in->nb_segments > 0) {
510 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
511 harq_in->nb_segments);
512 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
515 TEST_ASSERT_NOT_NULL(mp,
516 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
520 ad->harq_in_mbuf_pool = mp;
524 if (harq_out->nb_segments > 0) {
525 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
526 harq_out->nb_segments);
527 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
530 TEST_ASSERT_NOT_NULL(mp,
531 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
535 ad->harq_out_mbuf_pool = mp;
542 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
543 struct test_bbdev_vector *vector)
546 unsigned int queue_id;
547 struct rte_bbdev_queue_conf qconf;
548 struct active_device *ad = &active_devs[nb_active_devs];
549 unsigned int nb_queues;
550 enum rte_bbdev_op_type op_type = vector->op_type;
552 /* Configure fpga lte fec with PF & VF values
553 * if '-i' flag is set and using fpga device
555 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
556 if ((get_init_device() == true) &&
557 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
558 struct fpga_lte_fec_conf conf;
561 printf("Configure FPGA LTE FEC Driver %s with default values\n",
562 info->drv.driver_name);
564 /* clear default configuration before initialization */
565 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
568 * true if PF is used for data plane
571 conf.pf_mode_en = true;
573 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
574 /* Number of UL queues per VF (fpga supports 8 VFs) */
575 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
576 /* Number of DL queues per VF (fpga supports 8 VFs) */
577 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
580 /* UL bandwidth. Needed for schedule algorithm */
581 conf.ul_bandwidth = UL_4G_BANDWIDTH;
583 conf.dl_bandwidth = DL_4G_BANDWIDTH;
585 /* UL & DL load Balance Factor to 64 */
586 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
587 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
589 /**< FLR timeout value */
590 conf.flr_time_out = FLR_4G_TIMEOUT;
592 /* setup FPGA PF with configuration information */
593 ret = fpga_lte_fec_configure(info->dev_name, &conf);
594 TEST_ASSERT_SUCCESS(ret,
595 "Failed to configure 4G FPGA PF for bbdev %s",
599 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
600 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
603 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
605 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
606 dev_id, nb_queues, info->socket_id, ret);
610 /* configure interrupts if needed */
612 ret = rte_bbdev_intr_enable(dev_id);
614 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
620 /* setup device queues */
621 qconf.socket = info->socket_id;
622 qconf.queue_size = info->drv.default_queue_conf.queue_size;
624 qconf.deferred_start = 0;
625 qconf.op_type = op_type;
627 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
628 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
631 "Allocated all queues (id=%u) at prio%u on dev%u\n",
632 queue_id, qconf.priority, dev_id);
634 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
638 printf("All queues on dev %u allocated: %u\n",
642 ad->queue_ids[queue_id] = queue_id;
644 TEST_ASSERT(queue_id != 0,
645 "ERROR Failed to configure any queues on dev %u",
647 ad->nb_queues = queue_id;
649 set_avail_op(ad, op_type);
655 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
656 struct test_bbdev_vector *vector)
660 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
661 active_devs[nb_active_devs].dev_id = dev_id;
663 ret = add_bbdev_dev(dev_id, info, vector);
664 if (ret == TEST_SUCCESS)
670 populate_active_devices(void)
674 uint8_t nb_devs_added = 0;
675 struct rte_bbdev_info info;
677 RTE_BBDEV_FOREACH(dev_id) {
678 rte_bbdev_info_get(dev_id, &info);
680 if (check_dev_cap(&info)) {
682 "Device %d (%s) does not support specified capabilities\n",
683 dev_id, info.dev_name);
687 ret = add_active_device(dev_id, &info, &test_vector);
689 printf("Adding active bbdev %s skipped\n",
696 return nb_devs_added;
700 read_test_vector(void)
704 memset(&test_vector, 0, sizeof(test_vector));
705 printf("Test vector file = %s\n", get_vector_filename());
706 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
707 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
708 get_vector_filename());
714 testsuite_setup(void)
716 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
718 if (populate_active_devices() == 0) {
719 printf("No suitable devices found!\n");
727 interrupt_testsuite_setup(void)
729 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
731 /* Enable interrupts */
734 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
735 if (populate_active_devices() == 0 ||
736 test_vector.op_type == RTE_BBDEV_OP_NONE) {
737 intr_enabled = false;
738 printf("No suitable devices found!\n");
746 testsuite_teardown(void)
750 /* Unconfigure devices */
751 RTE_BBDEV_FOREACH(dev_id)
752 rte_bbdev_close(dev_id);
754 /* Clear active devices structs. */
755 memset(active_devs, 0, sizeof(active_devs));
764 for (i = 0; i < nb_active_devs; i++) {
765 dev_id = active_devs[i].dev_id;
766 /* reset bbdev stats */
767 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
768 "Failed to reset stats of bbdev %u", dev_id);
769 /* start the device */
770 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
771 "Failed to start bbdev %u", dev_id);
781 struct rte_bbdev_stats stats;
783 for (i = 0; i < nb_active_devs; i++) {
784 dev_id = active_devs[i].dev_id;
785 /* read stats and print */
786 rte_bbdev_stats_get(dev_id, &stats);
787 /* Stop the device */
788 rte_bbdev_stop(dev_id);
793 init_op_data_objs(struct rte_bbdev_op_data *bufs,
794 struct op_data_entries *ref_entries,
795 struct rte_mempool *mbuf_pool, const uint16_t n,
796 enum op_data_type op_type, uint16_t min_alignment)
800 bool large_input = false;
802 for (i = 0; i < n; ++i) {
804 struct op_data_buf *seg = &ref_entries->segments[0];
805 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
806 TEST_ASSERT_NOT_NULL(m_head,
807 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
808 op_type, n * ref_entries->nb_segments,
811 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
813 * Special case when DPDK mbuf cannot handle
814 * the required input size
816 printf("Warning: Larger input size than DPDK mbuf %d\n",
820 bufs[i].data = m_head;
824 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
825 if ((op_type == DATA_INPUT) && large_input) {
826 /* Allocate a fake overused mbuf */
827 data = rte_malloc(NULL, seg->length, 0);
828 memcpy(data, seg->addr, seg->length);
829 m_head->buf_addr = data;
830 m_head->buf_iova = rte_malloc_virt2iova(data);
831 m_head->data_off = 0;
832 m_head->data_len = seg->length;
834 data = rte_pktmbuf_append(m_head, seg->length);
835 TEST_ASSERT_NOT_NULL(data,
836 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
837 seg->length, op_type);
839 TEST_ASSERT(data == RTE_PTR_ALIGN(
840 data, min_alignment),
841 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
842 data, min_alignment);
843 rte_memcpy(data, seg->addr, seg->length);
846 bufs[i].length += seg->length;
848 for (j = 1; j < ref_entries->nb_segments; ++j) {
849 struct rte_mbuf *m_tail =
850 rte_pktmbuf_alloc(mbuf_pool);
851 TEST_ASSERT_NOT_NULL(m_tail,
852 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
854 n * ref_entries->nb_segments,
858 data = rte_pktmbuf_append(m_tail, seg->length);
859 TEST_ASSERT_NOT_NULL(data,
860 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
861 seg->length, op_type);
863 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
865 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
866 data, min_alignment);
867 rte_memcpy(data, seg->addr, seg->length);
868 bufs[i].length += seg->length;
870 ret = rte_pktmbuf_chain(m_head, m_tail);
871 TEST_ASSERT_SUCCESS(ret,
872 "Couldn't chain mbufs from %d data type mbuf pool",
877 /* allocate chained-mbuf for output buffer */
878 for (j = 1; j < ref_entries->nb_segments; ++j) {
879 struct rte_mbuf *m_tail =
880 rte_pktmbuf_alloc(mbuf_pool);
881 TEST_ASSERT_NOT_NULL(m_tail,
882 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
884 n * ref_entries->nb_segments,
887 ret = rte_pktmbuf_chain(m_head, m_tail);
888 TEST_ASSERT_SUCCESS(ret,
889 "Couldn't chain mbufs from %d data type mbuf pool",
899 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
904 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
905 if (*buffers == NULL) {
906 printf("WARNING: Failed to allocate op_data on socket %d\n",
908 /* try to allocate memory on other detected sockets */
909 for (i = 0; i < socket; i++) {
910 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
911 if (*buffers != NULL)
916 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
920 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
921 const uint16_t n, const int8_t max_llr_modulus)
923 uint16_t i, byte_idx;
925 for (i = 0; i < n; ++i) {
926 struct rte_mbuf *m = input_ops[i].data;
928 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
929 input_ops[i].offset);
930 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
932 llr[byte_idx] = round((double)max_llr_modulus *
933 llr[byte_idx] / INT8_MAX);
941 * We may have to insert filler bits
942 * when they are required by the HARQ assumption
945 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
946 const uint16_t n, struct test_op_params *op_params)
948 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
950 if (input_ops == NULL)
952 /* No need to add filler if not required by device */
953 if (!(ldpc_cap_flags &
954 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
956 /* No need to add filler for loopback operation */
957 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
960 uint16_t i, j, parity_offset;
961 for (i = 0; i < n; ++i) {
962 struct rte_mbuf *m = input_ops[i].data;
963 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
964 input_ops[i].offset);
965 parity_offset = (dec.basegraph == 1 ? 20 : 8)
966 * dec.z_c - dec.n_filler;
967 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
968 m->data_len = new_hin_size;
969 input_ops[i].length = new_hin_size;
970 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
972 llr[j] = llr[j - dec.n_filler];
973 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
974 for (j = 0; j < dec.n_filler; j++)
975 llr[parity_offset + j] = llr_max_pre_scaling;
980 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
981 const uint16_t n, const int8_t llr_size,
982 const int8_t llr_decimals)
984 if (input_ops == NULL)
987 uint16_t i, byte_idx;
989 int16_t llr_max, llr_min, llr_tmp;
990 llr_max = (1 << (llr_size - 1)) - 1;
992 for (i = 0; i < n; ++i) {
993 struct rte_mbuf *m = input_ops[i].data;
995 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
996 input_ops[i].offset);
997 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1000 llr_tmp = llr[byte_idx];
1001 if (llr_decimals == 4)
1003 else if (llr_decimals == 2)
1005 else if (llr_decimals == 0)
1007 llr_tmp = RTE_MIN(llr_max,
1008 RTE_MAX(llr_min, llr_tmp));
1009 llr[byte_idx] = (int8_t) llr_tmp;
1020 fill_queue_buffers(struct test_op_params *op_params,
1021 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1022 struct rte_mempool *soft_out_mp,
1023 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1025 const struct rte_bbdev_op_cap *capabilities,
1026 uint16_t min_alignment, const int socket_id)
1029 enum op_data_type type;
1030 const uint16_t n = op_params->num_to_process;
1032 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1040 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1041 &op_params->q_bufs[socket_id][queue_id].inputs,
1042 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1043 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1044 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1045 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1048 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1049 struct op_data_entries *ref_entries =
1050 &test_vector.entries[type];
1051 if (ref_entries->nb_segments == 0)
1054 ret = allocate_buffers_on_socket(queue_ops[type],
1055 n * sizeof(struct rte_bbdev_op_data),
1057 TEST_ASSERT_SUCCESS(ret,
1058 "Couldn't allocate memory for rte_bbdev_op_data structs");
1060 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1061 mbuf_pools[type], n, type, min_alignment);
1062 TEST_ASSERT_SUCCESS(ret,
1063 "Couldn't init rte_bbdev_op_data structs");
1066 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1067 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1068 capabilities->cap.turbo_dec.max_llr_modulus);
1070 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1071 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1072 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1073 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1074 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1075 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1076 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1077 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1078 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1079 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1080 if (!loopback && !llr_comp)
1081 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1082 ldpc_llr_size, ldpc_llr_decimals);
1083 if (!loopback && !harq_comp)
1084 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1085 ldpc_llr_size, ldpc_llr_decimals);
1087 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1095 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1099 rte_mempool_free(ad->ops_mempool);
1100 rte_mempool_free(ad->in_mbuf_pool);
1101 rte_mempool_free(ad->hard_out_mbuf_pool);
1102 rte_mempool_free(ad->soft_out_mbuf_pool);
1103 rte_mempool_free(ad->harq_in_mbuf_pool);
1104 rte_mempool_free(ad->harq_out_mbuf_pool);
1106 for (i = 0; i < rte_lcore_count(); ++i) {
1107 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1108 rte_free(op_params->q_bufs[j][i].inputs);
1109 rte_free(op_params->q_bufs[j][i].hard_outputs);
1110 rte_free(op_params->q_bufs[j][i].soft_outputs);
1111 rte_free(op_params->q_bufs[j][i].harq_inputs);
1112 rte_free(op_params->q_bufs[j][i].harq_outputs);
1118 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1119 unsigned int start_idx,
1120 struct rte_bbdev_op_data *inputs,
1121 struct rte_bbdev_op_data *hard_outputs,
1122 struct rte_bbdev_op_data *soft_outputs,
1123 struct rte_bbdev_dec_op *ref_op)
1126 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1128 for (i = 0; i < n; ++i) {
1129 if (turbo_dec->code_block_mode == 0) {
1130 ops[i]->turbo_dec.tb_params.ea =
1131 turbo_dec->tb_params.ea;
1132 ops[i]->turbo_dec.tb_params.eb =
1133 turbo_dec->tb_params.eb;
1134 ops[i]->turbo_dec.tb_params.k_pos =
1135 turbo_dec->tb_params.k_pos;
1136 ops[i]->turbo_dec.tb_params.k_neg =
1137 turbo_dec->tb_params.k_neg;
1138 ops[i]->turbo_dec.tb_params.c =
1139 turbo_dec->tb_params.c;
1140 ops[i]->turbo_dec.tb_params.c_neg =
1141 turbo_dec->tb_params.c_neg;
1142 ops[i]->turbo_dec.tb_params.cab =
1143 turbo_dec->tb_params.cab;
1144 ops[i]->turbo_dec.tb_params.r =
1145 turbo_dec->tb_params.r;
1147 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1148 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1151 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1152 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1153 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1154 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1155 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1156 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1157 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1159 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1160 ops[i]->turbo_dec.input = inputs[start_idx + i];
1161 if (soft_outputs != NULL)
1162 ops[i]->turbo_dec.soft_output =
1163 soft_outputs[start_idx + i];
1168 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1169 unsigned int start_idx,
1170 struct rte_bbdev_op_data *inputs,
1171 struct rte_bbdev_op_data *outputs,
1172 struct rte_bbdev_enc_op *ref_op)
1175 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1176 for (i = 0; i < n; ++i) {
1177 if (turbo_enc->code_block_mode == 0) {
1178 ops[i]->turbo_enc.tb_params.ea =
1179 turbo_enc->tb_params.ea;
1180 ops[i]->turbo_enc.tb_params.eb =
1181 turbo_enc->tb_params.eb;
1182 ops[i]->turbo_enc.tb_params.k_pos =
1183 turbo_enc->tb_params.k_pos;
1184 ops[i]->turbo_enc.tb_params.k_neg =
1185 turbo_enc->tb_params.k_neg;
1186 ops[i]->turbo_enc.tb_params.c =
1187 turbo_enc->tb_params.c;
1188 ops[i]->turbo_enc.tb_params.c_neg =
1189 turbo_enc->tb_params.c_neg;
1190 ops[i]->turbo_enc.tb_params.cab =
1191 turbo_enc->tb_params.cab;
1192 ops[i]->turbo_enc.tb_params.ncb_pos =
1193 turbo_enc->tb_params.ncb_pos;
1194 ops[i]->turbo_enc.tb_params.ncb_neg =
1195 turbo_enc->tb_params.ncb_neg;
1196 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1198 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1199 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1200 ops[i]->turbo_enc.cb_params.ncb =
1201 turbo_enc->cb_params.ncb;
1203 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1204 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1205 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1207 ops[i]->turbo_enc.output = outputs[start_idx + i];
1208 ops[i]->turbo_enc.input = inputs[start_idx + i];
1213 /* Returns a random number drawn from a normal distribution
1214 * with mean of 0 and variance of 1
1215 * Marsaglia algorithm
1220 double S, Z, U1, U2, u, v, fac;
1223 U1 = (double)rand() / RAND_MAX;
1224 U2 = (double)rand() / RAND_MAX;
1228 } while (S >= 1 || S == 0);
1229 fac = sqrt(-2. * log(S) / S);
1230 Z = (n % 2) ? u * fac : v * fac;
1234 static inline double
1235 maxstar(double A, double B)
1237 if (fabs(A - B) > 5)
1238 return RTE_MAX(A, B);
1240 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1244 * Generate Qm LLRS for Qm==8
1245 * Modulation, AWGN and LLR estimation from max log development
1248 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1253 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1254 /* 5.1.4 of TS38.211 */
1255 const double symbols_I[256] = {
1256 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1257 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1258 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1259 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1260 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1261 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1262 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1263 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1264 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1265 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1266 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1267 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1268 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1269 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1270 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1271 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1272 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1273 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1274 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1275 -13, -13, -15, -15, -13, -13, -15, -15};
1276 const double symbols_Q[256] = {
1277 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1278 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1279 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1280 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1281 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1282 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1283 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1284 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1285 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1286 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1287 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1288 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1289 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1290 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1291 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1292 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1293 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1294 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1295 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1296 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1297 /* Average constellation point energy */
1299 for (k = 0; k < qm; k++)
1300 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1301 /* 5.1.4 of TS38.211 */
1302 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1303 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1304 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1305 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1307 I += sqrt(N0 / 2) * randn(0);
1308 Q += sqrt(N0 / 2) * randn(1);
1310 * Calculate the log of the probability that each of
1311 * the constellation points was transmitted
1313 for (m = 0; m < qam; m++)
1314 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1315 + pow(Q - symbols_Q[m], 2.0)) / N0;
1316 /* Calculate an LLR for each of the k_64QAM bits in the set */
1317 for (k = 0; k < qm; k++) {
1320 /* For each constellation point */
1321 for (m = 0; m < qam; m++) {
1322 if ((m >> (qm - k - 1)) & 1)
1323 p1 = maxstar(p1, log_syml_prob[m]);
1325 p0 = maxstar(p0, log_syml_prob[m]);
1327 /* Calculate the LLR */
1329 llr_ *= (1 << ldpc_llr_decimals);
1333 if (llr_ < -llr_max)
1335 llrs[qm * i + k] = (int8_t) llr_;
1341 * Generate Qm LLRS for Qm==6
1342 * Modulation, AWGN and LLR estimation from max log development
1345 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1350 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1351 /* 5.1.4 of TS38.211 */
1352 const double symbols_I[64] = {
1353 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1354 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1355 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1356 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1357 -5, -5, -7, -7, -5, -5, -7, -7};
1358 const double symbols_Q[64] = {
1359 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1360 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1361 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1362 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1363 -3, -1, -3, -1, -5, -7, -5, -7};
1364 /* Average constellation point energy */
1366 for (k = 0; k < qm; k++)
1367 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1368 /* 5.1.4 of TS38.211 */
1369 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1370 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1372 I += sqrt(N0 / 2) * randn(0);
1373 Q += sqrt(N0 / 2) * randn(1);
1375 * Calculate the log of the probability that each of
1376 * the constellation points was transmitted
1378 for (m = 0; m < qam; m++)
1379 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1380 + pow(Q - symbols_Q[m], 2.0)) / N0;
1381 /* Calculate an LLR for each of the k_64QAM bits in the set */
1382 for (k = 0; k < qm; k++) {
1385 /* For each constellation point */
1386 for (m = 0; m < qam; m++) {
1387 if ((m >> (qm - k - 1)) & 1)
1388 p1 = maxstar(p1, log_syml_prob[m]);
1390 p0 = maxstar(p0, log_syml_prob[m]);
1392 /* Calculate the LLR */
1394 llr_ *= (1 << ldpc_llr_decimals);
1398 if (llr_ < -llr_max)
1400 llrs[qm * i + k] = (int8_t) llr_;
1405 * Generate Qm LLRS for Qm==4
1406 * Modulation, AWGN and LLR estimation from max log development
1409 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1414 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1415 /* 5.1.4 of TS38.211 */
1416 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1417 -1, -1, -3, -3, -1, -1, -3, -3};
1418 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1419 1, 3, 1, 3, -1, -3, -1, -3};
1420 /* Average constellation point energy */
1422 for (k = 0; k < qm; k++)
1423 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1424 /* 5.1.4 of TS38.211 */
1425 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1426 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1428 I += sqrt(N0 / 2) * randn(0);
1429 Q += sqrt(N0 / 2) * randn(1);
1431 * Calculate the log of the probability that each of
1432 * the constellation points was transmitted
1434 for (m = 0; m < qam; m++)
1435 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1436 + pow(Q - symbols_Q[m], 2.0)) / N0;
1437 /* Calculate an LLR for each of the k_64QAM bits in the set */
1438 for (k = 0; k < qm; k++) {
1441 /* For each constellation point */
1442 for (m = 0; m < qam; m++) {
1443 if ((m >> (qm - k - 1)) & 1)
1444 p1 = maxstar(p1, log_syml_prob[m]);
1446 p0 = maxstar(p0, log_syml_prob[m]);
1448 /* Calculate the LLR */
1450 llr_ *= (1 << ldpc_llr_decimals);
1454 if (llr_ < -llr_max)
1456 llrs[qm * i + k] = (int8_t) llr_;
1461 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1464 double coeff = 2.0 * sqrt(N0);
1466 /* Ignore in vectors rare quasi null LLRs not to be saturated */
1467 if (llrs[j] < 8 && llrs[j] > -8)
1470 /* Note don't change sign here */
1472 b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1474 b = b1 * (1 << ldpc_llr_decimals);
1480 llrs[j] = (int8_t) b;
1483 /* Generate LLR for a given SNR */
1485 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1486 struct rte_bbdev_dec_op *ref_op)
1490 uint32_t i, j, e, range;
1493 e = ref_op->ldpc_dec.cb_params.e;
1494 qm = ref_op->ldpc_dec.q_m;
1495 llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1497 N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1499 for (i = 0; i < n; ++i) {
1501 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1503 for (j = 0; j < range; ++j)
1504 gen_qm8_llr(llrs, j, N0, llr_max);
1505 } else if (qm == 6) {
1506 for (j = 0; j < range; ++j)
1507 gen_qm6_llr(llrs, j, N0, llr_max);
1508 } else if (qm == 4) {
1509 for (j = 0; j < range; ++j)
1510 gen_qm4_llr(llrs, j, N0, llr_max);
1512 for (j = 0; j < e; ++j)
1513 gen_qm2_llr(llrs, j, N0, llr_max);
1519 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1520 unsigned int start_idx,
1521 struct rte_bbdev_op_data *inputs,
1522 struct rte_bbdev_op_data *hard_outputs,
1523 struct rte_bbdev_op_data *soft_outputs,
1524 struct rte_bbdev_op_data *harq_inputs,
1525 struct rte_bbdev_op_data *harq_outputs,
1526 struct rte_bbdev_dec_op *ref_op)
1529 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1531 for (i = 0; i < n; ++i) {
1532 if (ldpc_dec->code_block_mode == 0) {
1533 ops[i]->ldpc_dec.tb_params.ea =
1534 ldpc_dec->tb_params.ea;
1535 ops[i]->ldpc_dec.tb_params.eb =
1536 ldpc_dec->tb_params.eb;
1537 ops[i]->ldpc_dec.tb_params.c =
1538 ldpc_dec->tb_params.c;
1539 ops[i]->ldpc_dec.tb_params.cab =
1540 ldpc_dec->tb_params.cab;
1541 ops[i]->ldpc_dec.tb_params.r =
1542 ldpc_dec->tb_params.r;
1544 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1547 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1548 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1549 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1550 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1551 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1552 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1553 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1554 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1555 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1557 if (hard_outputs != NULL)
1558 ops[i]->ldpc_dec.hard_output =
1559 hard_outputs[start_idx + i];
1561 ops[i]->ldpc_dec.input =
1562 inputs[start_idx + i];
1563 if (soft_outputs != NULL)
1564 ops[i]->ldpc_dec.soft_output =
1565 soft_outputs[start_idx + i];
1566 if (harq_inputs != NULL)
1567 ops[i]->ldpc_dec.harq_combined_input =
1568 harq_inputs[start_idx + i];
1569 if (harq_outputs != NULL)
1570 ops[i]->ldpc_dec.harq_combined_output =
1571 harq_outputs[start_idx + i];
1577 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1578 unsigned int start_idx,
1579 struct rte_bbdev_op_data *inputs,
1580 struct rte_bbdev_op_data *outputs,
1581 struct rte_bbdev_enc_op *ref_op)
1584 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1585 for (i = 0; i < n; ++i) {
1586 if (ldpc_enc->code_block_mode == 0) {
1587 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1588 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1589 ops[i]->ldpc_enc.tb_params.cab =
1590 ldpc_enc->tb_params.cab;
1591 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1592 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1594 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1596 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1597 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1598 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1599 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1600 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1601 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1602 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1603 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1604 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1605 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1610 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1611 unsigned int order_idx, const int expected_status)
1613 int status = op->status;
1614 /* ignore parity mismatch false alarms for long iterations */
1615 if (get_iter_max() >= 10) {
1616 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1617 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1618 printf("WARNING: Ignore Syndrome Check mismatch\n");
1619 status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1621 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1622 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1623 printf("WARNING: Ignore Syndrome Check mismatch\n");
1624 status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1628 TEST_ASSERT(status == expected_status,
1629 "op_status (%d) != expected_status (%d)",
1630 op->status, expected_status);
1632 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1633 "Ordering error, expected %p, got %p",
1634 (void *)(uintptr_t)order_idx, op->opaque_data);
1636 return TEST_SUCCESS;
1640 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1641 unsigned int order_idx, const int expected_status)
1643 TEST_ASSERT(op->status == expected_status,
1644 "op_status (%d) != expected_status (%d)",
1645 op->status, expected_status);
1647 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1648 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1649 "Ordering error, expected %p, got %p",
1650 (void *)(uintptr_t)order_idx, op->opaque_data);
1652 return TEST_SUCCESS;
1656 validate_op_chain(struct rte_bbdev_op_data *op,
1657 struct op_data_entries *orig_op)
1660 struct rte_mbuf *m = op->data;
1661 uint8_t nb_dst_segments = orig_op->nb_segments;
1662 uint32_t total_data_size = 0;
1664 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1665 "Number of segments differ in original (%u) and filled (%u) op",
1666 nb_dst_segments, m->nb_segs);
1668 /* Validate each mbuf segment length */
1669 for (i = 0; i < nb_dst_segments; ++i) {
1670 /* Apply offset to the first mbuf segment */
1671 uint16_t offset = (i == 0) ? op->offset : 0;
1672 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1673 total_data_size += orig_op->segments[i].length;
1675 TEST_ASSERT(orig_op->segments[i].length == data_len,
1676 "Length of segment differ in original (%u) and filled (%u) op",
1677 orig_op->segments[i].length, data_len);
1678 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1679 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1681 "Output buffers (CB=%u) are not equal", i);
1685 /* Validate total mbuf pkt length */
1686 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1687 TEST_ASSERT(total_data_size == pkt_len,
1688 "Length of data differ in original (%u) and filled (%u) op",
1689 total_data_size, pkt_len);
1691 return TEST_SUCCESS;
1695 * Compute K0 for a given configuration for HARQ output length computation
1696 * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1698 static inline uint16_t
1699 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1703 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1706 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1707 else if (rv_index == 2)
1708 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1710 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1712 /* LBRM case - includes a division by N */
1714 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1716 else if (rv_index == 2)
1717 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1720 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1724 /* HARQ output length including the Filler bits */
1725 static inline uint16_t
1726 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1729 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1730 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1731 /* Compute RM out size and number of rows */
1732 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1733 * ops_ld->z_c - ops_ld->n_filler;
1734 uint16_t deRmOutSize = RTE_MIN(
1735 k0 + ops_ld->cb_params.e +
1736 ((k0 > parity_offset) ?
1737 0 : ops_ld->n_filler),
1739 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1741 uint16_t harq_output_len = numRows * ops_ld->z_c;
1742 return harq_output_len;
1746 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1747 struct op_data_entries *orig_op,
1748 struct rte_bbdev_op_ldpc_dec *ops_ld)
1752 struct rte_mbuf *m = op->data;
1753 uint8_t nb_dst_segments = orig_op->nb_segments;
1754 uint32_t total_data_size = 0;
1755 int8_t *harq_orig, *harq_out, abs_harq_origin;
1756 uint32_t byte_error = 0, cum_error = 0, error;
1757 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1758 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1759 uint16_t parity_offset;
1761 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1762 "Number of segments differ in original (%u) and filled (%u) op",
1763 nb_dst_segments, m->nb_segs);
1765 /* Validate each mbuf segment length */
1766 for (i = 0; i < nb_dst_segments; ++i) {
1767 /* Apply offset to the first mbuf segment */
1768 uint16_t offset = (i == 0) ? op->offset : 0;
1769 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1770 total_data_size += orig_op->segments[i].length;
1772 TEST_ASSERT(orig_op->segments[i].length <
1773 (uint32_t)(data_len + 64),
1774 "Length of segment differ in original (%u) and filled (%u) op",
1775 orig_op->segments[i].length, data_len);
1776 harq_orig = (int8_t *) orig_op->segments[i].addr;
1777 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1779 if (!(ldpc_cap_flags &
1780 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1781 ) || (ops_ld->op_flags &
1782 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1783 data_len -= ops_ld->z_c;
1784 parity_offset = data_len;
1786 /* Compute RM out size and number of rows */
1787 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1788 * ops_ld->z_c - ops_ld->n_filler;
1789 uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1791 if (data_len > deRmOutSize)
1792 data_len = deRmOutSize;
1793 if (data_len > orig_op->segments[i].length)
1794 data_len = orig_op->segments[i].length;
1797 * HARQ output can have minor differences
1798 * due to integer representation and related scaling
1800 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1801 if (j == parity_offset) {
1802 /* Special Handling of the filler bits */
1803 for (k = 0; k < ops_ld->n_filler; k++) {
1805 llr_max_pre_scaling) {
1806 printf("HARQ Filler issue %d: %d %d\n",
1814 if (!(ops_ld->op_flags &
1815 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1816 if (ldpc_llr_decimals > 1)
1817 harq_out[jj] = (harq_out[jj] + 1)
1818 >> (ldpc_llr_decimals - 1);
1819 /* Saturated to S7 */
1820 if (harq_orig[j] > llr_max)
1821 harq_orig[j] = llr_max;
1822 if (harq_orig[j] < -llr_max)
1823 harq_orig[j] = -llr_max;
1825 if (harq_orig[j] != harq_out[jj]) {
1826 error = (harq_orig[j] > harq_out[jj]) ?
1827 harq_orig[j] - harq_out[jj] :
1828 harq_out[jj] - harq_orig[j];
1829 abs_harq_origin = harq_orig[j] > 0 ?
1832 /* Residual quantization error */
1833 if ((error > 8 && (abs_harq_origin <
1836 printf("HARQ mismatch %d: exp %d act %d => %d\n",
1838 harq_out[jj], error);
1848 TEST_ASSERT(byte_error <= 1,
1849 "HARQ output mismatch (%d) %d",
1850 byte_error, cum_error);
1852 /* Validate total mbuf pkt length */
1853 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1854 TEST_ASSERT(total_data_size < pkt_len + 64,
1855 "Length of data differ in original (%u) and filled (%u) op",
1856 total_data_size, pkt_len);
1858 return TEST_SUCCESS;
1862 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1863 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1867 struct op_data_entries *hard_data_orig =
1868 &test_vector.entries[DATA_HARD_OUTPUT];
1869 struct op_data_entries *soft_data_orig =
1870 &test_vector.entries[DATA_SOFT_OUTPUT];
1871 struct rte_bbdev_op_turbo_dec *ops_td;
1872 struct rte_bbdev_op_data *hard_output;
1873 struct rte_bbdev_op_data *soft_output;
1874 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
1876 for (i = 0; i < n; ++i) {
1877 ops_td = &ops[i]->turbo_dec;
1878 hard_output = &ops_td->hard_output;
1879 soft_output = &ops_td->soft_output;
1881 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1882 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1883 "Returned iter_count (%d) > expected iter_count (%d)",
1884 ops_td->iter_count, ref_td->iter_count);
1885 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1886 TEST_ASSERT_SUCCESS(ret,
1887 "Checking status and ordering for decoder failed");
1889 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1891 "Hard output buffers (CB=%u) are not equal",
1894 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
1895 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1897 "Soft output buffers (CB=%u) are not equal",
1901 return TEST_SUCCESS;
1904 /* Check Number of code blocks errors */
1906 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
1909 struct op_data_entries *hard_data_orig =
1910 &test_vector.entries[DATA_HARD_OUTPUT];
1911 struct rte_bbdev_op_ldpc_dec *ops_td;
1912 struct rte_bbdev_op_data *hard_output;
1916 for (i = 0; i < n; ++i) {
1917 ops_td = &ops[i]->ldpc_dec;
1918 hard_output = &ops_td->hard_output;
1919 m = hard_output->data;
1920 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
1921 hard_data_orig->segments[0].addr,
1922 hard_data_orig->segments[0].length))
1929 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1930 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1934 struct op_data_entries *hard_data_orig =
1935 &test_vector.entries[DATA_HARD_OUTPUT];
1936 struct op_data_entries *soft_data_orig =
1937 &test_vector.entries[DATA_SOFT_OUTPUT];
1938 struct op_data_entries *harq_data_orig =
1939 &test_vector.entries[DATA_HARQ_OUTPUT];
1940 struct rte_bbdev_op_ldpc_dec *ops_td;
1941 struct rte_bbdev_op_data *hard_output;
1942 struct rte_bbdev_op_data *harq_output;
1943 struct rte_bbdev_op_data *soft_output;
1944 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
1946 for (i = 0; i < n; ++i) {
1947 ops_td = &ops[i]->ldpc_dec;
1948 hard_output = &ops_td->hard_output;
1949 harq_output = &ops_td->harq_combined_output;
1950 soft_output = &ops_td->soft_output;
1952 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1953 TEST_ASSERT_SUCCESS(ret,
1954 "Checking status and ordering for decoder failed");
1955 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1956 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1957 "Returned iter_count (%d) > expected iter_count (%d)",
1958 ops_td->iter_count, ref_td->iter_count);
1960 * We can ignore output data when the decoding failed to
1961 * converge or for loop-back cases
1963 if (!check_bit(ops[i]->ldpc_dec.op_flags,
1964 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
1966 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
1968 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1970 "Hard output buffers (CB=%u) are not equal",
1973 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
1974 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1976 "Soft output buffers (CB=%u) are not equal",
1978 if (ref_op->ldpc_dec.op_flags &
1979 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
1980 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
1981 harq_data_orig, ops_td),
1982 "HARQ output buffers (CB=%u) are not equal",
1985 if (ref_op->ldpc_dec.op_flags &
1986 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1987 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
1988 harq_data_orig, ops_td),
1989 "HARQ output buffers (CB=%u) are not equal",
1994 return TEST_SUCCESS;
1999 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2000 struct rte_bbdev_enc_op *ref_op)
2004 struct op_data_entries *hard_data_orig =
2005 &test_vector.entries[DATA_HARD_OUTPUT];
2007 for (i = 0; i < n; ++i) {
2008 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2009 TEST_ASSERT_SUCCESS(ret,
2010 "Checking status and ordering for encoder failed");
2011 TEST_ASSERT_SUCCESS(validate_op_chain(
2012 &ops[i]->turbo_enc.output,
2014 "Output buffers (CB=%u) are not equal",
2018 return TEST_SUCCESS;
2022 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2023 struct rte_bbdev_enc_op *ref_op)
2027 struct op_data_entries *hard_data_orig =
2028 &test_vector.entries[DATA_HARD_OUTPUT];
2030 for (i = 0; i < n; ++i) {
2031 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2032 TEST_ASSERT_SUCCESS(ret,
2033 "Checking status and ordering for encoder failed");
2034 TEST_ASSERT_SUCCESS(validate_op_chain(
2035 &ops[i]->ldpc_enc.output,
2037 "Output buffers (CB=%u) are not equal",
2041 return TEST_SUCCESS;
2045 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2048 struct op_data_entries *entry;
2050 op->turbo_dec = test_vector.turbo_dec;
2051 entry = &test_vector.entries[DATA_INPUT];
2052 for (i = 0; i < entry->nb_segments; ++i)
2053 op->turbo_dec.input.length +=
2054 entry->segments[i].length;
2058 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2061 struct op_data_entries *entry;
2063 op->ldpc_dec = test_vector.ldpc_dec;
2064 entry = &test_vector.entries[DATA_INPUT];
2065 for (i = 0; i < entry->nb_segments; ++i)
2066 op->ldpc_dec.input.length +=
2067 entry->segments[i].length;
2068 if (test_vector.ldpc_dec.op_flags &
2069 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2070 entry = &test_vector.entries[DATA_HARQ_INPUT];
2071 for (i = 0; i < entry->nb_segments; ++i)
2072 op->ldpc_dec.harq_combined_input.length +=
2073 entry->segments[i].length;
2079 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2082 struct op_data_entries *entry;
2084 op->turbo_enc = test_vector.turbo_enc;
2085 entry = &test_vector.entries[DATA_INPUT];
2086 for (i = 0; i < entry->nb_segments; ++i)
2087 op->turbo_enc.input.length +=
2088 entry->segments[i].length;
2092 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2095 struct op_data_entries *entry;
2097 op->ldpc_enc = test_vector.ldpc_enc;
2098 entry = &test_vector.entries[DATA_INPUT];
2099 for (i = 0; i < entry->nb_segments; ++i)
2100 op->ldpc_enc.input.length +=
2101 entry->segments[i].length;
2105 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2108 uint32_t c, r, tb_size = 0;
2110 if (op->turbo_dec.code_block_mode) {
2111 tb_size = op->turbo_dec.tb_params.k_neg;
2113 c = op->turbo_dec.tb_params.c;
2114 r = op->turbo_dec.tb_params.r;
2115 for (i = 0; i < c-r; i++)
2116 tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2117 op->turbo_dec.tb_params.k_neg :
2118 op->turbo_dec.tb_params.k_pos;
2124 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2127 uint32_t c, r, tb_size = 0;
2128 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2130 if (op->ldpc_dec.code_block_mode) {
2131 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2133 c = op->ldpc_dec.tb_params.c;
2134 r = op->ldpc_dec.tb_params.r;
2135 for (i = 0; i < c-r; i++)
2136 tb_size += sys_cols * op->ldpc_dec.z_c
2137 - op->ldpc_dec.n_filler;
2143 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2146 uint32_t c, r, tb_size = 0;
2148 if (op->turbo_enc.code_block_mode) {
2149 tb_size = op->turbo_enc.tb_params.k_neg;
2151 c = op->turbo_enc.tb_params.c;
2152 r = op->turbo_enc.tb_params.r;
2153 for (i = 0; i < c-r; i++)
2154 tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2155 op->turbo_enc.tb_params.k_neg :
2156 op->turbo_enc.tb_params.k_pos;
2162 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2165 uint32_t c, r, tb_size = 0;
2166 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2168 if (op->turbo_enc.code_block_mode) {
2169 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2171 c = op->turbo_enc.tb_params.c;
2172 r = op->turbo_enc.tb_params.r;
2173 for (i = 0; i < c-r; i++)
2174 tb_size += sys_cols * op->ldpc_enc.z_c
2175 - op->ldpc_enc.n_filler;
2182 init_test_op_params(struct test_op_params *op_params,
2183 enum rte_bbdev_op_type op_type, const int expected_status,
2184 const int vector_mask, struct rte_mempool *ops_mp,
2185 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2188 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2189 op_type == RTE_BBDEV_OP_LDPC_DEC)
2190 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2191 &op_params->ref_dec_op, 1);
2193 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2194 &op_params->ref_enc_op, 1);
2196 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2198 op_params->mp = ops_mp;
2199 op_params->burst_sz = burst_sz;
2200 op_params->num_to_process = num_to_process;
2201 op_params->num_lcores = num_lcores;
2202 op_params->vector_mask = vector_mask;
2203 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2204 op_type == RTE_BBDEV_OP_LDPC_DEC)
2205 op_params->ref_dec_op->status = expected_status;
2206 else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2207 || op_type == RTE_BBDEV_OP_LDPC_ENC)
2208 op_params->ref_enc_op->status = expected_status;
2213 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2214 struct test_op_params *op_params)
2216 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2218 struct active_device *ad;
2219 unsigned int burst_sz = get_burst_sz();
2220 enum rte_bbdev_op_type op_type = test_vector.op_type;
2221 const struct rte_bbdev_op_cap *capabilities = NULL;
2223 ad = &active_devs[dev_id];
2225 /* Check if device supports op_type */
2226 if (!is_avail_op(ad, test_vector.op_type))
2227 return TEST_SUCCESS;
2229 struct rte_bbdev_info info;
2230 rte_bbdev_info_get(ad->dev_id, &info);
2231 socket_id = GET_SOCKET(info.socket_id);
2233 f_ret = create_mempools(ad, socket_id, op_type,
2235 if (f_ret != TEST_SUCCESS) {
2236 printf("Couldn't create mempools");
2239 if (op_type == RTE_BBDEV_OP_NONE)
2240 op_type = RTE_BBDEV_OP_TURBO_ENC;
2242 f_ret = init_test_op_params(op_params, test_vector.op_type,
2243 test_vector.expected_status,
2249 if (f_ret != TEST_SUCCESS) {
2250 printf("Couldn't init test op params");
2255 /* Find capabilities */
2256 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2257 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2258 if (cap->type == test_vector.op_type) {
2264 TEST_ASSERT_NOT_NULL(capabilities,
2265 "Couldn't find capabilities");
2267 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2268 create_reference_dec_op(op_params->ref_dec_op);
2269 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2270 create_reference_enc_op(op_params->ref_enc_op);
2271 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2272 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2273 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2274 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2276 for (i = 0; i < ad->nb_queues; ++i) {
2277 f_ret = fill_queue_buffers(op_params,
2279 ad->hard_out_mbuf_pool,
2280 ad->soft_out_mbuf_pool,
2281 ad->harq_in_mbuf_pool,
2282 ad->harq_out_mbuf_pool,
2285 info.drv.min_alignment,
2287 if (f_ret != TEST_SUCCESS) {
2288 printf("Couldn't init queue buffers");
2293 /* Run test case function */
2294 t_ret = test_case_func(ad, op_params);
2296 /* Free active device resources and return */
2297 free_buffers(ad, op_params);
2301 free_buffers(ad, op_params);
2305 /* Run given test function per active device per supported op type
2309 run_test_case(test_case_function *test_case_func)
2314 /* Alloc op_params */
2315 struct test_op_params *op_params = rte_zmalloc(NULL,
2316 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2317 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2318 RTE_ALIGN(sizeof(struct test_op_params),
2319 RTE_CACHE_LINE_SIZE));
2321 /* For each device run test case function */
2322 for (dev = 0; dev < nb_active_devs; ++dev)
2323 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2325 rte_free(op_params);
2331 /* Push back the HARQ output from DDR to host */
2333 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2334 struct rte_bbdev_dec_op **ops,
2338 int save_status, ret;
2339 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2340 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2341 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2342 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2343 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2344 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2345 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2346 for (j = 0; j < n; ++j) {
2347 if ((loopback && mem_out) || hc_out) {
2348 save_status = ops[j]->status;
2349 ops[j]->ldpc_dec.op_flags =
2350 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2351 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2353 ops[j]->ldpc_dec.op_flags +=
2354 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2355 ops[j]->ldpc_dec.harq_combined_input.offset =
2357 ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2358 harq_offset += HARQ_INCR;
2360 ops[j]->ldpc_dec.harq_combined_input.length =
2361 ops[j]->ldpc_dec.harq_combined_output.length;
2362 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2366 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2369 ops[j]->ldpc_dec.op_flags = flags;
2370 ops[j]->status = save_status;
2376 * Push back the HARQ output from HW DDR to Host
2377 * Preload HARQ memory input and adjust HARQ offset
2380 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2381 struct rte_bbdev_dec_op **ops, const uint16_t n,
2386 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2387 struct rte_bbdev_op_data save_hc_in, save_hc_out;
2388 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2389 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2390 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2391 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2392 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2393 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2394 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2395 for (j = 0; j < n; ++j) {
2396 if ((mem_in || hc_in) && preload) {
2397 save_hc_in = ops[j]->ldpc_dec.harq_combined_input;
2398 save_hc_out = ops[j]->ldpc_dec.harq_combined_output;
2399 ops[j]->ldpc_dec.op_flags =
2400 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2401 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2403 ops[j]->ldpc_dec.op_flags +=
2404 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2405 ops[j]->ldpc_dec.harq_combined_output.offset =
2407 ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2408 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2412 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2413 dev_id, queue_id, &ops_deq[j], 1);
2414 ops[j]->ldpc_dec.op_flags = flags;
2415 ops[j]->ldpc_dec.harq_combined_input = save_hc_in;
2416 ops[j]->ldpc_dec.harq_combined_output = save_hc_out;
2418 /* Adjust HARQ offset when we reach external DDR */
2419 if (mem_in || hc_in)
2420 ops[j]->ldpc_dec.harq_combined_input.offset
2422 if (mem_out || hc_out)
2423 ops[j]->ldpc_dec.harq_combined_output.offset
2425 harq_offset += HARQ_INCR;
2430 dequeue_event_callback(uint16_t dev_id,
2431 enum rte_bbdev_event_type event, void *cb_arg,
2436 uint64_t total_time;
2437 uint16_t deq, burst_sz, num_ops;
2438 uint16_t queue_id = *(uint16_t *) ret_param;
2439 struct rte_bbdev_info info;
2441 struct thread_params *tp = cb_arg;
2443 /* Find matching thread params using queue_id */
2444 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2445 if (tp->queue_id == queue_id)
2448 if (i == MAX_QUEUES) {
2449 printf("%s: Queue_id from interrupt details was not found!\n",
2454 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2455 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2457 "Dequeue interrupt handler called for incorrect event!\n");
2461 burst_sz = rte_atomic16_read(&tp->burst_sz);
2462 num_ops = tp->op_params->num_to_process;
2464 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2465 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2467 rte_atomic16_read(&tp->nb_dequeued)],
2469 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2470 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2472 rte_atomic16_read(&tp->nb_dequeued)],
2474 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2475 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2477 rte_atomic16_read(&tp->nb_dequeued)],
2479 else /*RTE_BBDEV_OP_TURBO_ENC*/
2480 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2482 rte_atomic16_read(&tp->nb_dequeued)],
2485 if (deq < burst_sz) {
2487 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2489 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2493 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2494 rte_atomic16_add(&tp->nb_dequeued, deq);
2498 total_time = rte_rdtsc_precise() - tp->start_time;
2500 rte_bbdev_info_get(dev_id, &info);
2504 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2505 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2506 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2507 tp->op_params->vector_mask);
2508 /* get the max of iter_count for all dequeued ops */
2509 for (i = 0; i < num_ops; ++i)
2510 tp->iter_count = RTE_MAX(
2511 tp->dec_ops[i]->turbo_dec.iter_count,
2513 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2514 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2515 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2516 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2517 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2518 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2519 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2520 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2521 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2522 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2523 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2524 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2525 tp->op_params->vector_mask);
2526 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2530 printf("Buffers validation failed\n");
2531 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2534 switch (test_vector.op_type) {
2535 case RTE_BBDEV_OP_TURBO_DEC:
2536 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2538 case RTE_BBDEV_OP_TURBO_ENC:
2539 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2541 case RTE_BBDEV_OP_LDPC_DEC:
2542 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2544 case RTE_BBDEV_OP_LDPC_ENC:
2545 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2547 case RTE_BBDEV_OP_NONE:
2551 printf("Unknown op type: %d\n", test_vector.op_type);
2552 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2556 tp->ops_per_sec += ((double)num_ops) /
2557 ((double)total_time / (double)rte_get_tsc_hz());
2558 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2559 ((double)total_time / (double)rte_get_tsc_hz());
2561 rte_atomic16_add(&tp->nb_dequeued, deq);
2565 throughput_intr_lcore_dec(void *arg)
2567 struct thread_params *tp = arg;
2568 unsigned int enqueued;
2569 const uint16_t queue_id = tp->queue_id;
2570 const uint16_t burst_sz = tp->op_params->burst_sz;
2571 const uint16_t num_to_process = tp->op_params->num_to_process;
2572 struct rte_bbdev_dec_op *ops[num_to_process];
2573 struct test_buffers *bufs = NULL;
2574 struct rte_bbdev_info info;
2576 uint16_t num_to_enq, enq;
2578 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2579 "BURST_SIZE should be <= %u", MAX_BURST);
2581 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2582 "Failed to enable interrupts for dev: %u, queue_id: %u",
2583 tp->dev_id, queue_id);
2585 rte_bbdev_info_get(tp->dev_id, &info);
2587 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2588 "NUM_OPS cannot exceed %u for this device",
2589 info.drv.queue_size_lim);
2591 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2593 rte_atomic16_clear(&tp->processing_status);
2594 rte_atomic16_clear(&tp->nb_dequeued);
2596 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2599 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2601 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2603 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2604 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2605 bufs->hard_outputs, bufs->soft_outputs,
2606 tp->op_params->ref_dec_op);
2608 /* Set counter to validate the ordering */
2609 for (j = 0; j < num_to_process; ++j)
2610 ops[j]->opaque_data = (void *)(uintptr_t)j;
2612 for (j = 0; j < TEST_REPETITIONS; ++j) {
2613 for (i = 0; i < num_to_process; ++i)
2614 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2616 tp->start_time = rte_rdtsc_precise();
2617 for (enqueued = 0; enqueued < num_to_process;) {
2618 num_to_enq = burst_sz;
2620 if (unlikely(num_to_process - enqueued < num_to_enq))
2621 num_to_enq = num_to_process - enqueued;
2625 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2626 queue_id, &ops[enqueued],
2628 } while (unlikely(num_to_enq != enq));
2631 /* Write to thread burst_sz current number of enqueued
2632 * descriptors. It ensures that proper number of
2633 * descriptors will be dequeued in callback
2634 * function - needed for last batch in case where
2635 * the number of operations is not a multiple of
2638 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2640 /* Wait until processing of previous batch is
2643 while (rte_atomic16_read(&tp->nb_dequeued) !=
2647 if (j != TEST_REPETITIONS - 1)
2648 rte_atomic16_clear(&tp->nb_dequeued);
2651 return TEST_SUCCESS;
2655 throughput_intr_lcore_enc(void *arg)
2657 struct thread_params *tp = arg;
2658 unsigned int enqueued;
2659 const uint16_t queue_id = tp->queue_id;
2660 const uint16_t burst_sz = tp->op_params->burst_sz;
2661 const uint16_t num_to_process = tp->op_params->num_to_process;
2662 struct rte_bbdev_enc_op *ops[num_to_process];
2663 struct test_buffers *bufs = NULL;
2664 struct rte_bbdev_info info;
2666 uint16_t num_to_enq, enq;
2668 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2669 "BURST_SIZE should be <= %u", MAX_BURST);
2671 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2672 "Failed to enable interrupts for dev: %u, queue_id: %u",
2673 tp->dev_id, queue_id);
2675 rte_bbdev_info_get(tp->dev_id, &info);
2677 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2678 "NUM_OPS cannot exceed %u for this device",
2679 info.drv.queue_size_lim);
2681 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2683 rte_atomic16_clear(&tp->processing_status);
2684 rte_atomic16_clear(&tp->nb_dequeued);
2686 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2689 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2691 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2693 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2694 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2695 bufs->hard_outputs, tp->op_params->ref_enc_op);
2697 /* Set counter to validate the ordering */
2698 for (j = 0; j < num_to_process; ++j)
2699 ops[j]->opaque_data = (void *)(uintptr_t)j;
2701 for (j = 0; j < TEST_REPETITIONS; ++j) {
2702 for (i = 0; i < num_to_process; ++i)
2703 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2705 tp->start_time = rte_rdtsc_precise();
2706 for (enqueued = 0; enqueued < num_to_process;) {
2707 num_to_enq = burst_sz;
2709 if (unlikely(num_to_process - enqueued < num_to_enq))
2710 num_to_enq = num_to_process - enqueued;
2714 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2715 queue_id, &ops[enqueued],
2717 } while (unlikely(enq != num_to_enq));
2720 /* Write to thread burst_sz current number of enqueued
2721 * descriptors. It ensures that proper number of
2722 * descriptors will be dequeued in callback
2723 * function - needed for last batch in case where
2724 * the number of operations is not a multiple of
2727 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2729 /* Wait until processing of previous batch is
2732 while (rte_atomic16_read(&tp->nb_dequeued) !=
2736 if (j != TEST_REPETITIONS - 1)
2737 rte_atomic16_clear(&tp->nb_dequeued);
2740 return TEST_SUCCESS;
2744 throughput_pmd_lcore_dec(void *arg)
2746 struct thread_params *tp = arg;
2748 uint64_t total_time = 0, start_time;
2749 const uint16_t queue_id = tp->queue_id;
2750 const uint16_t burst_sz = tp->op_params->burst_sz;
2751 const uint16_t num_ops = tp->op_params->num_to_process;
2752 struct rte_bbdev_dec_op *ops_enq[num_ops];
2753 struct rte_bbdev_dec_op *ops_deq[num_ops];
2754 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2755 struct test_buffers *bufs = NULL;
2757 struct rte_bbdev_info info;
2758 uint16_t num_to_enq;
2760 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2761 "BURST_SIZE should be <= %u", MAX_BURST);
2763 rte_bbdev_info_get(tp->dev_id, &info);
2765 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2766 "NUM_OPS cannot exceed %u for this device",
2767 info.drv.queue_size_lim);
2769 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2771 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2774 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2775 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2777 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2778 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2779 bufs->hard_outputs, bufs->soft_outputs, ref_op);
2781 /* Set counter to validate the ordering */
2782 for (j = 0; j < num_ops; ++j)
2783 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2785 for (i = 0; i < TEST_REPETITIONS; ++i) {
2787 for (j = 0; j < num_ops; ++j)
2788 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
2790 start_time = rte_rdtsc_precise();
2792 for (enq = 0, deq = 0; enq < num_ops;) {
2793 num_to_enq = burst_sz;
2795 if (unlikely(num_ops - enq < num_to_enq))
2796 num_to_enq = num_ops - enq;
2798 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2799 queue_id, &ops_enq[enq], num_to_enq);
2801 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
2802 queue_id, &ops_deq[deq], enq - deq);
2805 /* dequeue the remaining */
2807 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
2808 queue_id, &ops_deq[deq], enq - deq);
2811 total_time += rte_rdtsc_precise() - start_time;
2815 /* get the max of iter_count for all dequeued ops */
2816 for (i = 0; i < num_ops; ++i) {
2817 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
2821 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2822 ret = validate_dec_op(ops_deq, num_ops, ref_op,
2823 tp->op_params->vector_mask);
2824 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2827 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
2829 double tb_len_bits = calc_dec_TB_size(ref_op);
2831 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2832 ((double)total_time / (double)rte_get_tsc_hz());
2833 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
2834 1000000.0) / ((double)total_time /
2835 (double)rte_get_tsc_hz());
2837 return TEST_SUCCESS;
2841 bler_pmd_lcore_ldpc_dec(void *arg)
2843 struct thread_params *tp = arg;
2845 uint64_t total_time = 0, start_time;
2846 const uint16_t queue_id = tp->queue_id;
2847 const uint16_t burst_sz = tp->op_params->burst_sz;
2848 const uint16_t num_ops = tp->op_params->num_to_process;
2849 struct rte_bbdev_dec_op *ops_enq[num_ops];
2850 struct rte_bbdev_dec_op *ops_deq[num_ops];
2851 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2852 struct test_buffers *bufs = NULL;
2854 float parity_bler = 0;
2855 struct rte_bbdev_info info;
2856 uint16_t num_to_enq;
2857 bool extDdr = check_bit(ldpc_cap_flags,
2858 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
2859 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2860 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2861 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2862 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2864 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2865 "BURST_SIZE should be <= %u", MAX_BURST);
2867 rte_bbdev_info_get(tp->dev_id, &info);
2869 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2870 "NUM_OPS cannot exceed %u for this device",
2871 info.drv.queue_size_lim);
2873 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2875 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2878 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2879 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2881 /* For BLER tests we need to enable early termination */
2882 if (!check_bit(ref_op->ldpc_dec.op_flags,
2883 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
2884 ref_op->ldpc_dec.op_flags +=
2885 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
2886 ref_op->ldpc_dec.iter_max = get_iter_max();
2887 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
2889 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2890 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2891 bufs->hard_outputs, bufs->soft_outputs,
2892 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2893 generate_llr_input(num_ops, bufs->inputs, ref_op);
2895 /* Set counter to validate the ordering */
2896 for (j = 0; j < num_ops; ++j)
2897 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2899 for (i = 0; i < 1; ++i) { /* Could add more iterations */
2900 for (j = 0; j < num_ops; ++j) {
2903 ops_enq[j]->ldpc_dec.hard_output.data);
2904 if (hc_out || loopback)
2906 ops_enq[j]->ldpc_dec.harq_combined_output.data);
2909 bool preload = i == (TEST_REPETITIONS - 1);
2910 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
2913 start_time = rte_rdtsc_precise();
2915 for (enq = 0, deq = 0; enq < num_ops;) {
2916 num_to_enq = burst_sz;
2918 if (unlikely(num_ops - enq < num_to_enq))
2919 num_to_enq = num_ops - enq;
2921 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
2922 queue_id, &ops_enq[enq], num_to_enq);
2924 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
2925 queue_id, &ops_deq[deq], enq - deq);
2928 /* dequeue the remaining */
2930 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
2931 queue_id, &ops_deq[deq], enq - deq);
2934 total_time += rte_rdtsc_precise() - start_time;
2938 tp->iter_average = 0;
2939 /* get the max of iter_count for all dequeued ops */
2940 for (i = 0; i < num_ops; ++i) {
2941 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
2943 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
2944 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
2948 parity_bler /= num_ops; /* This one is based on SYND */
2949 tp->iter_average /= num_ops;
2950 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
2952 if (test_vector.op_type != RTE_BBDEV_OP_NONE
2956 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
2957 tp->op_params->vector_mask);
2958 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2961 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
2963 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
2964 tp->ops_per_sec = ((double)num_ops * 1) /
2965 ((double)total_time / (double)rte_get_tsc_hz());
2966 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
2967 1000000.0) / ((double)total_time /
2968 (double)rte_get_tsc_hz());
2970 return TEST_SUCCESS;
2974 throughput_pmd_lcore_ldpc_dec(void *arg)
2976 struct thread_params *tp = arg;
2978 uint64_t total_time = 0, start_time;
2979 const uint16_t queue_id = tp->queue_id;
2980 const uint16_t burst_sz = tp->op_params->burst_sz;
2981 const uint16_t num_ops = tp->op_params->num_to_process;
2982 struct rte_bbdev_dec_op *ops_enq[num_ops];
2983 struct rte_bbdev_dec_op *ops_deq[num_ops];
2984 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2985 struct test_buffers *bufs = NULL;
2987 struct rte_bbdev_info info;
2988 uint16_t num_to_enq;
2989 bool extDdr = check_bit(ldpc_cap_flags,
2990 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
2991 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2992 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2993 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2994 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2996 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2997 "BURST_SIZE should be <= %u", MAX_BURST);
2999 rte_bbdev_info_get(tp->dev_id, &info);
3001 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3002 "NUM_OPS cannot exceed %u for this device",
3003 info.drv.queue_size_lim);
3005 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3007 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3010 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3011 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3013 /* For throughput tests we need to disable early termination */
3014 if (check_bit(ref_op->ldpc_dec.op_flags,
3015 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3016 ref_op->ldpc_dec.op_flags -=
3017 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3018 ref_op->ldpc_dec.iter_max = get_iter_max();
3019 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3021 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3022 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3023 bufs->hard_outputs, bufs->soft_outputs,
3024 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3026 /* Set counter to validate the ordering */
3027 for (j = 0; j < num_ops; ++j)
3028 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3030 for (i = 0; i < TEST_REPETITIONS; ++i) {
3031 for (j = 0; j < num_ops; ++j) {
3034 ops_enq[j]->ldpc_dec.hard_output.data);
3035 if (hc_out || loopback)
3037 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3040 bool preload = i == (TEST_REPETITIONS - 1);
3041 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3044 start_time = rte_rdtsc_precise();
3046 for (enq = 0, deq = 0; enq < num_ops;) {
3047 num_to_enq = burst_sz;
3049 if (unlikely(num_ops - enq < num_to_enq))
3050 num_to_enq = num_ops - enq;
3052 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3053 queue_id, &ops_enq[enq], num_to_enq);
3055 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3056 queue_id, &ops_deq[deq], enq - deq);
3059 /* dequeue the remaining */
3061 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3062 queue_id, &ops_deq[deq], enq - deq);
3065 total_time += rte_rdtsc_precise() - start_time;
3069 /* get the max of iter_count for all dequeued ops */
3070 for (i = 0; i < num_ops; ++i) {
3071 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3075 /* Read loopback is not thread safe */
3076 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3079 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3080 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3081 tp->op_params->vector_mask);
3082 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3085 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3087 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3089 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3090 ((double)total_time / (double)rte_get_tsc_hz());
3091 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3092 1000000.0) / ((double)total_time /
3093 (double)rte_get_tsc_hz());
3095 return TEST_SUCCESS;
3099 throughput_pmd_lcore_enc(void *arg)
3101 struct thread_params *tp = arg;
3103 uint64_t total_time = 0, start_time;
3104 const uint16_t queue_id = tp->queue_id;
3105 const uint16_t burst_sz = tp->op_params->burst_sz;
3106 const uint16_t num_ops = tp->op_params->num_to_process;
3107 struct rte_bbdev_enc_op *ops_enq[num_ops];
3108 struct rte_bbdev_enc_op *ops_deq[num_ops];
3109 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3110 struct test_buffers *bufs = NULL;
3112 struct rte_bbdev_info info;
3113 uint16_t num_to_enq;
3115 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3116 "BURST_SIZE should be <= %u", MAX_BURST);
3118 rte_bbdev_info_get(tp->dev_id, &info);
3120 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3121 "NUM_OPS cannot exceed %u for this device",
3122 info.drv.queue_size_lim);
3124 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3126 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3129 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3131 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3133 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3134 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3135 bufs->hard_outputs, ref_op);
3137 /* Set counter to validate the ordering */
3138 for (j = 0; j < num_ops; ++j)
3139 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3141 for (i = 0; i < TEST_REPETITIONS; ++i) {
3143 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3144 for (j = 0; j < num_ops; ++j)
3145 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3147 start_time = rte_rdtsc_precise();
3149 for (enq = 0, deq = 0; enq < num_ops;) {
3150 num_to_enq = burst_sz;
3152 if (unlikely(num_ops - enq < num_to_enq))
3153 num_to_enq = num_ops - enq;
3155 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3156 queue_id, &ops_enq[enq], num_to_enq);
3158 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3159 queue_id, &ops_deq[deq], enq - deq);
3162 /* dequeue the remaining */
3164 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3165 queue_id, &ops_deq[deq], enq - deq);
3168 total_time += rte_rdtsc_precise() - start_time;
3171 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3172 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3173 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3176 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3178 double tb_len_bits = calc_enc_TB_size(ref_op);
3180 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3181 ((double)total_time / (double)rte_get_tsc_hz());
3182 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3183 / 1000000.0) / ((double)total_time /
3184 (double)rte_get_tsc_hz());
3186 return TEST_SUCCESS;
3190 throughput_pmd_lcore_ldpc_enc(void *arg)
3192 struct thread_params *tp = arg;
3194 uint64_t total_time = 0, start_time;
3195 const uint16_t queue_id = tp->queue_id;
3196 const uint16_t burst_sz = tp->op_params->burst_sz;
3197 const uint16_t num_ops = tp->op_params->num_to_process;
3198 struct rte_bbdev_enc_op *ops_enq[num_ops];
3199 struct rte_bbdev_enc_op *ops_deq[num_ops];
3200 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3201 struct test_buffers *bufs = NULL;
3203 struct rte_bbdev_info info;
3204 uint16_t num_to_enq;
3206 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3207 "BURST_SIZE should be <= %u", MAX_BURST);
3209 rte_bbdev_info_get(tp->dev_id, &info);
3211 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3212 "NUM_OPS cannot exceed %u for this device",
3213 info.drv.queue_size_lim);
3215 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3217 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3220 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3222 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3224 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3225 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3226 bufs->hard_outputs, ref_op);
3228 /* Set counter to validate the ordering */
3229 for (j = 0; j < num_ops; ++j)
3230 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3232 for (i = 0; i < TEST_REPETITIONS; ++i) {
3234 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3235 for (j = 0; j < num_ops; ++j)
3236 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3238 start_time = rte_rdtsc_precise();
3240 for (enq = 0, deq = 0; enq < num_ops;) {
3241 num_to_enq = burst_sz;
3243 if (unlikely(num_ops - enq < num_to_enq))
3244 num_to_enq = num_ops - enq;
3246 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3247 queue_id, &ops_enq[enq], num_to_enq);
3249 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3250 queue_id, &ops_deq[deq], enq - deq);
3253 /* dequeue the remaining */
3255 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3256 queue_id, &ops_deq[deq], enq - deq);
3259 total_time += rte_rdtsc_precise() - start_time;
3262 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3263 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3264 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3267 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3269 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3271 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3272 ((double)total_time / (double)rte_get_tsc_hz());
3273 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3274 / 1000000.0) / ((double)total_time /
3275 (double)rte_get_tsc_hz());
3277 return TEST_SUCCESS;
3281 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3283 unsigned int iter = 0;
3284 double total_mops = 0, total_mbps = 0;
3286 for (iter = 0; iter < used_cores; iter++) {
3288 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3289 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3290 t_params[iter].mbps);
3291 total_mops += t_params[iter].ops_per_sec;
3292 total_mbps += t_params[iter].mbps;
3295 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3296 used_cores, total_mops, total_mbps);
3299 /* Aggregate the performance results over the number of cores used */
3301 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3303 unsigned int core_idx = 0;
3304 double total_mops = 0, total_mbps = 0;
3305 uint8_t iter_count = 0;
3307 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3309 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3310 t_params[core_idx].lcore_id,
3311 t_params[core_idx].ops_per_sec,
3312 t_params[core_idx].mbps,
3313 t_params[core_idx].iter_count);
3314 total_mops += t_params[core_idx].ops_per_sec;
3315 total_mbps += t_params[core_idx].mbps;
3316 iter_count = RTE_MAX(iter_count,
3317 t_params[core_idx].iter_count);
3320 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3321 used_cores, total_mops, total_mbps, iter_count);
3324 /* Aggregate the performance results over the number of cores used */
3326 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3328 unsigned int core_idx = 0;
3329 double total_mbps = 0, total_bler = 0, total_iter = 0;
3330 double snr = get_snr();
3332 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3333 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3334 t_params[core_idx].lcore_id,
3335 t_params[core_idx].bler * 100,
3336 t_params[core_idx].iter_average,
3337 t_params[core_idx].mbps,
3338 get_vector_filename());
3339 total_mbps += t_params[core_idx].mbps;
3340 total_bler += t_params[core_idx].bler;
3341 total_iter += t_params[core_idx].iter_average;
3343 total_bler /= used_cores;
3344 total_iter /= used_cores;
3346 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3347 snr, total_bler * 100, total_iter, get_iter_max(),
3348 total_mbps, get_vector_filename());
3352 * Test function that determines BLER wireless performance
3355 bler_test(struct active_device *ad,
3356 struct test_op_params *op_params)
3359 unsigned int lcore_id, used_cores = 0;
3360 struct thread_params *t_params;
3361 struct rte_bbdev_info info;
3362 lcore_function_t *bler_function;
3363 uint16_t num_lcores;
3364 const char *op_type_str;
3366 rte_bbdev_info_get(ad->dev_id, &info);
3368 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3369 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3370 test_vector.op_type);
3372 printf("+ ------------------------------------------------------- +\n");
3373 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3374 info.dev_name, ad->nb_queues, op_params->burst_sz,
3375 op_params->num_to_process, op_params->num_lcores,
3377 intr_enabled ? "Interrupt mode" : "PMD mode",
3378 (double)rte_get_tsc_hz() / 1000000000.0);
3380 /* Set number of lcores */
3381 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3383 : op_params->num_lcores;
3385 /* Allocate memory for thread parameters structure */
3386 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3387 RTE_CACHE_LINE_SIZE);
3388 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3389 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3390 RTE_CACHE_LINE_SIZE));
3392 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3393 bler_function = bler_pmd_lcore_ldpc_dec;
3395 return TEST_SKIPPED;
3397 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3399 /* Master core is set at first entry */
3400 t_params[0].dev_id = ad->dev_id;
3401 t_params[0].lcore_id = rte_lcore_id();
3402 t_params[0].op_params = op_params;
3403 t_params[0].queue_id = ad->queue_ids[used_cores++];
3404 t_params[0].iter_count = 0;
3406 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3407 if (used_cores >= num_lcores)
3410 t_params[used_cores].dev_id = ad->dev_id;
3411 t_params[used_cores].lcore_id = lcore_id;
3412 t_params[used_cores].op_params = op_params;
3413 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3414 t_params[used_cores].iter_count = 0;
3416 rte_eal_remote_launch(bler_function,
3417 &t_params[used_cores++], lcore_id);
3420 rte_atomic16_set(&op_params->sync, SYNC_START);
3421 ret = bler_function(&t_params[0]);
3423 /* Master core is always used */
3424 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3425 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3427 print_dec_bler(t_params, num_lcores);
3429 /* Return if test failed */
3435 /* Function to print something here*/
3441 * Test function that determines how long an enqueue + dequeue of a burst
3442 * takes on available lcores.
3445 throughput_test(struct active_device *ad,
3446 struct test_op_params *op_params)
3449 unsigned int lcore_id, used_cores = 0;
3450 struct thread_params *t_params, *tp;
3451 struct rte_bbdev_info info;
3452 lcore_function_t *throughput_function;
3453 uint16_t num_lcores;
3454 const char *op_type_str;
3456 rte_bbdev_info_get(ad->dev_id, &info);
3458 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3459 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3460 test_vector.op_type);
3462 printf("+ ------------------------------------------------------- +\n");
3463 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3464 info.dev_name, ad->nb_queues, op_params->burst_sz,
3465 op_params->num_to_process, op_params->num_lcores,
3467 intr_enabled ? "Interrupt mode" : "PMD mode",
3468 (double)rte_get_tsc_hz() / 1000000000.0);
3470 /* Set number of lcores */
3471 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3473 : op_params->num_lcores;
3475 /* Allocate memory for thread parameters structure */
3476 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3477 RTE_CACHE_LINE_SIZE);
3478 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3479 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3480 RTE_CACHE_LINE_SIZE));
3483 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3484 throughput_function = throughput_intr_lcore_dec;
3485 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3486 throughput_function = throughput_intr_lcore_dec;
3487 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3488 throughput_function = throughput_intr_lcore_enc;
3489 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3490 throughput_function = throughput_intr_lcore_enc;
3492 throughput_function = throughput_intr_lcore_enc;
3494 /* Dequeue interrupt callback registration */
3495 ret = rte_bbdev_callback_register(ad->dev_id,
3496 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3503 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3504 throughput_function = throughput_pmd_lcore_dec;
3505 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3506 throughput_function = throughput_pmd_lcore_ldpc_dec;
3507 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3508 throughput_function = throughput_pmd_lcore_enc;
3509 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3510 throughput_function = throughput_pmd_lcore_ldpc_enc;
3512 throughput_function = throughput_pmd_lcore_enc;
3515 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3517 /* Master core is set at first entry */
3518 t_params[0].dev_id = ad->dev_id;
3519 t_params[0].lcore_id = rte_lcore_id();
3520 t_params[0].op_params = op_params;
3521 t_params[0].queue_id = ad->queue_ids[used_cores++];
3522 t_params[0].iter_count = 0;
3524 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3525 if (used_cores >= num_lcores)
3528 t_params[used_cores].dev_id = ad->dev_id;
3529 t_params[used_cores].lcore_id = lcore_id;
3530 t_params[used_cores].op_params = op_params;
3531 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3532 t_params[used_cores].iter_count = 0;
3534 rte_eal_remote_launch(throughput_function,
3535 &t_params[used_cores++], lcore_id);
3538 rte_atomic16_set(&op_params->sync, SYNC_START);
3539 ret = throughput_function(&t_params[0]);
3541 /* Master core is always used */
3542 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3543 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3545 /* Return if test failed */
3551 /* Print throughput if interrupts are disabled and test passed */
3552 if (!intr_enabled) {
3553 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3554 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3555 print_dec_throughput(t_params, num_lcores);
3557 print_enc_throughput(t_params, num_lcores);
3562 /* In interrupt TC we need to wait for the interrupt callback to deqeue
3563 * all pending operations. Skip waiting for queues which reported an
3564 * error using processing_status variable.
3565 * Wait for master lcore operations.
3568 while ((rte_atomic16_read(&tp->nb_dequeued) <
3569 op_params->num_to_process) &&
3570 (rte_atomic16_read(&tp->processing_status) !=
3574 tp->ops_per_sec /= TEST_REPETITIONS;
3575 tp->mbps /= TEST_REPETITIONS;
3576 ret |= (int)rte_atomic16_read(&tp->processing_status);
3578 /* Wait for slave lcores operations */
3579 for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3580 tp = &t_params[used_cores];
3582 while ((rte_atomic16_read(&tp->nb_dequeued) <
3583 op_params->num_to_process) &&
3584 (rte_atomic16_read(&tp->processing_status) !=
3588 tp->ops_per_sec /= TEST_REPETITIONS;
3589 tp->mbps /= TEST_REPETITIONS;
3590 ret |= (int)rte_atomic16_read(&tp->processing_status);
3593 /* Print throughput if test passed */
3595 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3596 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3597 print_dec_throughput(t_params, num_lcores);
3598 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3599 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3600 print_enc_throughput(t_params, num_lcores);
3608 latency_test_dec(struct rte_mempool *mempool,
3609 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3610 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3611 const uint16_t num_to_process, uint16_t burst_sz,
3612 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3614 int ret = TEST_SUCCESS;
3615 uint16_t i, j, dequeued;
3616 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3617 uint64_t start_time = 0, last_time = 0;
3619 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3620 uint16_t enq = 0, deq = 0;
3621 bool first_time = true;
3624 if (unlikely(num_to_process - dequeued < burst_sz))
3625 burst_sz = num_to_process - dequeued;
3627 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3628 TEST_ASSERT_SUCCESS(ret,
3629 "rte_bbdev_dec_op_alloc_bulk() failed");
3630 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3631 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3637 /* Set counter to validate the ordering */
3638 for (j = 0; j < burst_sz; ++j)
3639 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3641 start_time = rte_rdtsc_precise();
3643 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
3645 TEST_ASSERT(enq == burst_sz,
3646 "Error enqueueing burst, expected %u, got %u",
3651 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3652 &ops_deq[deq], burst_sz - deq);
3653 if (likely(first_time && (deq > 0))) {
3654 last_time = rte_rdtsc_precise() - start_time;
3657 } while (unlikely(burst_sz != deq));
3659 *max_time = RTE_MAX(*max_time, last_time);
3660 *min_time = RTE_MIN(*min_time, last_time);
3661 *total_time += last_time;
3663 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3664 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
3666 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3669 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3677 latency_test_ldpc_dec(struct rte_mempool *mempool,
3678 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3679 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3680 const uint16_t num_to_process, uint16_t burst_sz,
3681 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3683 int ret = TEST_SUCCESS;
3684 uint16_t i, j, dequeued;
3685 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3686 uint64_t start_time = 0, last_time = 0;
3687 bool extDdr = ldpc_cap_flags &
3688 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
3690 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3691 uint16_t enq = 0, deq = 0;
3692 bool first_time = true;
3695 if (unlikely(num_to_process - dequeued < burst_sz))
3696 burst_sz = num_to_process - dequeued;
3698 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3699 TEST_ASSERT_SUCCESS(ret,
3700 "rte_bbdev_dec_op_alloc_bulk() failed");
3702 /* For latency tests we need to disable early termination */
3703 if (check_bit(ref_op->ldpc_dec.op_flags,
3704 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3705 ref_op->ldpc_dec.op_flags -=
3706 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3707 ref_op->ldpc_dec.iter_max = get_iter_max();
3708 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3710 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3711 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
3720 preload_harq_ddr(dev_id, queue_id, ops_enq,
3723 /* Set counter to validate the ordering */
3724 for (j = 0; j < burst_sz; ++j)
3725 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3727 start_time = rte_rdtsc_precise();
3729 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
3730 &ops_enq[enq], burst_sz);
3731 TEST_ASSERT(enq == burst_sz,
3732 "Error enqueueing burst, expected %u, got %u",
3737 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
3738 &ops_deq[deq], burst_sz - deq);
3739 if (likely(first_time && (deq > 0))) {
3740 last_time = rte_rdtsc_precise() - start_time;
3743 } while (unlikely(burst_sz != deq));
3745 *max_time = RTE_MAX(*max_time, last_time);
3746 *min_time = RTE_MIN(*min_time, last_time);
3747 *total_time += last_time;
3750 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
3752 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3753 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
3755 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3758 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3765 latency_test_enc(struct rte_mempool *mempool,
3766 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
3767 uint16_t dev_id, uint16_t queue_id,
3768 const uint16_t num_to_process, uint16_t burst_sz,
3769 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3771 int ret = TEST_SUCCESS;
3772 uint16_t i, j, dequeued;
3773 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3774 uint64_t start_time = 0, last_time = 0;
3776 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3777 uint16_t enq = 0, deq = 0;
3778 bool first_time = true;
3781 if (unlikely(num_to_process - dequeued < burst_sz))
3782 burst_sz = num_to_process - dequeued;
3784 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3785 TEST_ASSERT_SUCCESS(ret,
3786 "rte_bbdev_enc_op_alloc_bulk() failed");
3787 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3788 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
3793 /* Set counter to validate the ordering */
3794 for (j = 0; j < burst_sz; ++j)
3795 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3797 start_time = rte_rdtsc_precise();
3799 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
3801 TEST_ASSERT(enq == burst_sz,
3802 "Error enqueueing burst, expected %u, got %u",
3807 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
3808 &ops_deq[deq], burst_sz - deq);
3809 if (likely(first_time && (deq > 0))) {
3810 last_time += rte_rdtsc_precise() - start_time;
3813 } while (unlikely(burst_sz != deq));
3815 *max_time = RTE_MAX(*max_time, last_time);
3816 *min_time = RTE_MIN(*min_time, last_time);
3817 *total_time += last_time;
3819 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3820 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
3821 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3824 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
3832 latency_test_ldpc_enc(struct rte_mempool *mempool,
3833 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
3834 uint16_t dev_id, uint16_t queue_id,
3835 const uint16_t num_to_process, uint16_t burst_sz,
3836 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3838 int ret = TEST_SUCCESS;
3839 uint16_t i, j, dequeued;
3840 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3841 uint64_t start_time = 0, last_time = 0;
3843 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3844 uint16_t enq = 0, deq = 0;
3845 bool first_time = true;
3848 if (unlikely(num_to_process - dequeued < burst_sz))
3849 burst_sz = num_to_process - dequeued;
3851 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3852 TEST_ASSERT_SUCCESS(ret,
3853 "rte_bbdev_enc_op_alloc_bulk() failed");
3854 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3855 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
3860 /* Set counter to validate the ordering */
3861 for (j = 0; j < burst_sz; ++j)
3862 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3864 start_time = rte_rdtsc_precise();
3866 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
3867 &ops_enq[enq], burst_sz);
3868 TEST_ASSERT(enq == burst_sz,
3869 "Error enqueueing burst, expected %u, got %u",
3874 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
3875 &ops_deq[deq], burst_sz - deq);
3876 if (likely(first_time && (deq > 0))) {
3877 last_time += rte_rdtsc_precise() - start_time;
3880 } while (unlikely(burst_sz != deq));
3882 *max_time = RTE_MAX(*max_time, last_time);
3883 *min_time = RTE_MIN(*min_time, last_time);
3884 *total_time += last_time;
3886 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3887 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
3888 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3891 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
3899 latency_test(struct active_device *ad,
3900 struct test_op_params *op_params)
3903 uint16_t burst_sz = op_params->burst_sz;
3904 const uint16_t num_to_process = op_params->num_to_process;
3905 const enum rte_bbdev_op_type op_type = test_vector.op_type;
3906 const uint16_t queue_id = ad->queue_ids[0];
3907 struct test_buffers *bufs = NULL;
3908 struct rte_bbdev_info info;
3909 uint64_t total_time, min_time, max_time;
3910 const char *op_type_str;
3912 total_time = max_time = 0;
3913 min_time = UINT64_MAX;
3915 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3916 "BURST_SIZE should be <= %u", MAX_BURST);
3918 rte_bbdev_info_get(ad->dev_id, &info);
3919 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3921 op_type_str = rte_bbdev_op_type_str(op_type);
3922 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
3924 printf("+ ------------------------------------------------------- +\n");
3925 printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
3926 info.dev_name, burst_sz, num_to_process, op_type_str);
3928 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
3929 iter = latency_test_dec(op_params->mp, bufs,
3930 op_params->ref_dec_op, op_params->vector_mask,
3931 ad->dev_id, queue_id, num_to_process,
3932 burst_sz, &total_time, &min_time, &max_time);
3933 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
3934 iter = latency_test_enc(op_params->mp, bufs,
3935 op_params->ref_enc_op, ad->dev_id, queue_id,
3936 num_to_process, burst_sz, &total_time,
3937 &min_time, &max_time);
3938 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
3939 iter = latency_test_ldpc_enc(op_params->mp, bufs,
3940 op_params->ref_enc_op, ad->dev_id, queue_id,
3941 num_to_process, burst_sz, &total_time,
3942 &min_time, &max_time);
3943 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
3944 iter = latency_test_ldpc_dec(op_params->mp, bufs,
3945 op_params->ref_dec_op, op_params->vector_mask,
3946 ad->dev_id, queue_id, num_to_process,
3947 burst_sz, &total_time, &min_time, &max_time);
3949 iter = latency_test_enc(op_params->mp, bufs,
3950 op_params->ref_enc_op,
3951 ad->dev_id, queue_id,
3952 num_to_process, burst_sz, &total_time,
3953 &min_time, &max_time);
3958 printf("Operation latency:\n"
3959 "\tavg: %lg cycles, %lg us\n"
3960 "\tmin: %lg cycles, %lg us\n"
3961 "\tmax: %lg cycles, %lg us\n",
3962 (double)total_time / (double)iter,
3963 (double)(total_time * 1000000) / (double)iter /
3964 (double)rte_get_tsc_hz(), (double)min_time,
3965 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
3966 (double)max_time, (double)(max_time * 1000000) /
3967 (double)rte_get_tsc_hz());
3969 return TEST_SUCCESS;
3972 #ifdef RTE_BBDEV_OFFLOAD_COST
3974 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
3975 struct rte_bbdev_stats *stats)
3977 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
3978 struct rte_bbdev_stats *q_stats;
3980 if (queue_id >= dev->data->num_queues)
3983 q_stats = &dev->data->queues[queue_id].queue_stats;
3985 stats->enqueued_count = q_stats->enqueued_count;
3986 stats->dequeued_count = q_stats->dequeued_count;
3987 stats->enqueue_err_count = q_stats->enqueue_err_count;
3988 stats->dequeue_err_count = q_stats->dequeue_err_count;
3989 stats->acc_offload_cycles = q_stats->acc_offload_cycles;
3995 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
3996 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
3997 uint16_t queue_id, const uint16_t num_to_process,
3998 uint16_t burst_sz, struct test_time_stats *time_st)
4000 int i, dequeued, ret;
4001 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4002 uint64_t enq_start_time, deq_start_time;
4003 uint64_t enq_sw_last_time, deq_last_time;
4004 struct rte_bbdev_stats stats;
4006 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4007 uint16_t enq = 0, deq = 0;
4009 if (unlikely(num_to_process - dequeued < burst_sz))
4010 burst_sz = num_to_process - dequeued;
4012 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4013 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4014 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4020 /* Start time meas for enqueue function offload latency */
4021 enq_start_time = rte_rdtsc_precise();
4023 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4024 &ops_enq[enq], burst_sz - enq);
4025 } while (unlikely(burst_sz != enq));
4027 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4028 TEST_ASSERT_SUCCESS(ret,
4029 "Failed to get stats for queue (%u) of device (%u)",
4032 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4033 stats.acc_offload_cycles;
4034 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4036 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4038 time_st->enq_sw_total_time += enq_sw_last_time;
4040 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4041 stats.acc_offload_cycles);
4042 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4043 stats.acc_offload_cycles);
4044 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4046 /* give time for device to process ops */
4049 /* Start time meas for dequeue function offload latency */
4050 deq_start_time = rte_rdtsc_precise();
4051 /* Dequeue one operation */
4053 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4055 } while (unlikely(deq != 1));
4057 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4058 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4060 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4062 time_st->deq_total_time += deq_last_time;
4064 /* Dequeue remaining operations if needed*/
4065 while (burst_sz != deq)
4066 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4067 &ops_deq[deq], burst_sz - deq);
4069 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4077 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4078 struct test_buffers *bufs,
4079 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4080 uint16_t queue_id, const uint16_t num_to_process,
4081 uint16_t burst_sz, struct test_time_stats *time_st)
4083 int i, dequeued, ret;
4084 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4085 uint64_t enq_start_time, deq_start_time;
4086 uint64_t enq_sw_last_time, deq_last_time;
4087 struct rte_bbdev_stats stats;
4088 bool extDdr = ldpc_cap_flags &
4089 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4091 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4092 uint16_t enq = 0, deq = 0;
4094 if (unlikely(num_to_process - dequeued < burst_sz))
4095 burst_sz = num_to_process - dequeued;
4097 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4098 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4099 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4108 preload_harq_ddr(dev_id, queue_id, ops_enq,
4111 /* Start time meas for enqueue function offload latency */
4112 enq_start_time = rte_rdtsc_precise();
4114 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4115 &ops_enq[enq], burst_sz - enq);
4116 } while (unlikely(burst_sz != enq));
4118 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4119 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4120 TEST_ASSERT_SUCCESS(ret,
4121 "Failed to get stats for queue (%u) of device (%u)",
4124 enq_sw_last_time -= stats.acc_offload_cycles;
4125 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4127 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4129 time_st->enq_sw_total_time += enq_sw_last_time;
4131 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4132 stats.acc_offload_cycles);
4133 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4134 stats.acc_offload_cycles);
4135 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4137 /* give time for device to process ops */
4140 /* Start time meas for dequeue function offload latency */
4141 deq_start_time = rte_rdtsc_precise();
4142 /* Dequeue one operation */
4144 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4146 } while (unlikely(deq != 1));
4148 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4149 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4151 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4153 time_st->deq_total_time += deq_last_time;
4155 /* Dequeue remaining operations if needed*/
4156 while (burst_sz != deq)
4157 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4158 &ops_deq[deq], burst_sz - deq);
4161 /* Read loopback is not thread safe */
4162 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4165 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4173 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4174 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4175 uint16_t queue_id, const uint16_t num_to_process,
4176 uint16_t burst_sz, struct test_time_stats *time_st)
4178 int i, dequeued, ret;
4179 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4180 uint64_t enq_start_time, deq_start_time;
4181 uint64_t enq_sw_last_time, deq_last_time;
4182 struct rte_bbdev_stats stats;
4184 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4185 uint16_t enq = 0, deq = 0;
4187 if (unlikely(num_to_process - dequeued < burst_sz))
4188 burst_sz = num_to_process - dequeued;
4190 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4191 TEST_ASSERT_SUCCESS(ret,
4192 "rte_bbdev_enc_op_alloc_bulk() failed");
4193 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4194 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4199 /* Start time meas for enqueue function offload latency */
4200 enq_start_time = rte_rdtsc_precise();
4202 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4203 &ops_enq[enq], burst_sz - enq);
4204 } while (unlikely(burst_sz != enq));
4206 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4208 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4209 TEST_ASSERT_SUCCESS(ret,
4210 "Failed to get stats for queue (%u) of device (%u)",
4212 enq_sw_last_time -= stats.acc_offload_cycles;
4213 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4215 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4217 time_st->enq_sw_total_time += enq_sw_last_time;
4219 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4220 stats.acc_offload_cycles);
4221 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4222 stats.acc_offload_cycles);
4223 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4225 /* give time for device to process ops */
4228 /* Start time meas for dequeue function offload latency */
4229 deq_start_time = rte_rdtsc_precise();
4230 /* Dequeue one operation */
4232 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4234 } while (unlikely(deq != 1));
4236 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4237 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4239 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4241 time_st->deq_total_time += deq_last_time;
4243 while (burst_sz != deq)
4244 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4245 &ops_deq[deq], burst_sz - deq);
4247 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4255 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4256 struct test_buffers *bufs,
4257 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4258 uint16_t queue_id, const uint16_t num_to_process,
4259 uint16_t burst_sz, struct test_time_stats *time_st)
4261 int i, dequeued, ret;
4262 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4263 uint64_t enq_start_time, deq_start_time;
4264 uint64_t enq_sw_last_time, deq_last_time;
4265 struct rte_bbdev_stats stats;
4267 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4268 uint16_t enq = 0, deq = 0;
4270 if (unlikely(num_to_process - dequeued < burst_sz))
4271 burst_sz = num_to_process - dequeued;
4273 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4274 TEST_ASSERT_SUCCESS(ret,
4275 "rte_bbdev_enc_op_alloc_bulk() failed");
4276 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4277 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4282 /* Start time meas for enqueue function offload latency */
4283 enq_start_time = rte_rdtsc_precise();
4285 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4286 &ops_enq[enq], burst_sz - enq);
4287 } while (unlikely(burst_sz != enq));
4289 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4290 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4291 TEST_ASSERT_SUCCESS(ret,
4292 "Failed to get stats for queue (%u) of device (%u)",
4295 enq_sw_last_time -= stats.acc_offload_cycles;
4296 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4298 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4300 time_st->enq_sw_total_time += enq_sw_last_time;
4302 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4303 stats.acc_offload_cycles);
4304 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4305 stats.acc_offload_cycles);
4306 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4308 /* give time for device to process ops */
4311 /* Start time meas for dequeue function offload latency */
4312 deq_start_time = rte_rdtsc_precise();
4313 /* Dequeue one operation */
4315 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4317 } while (unlikely(deq != 1));
4319 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4320 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4322 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4324 time_st->deq_total_time += deq_last_time;
4326 while (burst_sz != deq)
4327 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4328 &ops_deq[deq], burst_sz - deq);
4330 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4339 offload_cost_test(struct active_device *ad,
4340 struct test_op_params *op_params)
4342 #ifndef RTE_BBDEV_OFFLOAD_COST
4344 RTE_SET_USED(op_params);
4345 printf("Offload latency test is disabled.\n");
4346 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4347 return TEST_SKIPPED;
4350 uint16_t burst_sz = op_params->burst_sz;
4351 const uint16_t num_to_process = op_params->num_to_process;
4352 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4353 const uint16_t queue_id = ad->queue_ids[0];
4354 struct test_buffers *bufs = NULL;
4355 struct rte_bbdev_info info;
4356 const char *op_type_str;
4357 struct test_time_stats time_st;
4359 memset(&time_st, 0, sizeof(struct test_time_stats));
4360 time_st.enq_sw_min_time = UINT64_MAX;
4361 time_st.enq_acc_min_time = UINT64_MAX;
4362 time_st.deq_min_time = UINT64_MAX;
4364 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4365 "BURST_SIZE should be <= %u", MAX_BURST);
4367 rte_bbdev_info_get(ad->dev_id, &info);
4368 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4370 op_type_str = rte_bbdev_op_type_str(op_type);
4371 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4373 printf("+ ------------------------------------------------------- +\n");
4374 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4375 info.dev_name, burst_sz, num_to_process, op_type_str);
4377 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4378 iter = offload_latency_test_dec(op_params->mp, bufs,
4379 op_params->ref_dec_op, ad->dev_id, queue_id,
4380 num_to_process, burst_sz, &time_st);
4381 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4382 iter = offload_latency_test_enc(op_params->mp, bufs,
4383 op_params->ref_enc_op, ad->dev_id, queue_id,
4384 num_to_process, burst_sz, &time_st);
4385 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4386 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4387 op_params->ref_enc_op, ad->dev_id, queue_id,
4388 num_to_process, burst_sz, &time_st);
4389 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4390 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4391 op_params->ref_dec_op, ad->dev_id, queue_id,
4392 num_to_process, burst_sz, &time_st);
4394 iter = offload_latency_test_enc(op_params->mp, bufs,
4395 op_params->ref_enc_op, ad->dev_id, queue_id,
4396 num_to_process, burst_sz, &time_st);
4401 printf("Enqueue driver offload cost latency:\n"
4402 "\tavg: %lg cycles, %lg us\n"
4403 "\tmin: %lg cycles, %lg us\n"
4404 "\tmax: %lg cycles, %lg us\n"
4405 "Enqueue accelerator offload cost latency:\n"
4406 "\tavg: %lg cycles, %lg us\n"
4407 "\tmin: %lg cycles, %lg us\n"
4408 "\tmax: %lg cycles, %lg us\n",
4409 (double)time_st.enq_sw_total_time / (double)iter,
4410 (double)(time_st.enq_sw_total_time * 1000000) /
4411 (double)iter / (double)rte_get_tsc_hz(),
4412 (double)time_st.enq_sw_min_time,
4413 (double)(time_st.enq_sw_min_time * 1000000) /
4414 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4415 (double)(time_st.enq_sw_max_time * 1000000) /
4416 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4418 (double)(time_st.enq_acc_total_time * 1000000) /
4419 (double)iter / (double)rte_get_tsc_hz(),
4420 (double)time_st.enq_acc_min_time,
4421 (double)(time_st.enq_acc_min_time * 1000000) /
4422 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4423 (double)(time_st.enq_acc_max_time * 1000000) /
4426 printf("Dequeue offload cost latency - one op:\n"
4427 "\tavg: %lg cycles, %lg us\n"
4428 "\tmin: %lg cycles, %lg us\n"
4429 "\tmax: %lg cycles, %lg us\n",
4430 (double)time_st.deq_total_time / (double)iter,
4431 (double)(time_st.deq_total_time * 1000000) /
4432 (double)iter / (double)rte_get_tsc_hz(),
4433 (double)time_st.deq_min_time,
4434 (double)(time_st.deq_min_time * 1000000) /
4435 rte_get_tsc_hz(), (double)time_st.deq_max_time,
4436 (double)(time_st.deq_max_time * 1000000) /
4439 return TEST_SUCCESS;
4443 #ifdef RTE_BBDEV_OFFLOAD_COST
4445 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4446 const uint16_t num_to_process, uint16_t burst_sz,
4447 uint64_t *deq_total_time, uint64_t *deq_min_time,
4448 uint64_t *deq_max_time)
4451 struct rte_bbdev_dec_op *ops[MAX_BURST];
4452 uint64_t deq_start_time, deq_last_time;
4454 /* Test deq offload latency from an empty queue */
4456 for (i = 0, deq_total = 0; deq_total < num_to_process;
4457 ++i, deq_total += burst_sz) {
4458 deq_start_time = rte_rdtsc_precise();
4460 if (unlikely(num_to_process - deq_total < burst_sz))
4461 burst_sz = num_to_process - deq_total;
4462 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
4464 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4465 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4466 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4467 *deq_total_time += deq_last_time;
4474 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4475 const uint16_t num_to_process, uint16_t burst_sz,
4476 uint64_t *deq_total_time, uint64_t *deq_min_time,
4477 uint64_t *deq_max_time)
4480 struct rte_bbdev_enc_op *ops[MAX_BURST];
4481 uint64_t deq_start_time, deq_last_time;
4483 /* Test deq offload latency from an empty queue */
4484 for (i = 0, deq_total = 0; deq_total < num_to_process;
4485 ++i, deq_total += burst_sz) {
4486 deq_start_time = rte_rdtsc_precise();
4488 if (unlikely(num_to_process - deq_total < burst_sz))
4489 burst_sz = num_to_process - deq_total;
4490 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
4492 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4493 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4494 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4495 *deq_total_time += deq_last_time;
4503 offload_latency_empty_q_test(struct active_device *ad,
4504 struct test_op_params *op_params)
4506 #ifndef RTE_BBDEV_OFFLOAD_COST
4508 RTE_SET_USED(op_params);
4509 printf("Offload latency empty dequeue test is disabled.\n");
4510 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4511 return TEST_SKIPPED;
4514 uint64_t deq_total_time, deq_min_time, deq_max_time;
4515 uint16_t burst_sz = op_params->burst_sz;
4516 const uint16_t num_to_process = op_params->num_to_process;
4517 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4518 const uint16_t queue_id = ad->queue_ids[0];
4519 struct rte_bbdev_info info;
4520 const char *op_type_str;
4522 deq_total_time = deq_max_time = 0;
4523 deq_min_time = UINT64_MAX;
4525 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4526 "BURST_SIZE should be <= %u", MAX_BURST);
4528 rte_bbdev_info_get(ad->dev_id, &info);
4530 op_type_str = rte_bbdev_op_type_str(op_type);
4531 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4533 printf("+ ------------------------------------------------------- +\n");
4534 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4535 info.dev_name, burst_sz, num_to_process, op_type_str);
4537 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4538 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4539 num_to_process, burst_sz, &deq_total_time,
4540 &deq_min_time, &deq_max_time);
4542 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4543 num_to_process, burst_sz, &deq_total_time,
4544 &deq_min_time, &deq_max_time);
4549 printf("Empty dequeue offload:\n"
4550 "\tavg: %lg cycles, %lg us\n"
4551 "\tmin: %lg cycles, %lg us\n"
4552 "\tmax: %lg cycles, %lg us\n",
4553 (double)deq_total_time / (double)iter,
4554 (double)(deq_total_time * 1000000) / (double)iter /
4555 (double)rte_get_tsc_hz(), (double)deq_min_time,
4556 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4557 (double)deq_max_time, (double)(deq_max_time * 1000000) /
4560 return TEST_SUCCESS;
4567 return run_test_case(bler_test);
4573 return run_test_case(throughput_test);
4577 offload_cost_tc(void)
4579 return run_test_case(offload_cost_test);
4583 offload_latency_empty_q_tc(void)
4585 return run_test_case(offload_latency_empty_q_test);
4591 return run_test_case(latency_test);
4597 return run_test_case(throughput_test);
4600 static struct unit_test_suite bbdev_bler_testsuite = {
4601 .suite_name = "BBdev BLER Tests",
4602 .setup = testsuite_setup,
4603 .teardown = testsuite_teardown,
4604 .unit_test_cases = {
4605 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
4606 TEST_CASES_END() /**< NULL terminate unit test array */
4610 static struct unit_test_suite bbdev_throughput_testsuite = {
4611 .suite_name = "BBdev Throughput Tests",
4612 .setup = testsuite_setup,
4613 .teardown = testsuite_teardown,
4614 .unit_test_cases = {
4615 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
4616 TEST_CASES_END() /**< NULL terminate unit test array */
4620 static struct unit_test_suite bbdev_validation_testsuite = {
4621 .suite_name = "BBdev Validation Tests",
4622 .setup = testsuite_setup,
4623 .teardown = testsuite_teardown,
4624 .unit_test_cases = {
4625 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4626 TEST_CASES_END() /**< NULL terminate unit test array */
4630 static struct unit_test_suite bbdev_latency_testsuite = {
4631 .suite_name = "BBdev Latency Tests",
4632 .setup = testsuite_setup,
4633 .teardown = testsuite_teardown,
4634 .unit_test_cases = {
4635 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4636 TEST_CASES_END() /**< NULL terminate unit test array */
4640 static struct unit_test_suite bbdev_offload_cost_testsuite = {
4641 .suite_name = "BBdev Offload Cost Tests",
4642 .setup = testsuite_setup,
4643 .teardown = testsuite_teardown,
4644 .unit_test_cases = {
4645 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
4646 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
4647 TEST_CASES_END() /**< NULL terminate unit test array */
4651 static struct unit_test_suite bbdev_interrupt_testsuite = {
4652 .suite_name = "BBdev Interrupt Tests",
4653 .setup = interrupt_testsuite_setup,
4654 .teardown = testsuite_teardown,
4655 .unit_test_cases = {
4656 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
4657 TEST_CASES_END() /**< NULL terminate unit test array */
4661 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
4662 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
4663 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
4664 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
4665 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
4666 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);