1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
19 #include <rte_interrupts.h>
22 #include "test_bbdev_vector.h"
24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
26 #define MAX_QUEUES RTE_MAX_LCORE
27 #define TEST_REPETITIONS 1000
29 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
30 #include <fpga_lte_fec.h>
31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
33 #define VF_UL_4G_QUEUE_VALUE 4
34 #define VF_DL_4G_QUEUE_VALUE 4
35 #define UL_4G_BANDWIDTH 3
36 #define DL_4G_BANDWIDTH 3
37 #define UL_4G_LOAD_BALANCE 128
38 #define DL_4G_LOAD_BALANCE 128
39 #define FLR_4G_TIMEOUT 610
42 #define OPS_CACHE_SIZE 256U
43 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
47 #define INVALID_OPAQUE -1
49 #define INVALID_QUEUE_ID -1
50 /* Increment for next code block in external HARQ memory */
51 #define HARQ_INCR 32768
52 /* Headroom for filler LLRs insertion in HARQ buffer */
53 #define FILLER_HEADROOM 1024
54 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */
55 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */
56 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */
57 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */
58 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */
59 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */
60 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */
61 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */
62 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */
64 static struct test_bbdev_vector test_vector;
66 /* Switch between PMD and Interrupt for throughput TC */
67 static bool intr_enabled;
69 /* LLR arithmetic representation for numerical conversion */
70 static int ldpc_llr_decimals;
71 static int ldpc_llr_size;
72 /* Keep track of the LDPC decoder device capability flag */
73 static uint32_t ldpc_cap_flags;
75 /* Represents tested active devices */
76 static struct active_device {
77 const char *driver_name;
79 uint16_t supported_ops;
80 uint16_t queue_ids[MAX_QUEUES];
82 struct rte_mempool *ops_mempool;
83 struct rte_mempool *in_mbuf_pool;
84 struct rte_mempool *hard_out_mbuf_pool;
85 struct rte_mempool *soft_out_mbuf_pool;
86 struct rte_mempool *harq_in_mbuf_pool;
87 struct rte_mempool *harq_out_mbuf_pool;
88 } active_devs[RTE_BBDEV_MAX_DEVS];
90 static uint8_t nb_active_devs;
92 /* Data buffers used by BBDEV ops */
94 struct rte_bbdev_op_data *inputs;
95 struct rte_bbdev_op_data *hard_outputs;
96 struct rte_bbdev_op_data *soft_outputs;
97 struct rte_bbdev_op_data *harq_inputs;
98 struct rte_bbdev_op_data *harq_outputs;
101 /* Operation parameters specific for given test case */
102 struct test_op_params {
103 struct rte_mempool *mp;
104 struct rte_bbdev_dec_op *ref_dec_op;
105 struct rte_bbdev_enc_op *ref_enc_op;
107 uint16_t num_to_process;
111 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
114 /* Contains per lcore params */
115 struct thread_params {
125 rte_atomic16_t nb_dequeued;
126 rte_atomic16_t processing_status;
127 rte_atomic16_t burst_sz;
128 struct test_op_params *op_params;
129 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
130 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
133 #ifdef RTE_BBDEV_OFFLOAD_COST
134 /* Stores time statistics */
135 struct test_time_stats {
136 /* Stores software enqueue total working time */
137 uint64_t enq_sw_total_time;
138 /* Stores minimum value of software enqueue working time */
139 uint64_t enq_sw_min_time;
140 /* Stores maximum value of software enqueue working time */
141 uint64_t enq_sw_max_time;
142 /* Stores turbo enqueue total working time */
143 uint64_t enq_acc_total_time;
144 /* Stores minimum value of accelerator enqueue working time */
145 uint64_t enq_acc_min_time;
146 /* Stores maximum value of accelerator enqueue working time */
147 uint64_t enq_acc_max_time;
148 /* Stores dequeue total working time */
149 uint64_t deq_total_time;
150 /* Stores minimum value of dequeue working time */
151 uint64_t deq_min_time;
152 /* Stores maximum value of dequeue working time */
153 uint64_t deq_max_time;
157 typedef int (test_case_function)(struct active_device *ad,
158 struct test_op_params *op_params);
161 mbuf_reset(struct rte_mbuf *m)
171 /* Read flag value 0/1 from bitmap */
173 check_bit(uint32_t bitmap, uint32_t bitmask)
175 return bitmap & bitmask;
179 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
181 ad->supported_ops |= (1 << op_type);
185 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
187 return ad->supported_ops & (1 << op_type);
191 flags_match(uint32_t flags_req, uint32_t flags_present)
193 return (flags_req & flags_present) == flags_req;
197 clear_soft_out_cap(uint32_t *op_flags)
199 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
200 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
201 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
205 check_dev_cap(const struct rte_bbdev_info *dev_info)
208 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
209 nb_harq_inputs, nb_harq_outputs;
210 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
212 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
213 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
214 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
215 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
216 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
218 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
219 if (op_cap->type != test_vector.op_type)
222 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
223 const struct rte_bbdev_op_cap_turbo_dec *cap =
224 &op_cap->cap.turbo_dec;
225 /* Ignore lack of soft output capability, just skip
226 * checking if soft output is valid.
228 if ((test_vector.turbo_dec.op_flags &
229 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
230 !(cap->capability_flags &
231 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
233 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
236 &test_vector.turbo_dec.op_flags);
239 if (!flags_match(test_vector.turbo_dec.op_flags,
240 cap->capability_flags))
242 if (nb_inputs > cap->num_buffers_src) {
243 printf("Too many inputs defined: %u, max: %u\n",
244 nb_inputs, cap->num_buffers_src);
247 if (nb_soft_outputs > cap->num_buffers_soft_out &&
248 (test_vector.turbo_dec.op_flags &
249 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
251 "Too many soft outputs defined: %u, max: %u\n",
253 cap->num_buffers_soft_out);
256 if (nb_hard_outputs > cap->num_buffers_hard_out) {
258 "Too many hard outputs defined: %u, max: %u\n",
260 cap->num_buffers_hard_out);
263 if (intr_enabled && !(cap->capability_flags &
264 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
266 "Dequeue interrupts are not supported!\n");
271 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
272 const struct rte_bbdev_op_cap_turbo_enc *cap =
273 &op_cap->cap.turbo_enc;
275 if (!flags_match(test_vector.turbo_enc.op_flags,
276 cap->capability_flags))
278 if (nb_inputs > cap->num_buffers_src) {
279 printf("Too many inputs defined: %u, max: %u\n",
280 nb_inputs, cap->num_buffers_src);
283 if (nb_hard_outputs > cap->num_buffers_dst) {
285 "Too many hard outputs defined: %u, max: %u\n",
286 nb_hard_outputs, cap->num_buffers_dst);
289 if (intr_enabled && !(cap->capability_flags &
290 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
292 "Dequeue interrupts are not supported!\n");
297 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
298 const struct rte_bbdev_op_cap_ldpc_enc *cap =
299 &op_cap->cap.ldpc_enc;
301 if (!flags_match(test_vector.ldpc_enc.op_flags,
302 cap->capability_flags)){
303 printf("Flag Mismatch\n");
306 if (nb_inputs > cap->num_buffers_src) {
307 printf("Too many inputs defined: %u, max: %u\n",
308 nb_inputs, cap->num_buffers_src);
311 if (nb_hard_outputs > cap->num_buffers_dst) {
313 "Too many hard outputs defined: %u, max: %u\n",
314 nb_hard_outputs, cap->num_buffers_dst);
317 if (intr_enabled && !(cap->capability_flags &
318 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) {
320 "Dequeue interrupts are not supported!\n");
325 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
326 const struct rte_bbdev_op_cap_ldpc_dec *cap =
327 &op_cap->cap.ldpc_dec;
329 if (!flags_match(test_vector.ldpc_dec.op_flags,
330 cap->capability_flags)){
331 printf("Flag Mismatch\n");
334 if (nb_inputs > cap->num_buffers_src) {
335 printf("Too many inputs defined: %u, max: %u\n",
336 nb_inputs, cap->num_buffers_src);
339 if (nb_hard_outputs > cap->num_buffers_hard_out) {
341 "Too many hard outputs defined: %u, max: %u\n",
343 cap->num_buffers_hard_out);
346 if (nb_harq_inputs > cap->num_buffers_hard_out) {
348 "Too many HARQ inputs defined: %u, max: %u\n",
350 cap->num_buffers_hard_out);
353 if (nb_harq_outputs > cap->num_buffers_hard_out) {
355 "Too many HARQ outputs defined: %u, max: %u\n",
357 cap->num_buffers_hard_out);
360 if (intr_enabled && !(cap->capability_flags &
361 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) {
363 "Dequeue interrupts are not supported!\n");
366 if (intr_enabled && (test_vector.ldpc_dec.op_flags &
367 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
368 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
369 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
371 printf("Skip loop-back with interrupt\n");
378 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
379 return TEST_SUCCESS; /* Special case for NULL device */
384 /* calculates optimal mempool size not smaller than the val */
386 optimal_mempool_size(unsigned int val)
388 return rte_align32pow2(val + 1) - 1;
391 /* allocates mbuf mempool for inputs and outputs */
392 static struct rte_mempool *
393 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
394 int socket_id, unsigned int mbuf_pool_size,
395 const char *op_type_str)
398 uint32_t max_seg_sz = 0;
399 char pool_name[RTE_MEMPOOL_NAMESIZE];
401 /* find max input segment size */
402 for (i = 0; i < entries->nb_segments; ++i)
403 if (entries->segments[i].length > max_seg_sz)
404 max_seg_sz = entries->segments[i].length;
406 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
408 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
409 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM
411 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
415 create_mempools(struct active_device *ad, int socket_id,
416 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
418 struct rte_mempool *mp;
419 unsigned int ops_pool_size, mbuf_pool_size = 0;
420 char pool_name[RTE_MEMPOOL_NAMESIZE];
421 const char *op_type_str;
422 enum rte_bbdev_op_type op_type = org_op_type;
424 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
425 struct op_data_entries *hard_out =
426 &test_vector.entries[DATA_HARD_OUTPUT];
427 struct op_data_entries *soft_out =
428 &test_vector.entries[DATA_SOFT_OUTPUT];
429 struct op_data_entries *harq_in =
430 &test_vector.entries[DATA_HARQ_INPUT];
431 struct op_data_entries *harq_out =
432 &test_vector.entries[DATA_HARQ_OUTPUT];
434 /* allocate ops mempool */
435 ops_pool_size = optimal_mempool_size(RTE_MAX(
436 /* Ops used plus 1 reference op */
437 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
438 /* Minimal cache size plus 1 reference op */
439 (unsigned int)(1.5 * rte_lcore_count() *
440 OPS_CACHE_SIZE + 1)),
443 if (org_op_type == RTE_BBDEV_OP_NONE)
444 op_type = RTE_BBDEV_OP_TURBO_ENC;
446 op_type_str = rte_bbdev_op_type_str(op_type);
447 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
449 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
451 mp = rte_bbdev_op_pool_create(pool_name, op_type,
452 ops_pool_size, OPS_CACHE_SIZE, socket_id);
453 TEST_ASSERT_NOT_NULL(mp,
454 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
458 ad->ops_mempool = mp;
460 /* Do not create inputs and outputs mbufs for BaseBand Null Device */
461 if (org_op_type == RTE_BBDEV_OP_NONE)
465 if (in->nb_segments > 0) {
466 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
468 mp = create_mbuf_pool(in, ad->dev_id, socket_id,
469 mbuf_pool_size, "in");
470 TEST_ASSERT_NOT_NULL(mp,
471 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
475 ad->in_mbuf_pool = mp;
479 if (hard_out->nb_segments > 0) {
480 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
481 hard_out->nb_segments);
482 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id,
485 TEST_ASSERT_NOT_NULL(mp,
486 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
490 ad->hard_out_mbuf_pool = mp;
494 if (soft_out->nb_segments > 0) {
495 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
496 soft_out->nb_segments);
497 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
500 TEST_ASSERT_NOT_NULL(mp,
501 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
505 ad->soft_out_mbuf_pool = mp;
509 if (harq_in->nb_segments > 0) {
510 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
511 harq_in->nb_segments);
512 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
515 TEST_ASSERT_NOT_NULL(mp,
516 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
520 ad->harq_in_mbuf_pool = mp;
524 if (harq_out->nb_segments > 0) {
525 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
526 harq_out->nb_segments);
527 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
530 TEST_ASSERT_NOT_NULL(mp,
531 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
535 ad->harq_out_mbuf_pool = mp;
542 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
543 struct test_bbdev_vector *vector)
546 unsigned int queue_id;
547 struct rte_bbdev_queue_conf qconf;
548 struct active_device *ad = &active_devs[nb_active_devs];
549 unsigned int nb_queues;
550 enum rte_bbdev_op_type op_type = vector->op_type;
552 /* Configure fpga lte fec with PF & VF values
553 * if '-i' flag is set and using fpga device
555 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
556 if ((get_init_device() == true) &&
557 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
558 struct fpga_lte_fec_conf conf;
561 printf("Configure FPGA LTE FEC Driver %s with default values\n",
562 info->drv.driver_name);
564 /* clear default configuration before initialization */
565 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
568 * true if PF is used for data plane
571 conf.pf_mode_en = true;
573 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
574 /* Number of UL queues per VF (fpga supports 8 VFs) */
575 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
576 /* Number of DL queues per VF (fpga supports 8 VFs) */
577 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
580 /* UL bandwidth. Needed for schedule algorithm */
581 conf.ul_bandwidth = UL_4G_BANDWIDTH;
583 conf.dl_bandwidth = DL_4G_BANDWIDTH;
585 /* UL & DL load Balance Factor to 64 */
586 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
587 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
589 /**< FLR timeout value */
590 conf.flr_time_out = FLR_4G_TIMEOUT;
592 /* setup FPGA PF with configuration information */
593 ret = fpga_lte_fec_configure(info->dev_name, &conf);
594 TEST_ASSERT_SUCCESS(ret,
595 "Failed to configure 4G FPGA PF for bbdev %s",
599 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
600 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
603 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
605 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
606 dev_id, nb_queues, info->socket_id, ret);
610 /* configure interrupts if needed */
612 ret = rte_bbdev_intr_enable(dev_id);
614 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
620 /* setup device queues */
621 qconf.socket = info->socket_id;
622 qconf.queue_size = info->drv.default_queue_conf.queue_size;
624 qconf.deferred_start = 0;
625 qconf.op_type = op_type;
627 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
628 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
631 "Allocated all queues (id=%u) at prio%u on dev%u\n",
632 queue_id, qconf.priority, dev_id);
634 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
638 printf("All queues on dev %u allocated: %u\n",
642 ad->queue_ids[queue_id] = queue_id;
644 TEST_ASSERT(queue_id != 0,
645 "ERROR Failed to configure any queues on dev %u",
647 ad->nb_queues = queue_id;
649 set_avail_op(ad, op_type);
655 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
656 struct test_bbdev_vector *vector)
660 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
661 active_devs[nb_active_devs].dev_id = dev_id;
663 ret = add_bbdev_dev(dev_id, info, vector);
664 if (ret == TEST_SUCCESS)
670 populate_active_devices(void)
674 uint8_t nb_devs_added = 0;
675 struct rte_bbdev_info info;
677 RTE_BBDEV_FOREACH(dev_id) {
678 rte_bbdev_info_get(dev_id, &info);
680 if (check_dev_cap(&info)) {
682 "Device %d (%s) does not support specified capabilities\n",
683 dev_id, info.dev_name);
687 ret = add_active_device(dev_id, &info, &test_vector);
689 printf("Adding active bbdev %s skipped\n",
696 return nb_devs_added;
700 read_test_vector(void)
704 memset(&test_vector, 0, sizeof(test_vector));
705 printf("Test vector file = %s\n", get_vector_filename());
706 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
707 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
708 get_vector_filename());
714 testsuite_setup(void)
716 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
718 if (populate_active_devices() == 0) {
719 printf("No suitable devices found!\n");
727 interrupt_testsuite_setup(void)
729 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
731 /* Enable interrupts */
734 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
735 if (populate_active_devices() == 0 ||
736 test_vector.op_type == RTE_BBDEV_OP_NONE) {
737 intr_enabled = false;
738 printf("No suitable devices found!\n");
746 testsuite_teardown(void)
750 /* Unconfigure devices */
751 RTE_BBDEV_FOREACH(dev_id)
752 rte_bbdev_close(dev_id);
754 /* Clear active devices structs. */
755 memset(active_devs, 0, sizeof(active_devs));
758 /* Disable interrupts */
759 intr_enabled = false;
767 for (i = 0; i < nb_active_devs; i++) {
768 dev_id = active_devs[i].dev_id;
769 /* reset bbdev stats */
770 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
771 "Failed to reset stats of bbdev %u", dev_id);
772 /* start the device */
773 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
774 "Failed to start bbdev %u", dev_id);
784 struct rte_bbdev_stats stats;
786 for (i = 0; i < nb_active_devs; i++) {
787 dev_id = active_devs[i].dev_id;
788 /* read stats and print */
789 rte_bbdev_stats_get(dev_id, &stats);
790 /* Stop the device */
791 rte_bbdev_stop(dev_id);
796 init_op_data_objs(struct rte_bbdev_op_data *bufs,
797 struct op_data_entries *ref_entries,
798 struct rte_mempool *mbuf_pool, const uint16_t n,
799 enum op_data_type op_type, uint16_t min_alignment)
803 bool large_input = false;
805 for (i = 0; i < n; ++i) {
807 struct op_data_buf *seg = &ref_entries->segments[0];
808 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
809 TEST_ASSERT_NOT_NULL(m_head,
810 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
811 op_type, n * ref_entries->nb_segments,
814 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
816 * Special case when DPDK mbuf cannot handle
817 * the required input size
819 printf("Warning: Larger input size than DPDK mbuf %d\n",
823 bufs[i].data = m_head;
827 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
828 if ((op_type == DATA_INPUT) && large_input) {
829 /* Allocate a fake overused mbuf */
830 data = rte_malloc(NULL, seg->length, 0);
831 memcpy(data, seg->addr, seg->length);
832 m_head->buf_addr = data;
833 m_head->buf_iova = rte_malloc_virt2iova(data);
834 m_head->data_off = 0;
835 m_head->data_len = seg->length;
837 data = rte_pktmbuf_append(m_head, seg->length);
838 TEST_ASSERT_NOT_NULL(data,
839 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
840 seg->length, op_type);
842 TEST_ASSERT(data == RTE_PTR_ALIGN(
843 data, min_alignment),
844 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
845 data, min_alignment);
846 rte_memcpy(data, seg->addr, seg->length);
849 bufs[i].length += seg->length;
851 for (j = 1; j < ref_entries->nb_segments; ++j) {
852 struct rte_mbuf *m_tail =
853 rte_pktmbuf_alloc(mbuf_pool);
854 TEST_ASSERT_NOT_NULL(m_tail,
855 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
857 n * ref_entries->nb_segments,
861 data = rte_pktmbuf_append(m_tail, seg->length);
862 TEST_ASSERT_NOT_NULL(data,
863 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
864 seg->length, op_type);
866 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
868 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
869 data, min_alignment);
870 rte_memcpy(data, seg->addr, seg->length);
871 bufs[i].length += seg->length;
873 ret = rte_pktmbuf_chain(m_head, m_tail);
874 TEST_ASSERT_SUCCESS(ret,
875 "Couldn't chain mbufs from %d data type mbuf pool",
880 /* allocate chained-mbuf for output buffer */
881 for (j = 1; j < ref_entries->nb_segments; ++j) {
882 struct rte_mbuf *m_tail =
883 rte_pktmbuf_alloc(mbuf_pool);
884 TEST_ASSERT_NOT_NULL(m_tail,
885 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
887 n * ref_entries->nb_segments,
890 ret = rte_pktmbuf_chain(m_head, m_tail);
891 TEST_ASSERT_SUCCESS(ret,
892 "Couldn't chain mbufs from %d data type mbuf pool",
902 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
907 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
908 if (*buffers == NULL) {
909 printf("WARNING: Failed to allocate op_data on socket %d\n",
911 /* try to allocate memory on other detected sockets */
912 for (i = 0; i < socket; i++) {
913 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
914 if (*buffers != NULL)
919 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
923 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
924 const uint16_t n, const int8_t max_llr_modulus)
926 uint16_t i, byte_idx;
928 for (i = 0; i < n; ++i) {
929 struct rte_mbuf *m = input_ops[i].data;
931 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
932 input_ops[i].offset);
933 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
935 llr[byte_idx] = round((double)max_llr_modulus *
936 llr[byte_idx] / INT8_MAX);
944 * We may have to insert filler bits
945 * when they are required by the HARQ assumption
948 ldpc_add_filler(struct rte_bbdev_op_data *input_ops,
949 const uint16_t n, struct test_op_params *op_params)
951 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec;
953 if (input_ops == NULL)
955 /* No need to add filler if not required by device */
956 if (!(ldpc_cap_flags &
957 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS))
959 /* No need to add filler for loopback operation */
960 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
963 uint16_t i, j, parity_offset;
964 for (i = 0; i < n; ++i) {
965 struct rte_mbuf *m = input_ops[i].data;
966 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
967 input_ops[i].offset);
968 parity_offset = (dec.basegraph == 1 ? 20 : 8)
969 * dec.z_c - dec.n_filler;
970 uint16_t new_hin_size = input_ops[i].length + dec.n_filler;
971 m->data_len = new_hin_size;
972 input_ops[i].length = new_hin_size;
973 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler;
975 llr[j] = llr[j - dec.n_filler];
976 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
977 for (j = 0; j < dec.n_filler; j++)
978 llr[parity_offset + j] = llr_max_pre_scaling;
983 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
984 const uint16_t n, const int8_t llr_size,
985 const int8_t llr_decimals)
987 if (input_ops == NULL)
990 uint16_t i, byte_idx;
992 int16_t llr_max, llr_min, llr_tmp;
993 llr_max = (1 << (llr_size - 1)) - 1;
995 for (i = 0; i < n; ++i) {
996 struct rte_mbuf *m = input_ops[i].data;
998 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
999 input_ops[i].offset);
1000 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
1003 llr_tmp = llr[byte_idx];
1004 if (llr_decimals == 4)
1006 else if (llr_decimals == 2)
1008 else if (llr_decimals == 0)
1010 llr_tmp = RTE_MIN(llr_max,
1011 RTE_MAX(llr_min, llr_tmp));
1012 llr[byte_idx] = (int8_t) llr_tmp;
1023 fill_queue_buffers(struct test_op_params *op_params,
1024 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
1025 struct rte_mempool *soft_out_mp,
1026 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
1028 const struct rte_bbdev_op_cap *capabilities,
1029 uint16_t min_alignment, const int socket_id)
1032 enum op_data_type type;
1033 const uint16_t n = op_params->num_to_process;
1035 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
1043 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
1044 &op_params->q_bufs[socket_id][queue_id].inputs,
1045 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
1046 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
1047 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
1048 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
1051 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
1052 struct op_data_entries *ref_entries =
1053 &test_vector.entries[type];
1054 if (ref_entries->nb_segments == 0)
1057 ret = allocate_buffers_on_socket(queue_ops[type],
1058 n * sizeof(struct rte_bbdev_op_data),
1060 TEST_ASSERT_SUCCESS(ret,
1061 "Couldn't allocate memory for rte_bbdev_op_data structs");
1063 ret = init_op_data_objs(*queue_ops[type], ref_entries,
1064 mbuf_pools[type], n, type, min_alignment);
1065 TEST_ASSERT_SUCCESS(ret,
1066 "Couldn't init rte_bbdev_op_data structs");
1069 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1070 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
1071 capabilities->cap.turbo_dec.max_llr_modulus);
1073 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1074 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags &
1075 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
1076 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1077 RTE_BBDEV_LDPC_LLR_COMPRESSION;
1078 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags &
1079 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
1080 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals;
1081 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size;
1082 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags;
1083 if (!loopback && !llr_comp)
1084 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
1085 ldpc_llr_size, ldpc_llr_decimals);
1086 if (!loopback && !harq_comp)
1087 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
1088 ldpc_llr_size, ldpc_llr_decimals);
1090 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n,
1098 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1102 rte_mempool_free(ad->ops_mempool);
1103 rte_mempool_free(ad->in_mbuf_pool);
1104 rte_mempool_free(ad->hard_out_mbuf_pool);
1105 rte_mempool_free(ad->soft_out_mbuf_pool);
1106 rte_mempool_free(ad->harq_in_mbuf_pool);
1107 rte_mempool_free(ad->harq_out_mbuf_pool);
1109 for (i = 0; i < rte_lcore_count(); ++i) {
1110 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1111 rte_free(op_params->q_bufs[j][i].inputs);
1112 rte_free(op_params->q_bufs[j][i].hard_outputs);
1113 rte_free(op_params->q_bufs[j][i].soft_outputs);
1114 rte_free(op_params->q_bufs[j][i].harq_inputs);
1115 rte_free(op_params->q_bufs[j][i].harq_outputs);
1121 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1122 unsigned int start_idx,
1123 struct rte_bbdev_op_data *inputs,
1124 struct rte_bbdev_op_data *hard_outputs,
1125 struct rte_bbdev_op_data *soft_outputs,
1126 struct rte_bbdev_dec_op *ref_op)
1129 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1131 for (i = 0; i < n; ++i) {
1132 if (turbo_dec->code_block_mode == 0) {
1133 ops[i]->turbo_dec.tb_params.ea =
1134 turbo_dec->tb_params.ea;
1135 ops[i]->turbo_dec.tb_params.eb =
1136 turbo_dec->tb_params.eb;
1137 ops[i]->turbo_dec.tb_params.k_pos =
1138 turbo_dec->tb_params.k_pos;
1139 ops[i]->turbo_dec.tb_params.k_neg =
1140 turbo_dec->tb_params.k_neg;
1141 ops[i]->turbo_dec.tb_params.c =
1142 turbo_dec->tb_params.c;
1143 ops[i]->turbo_dec.tb_params.c_neg =
1144 turbo_dec->tb_params.c_neg;
1145 ops[i]->turbo_dec.tb_params.cab =
1146 turbo_dec->tb_params.cab;
1147 ops[i]->turbo_dec.tb_params.r =
1148 turbo_dec->tb_params.r;
1150 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1151 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1154 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1155 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1156 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1157 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1158 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1159 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1160 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1162 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1163 ops[i]->turbo_dec.input = inputs[start_idx + i];
1164 if (soft_outputs != NULL)
1165 ops[i]->turbo_dec.soft_output =
1166 soft_outputs[start_idx + i];
1171 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1172 unsigned int start_idx,
1173 struct rte_bbdev_op_data *inputs,
1174 struct rte_bbdev_op_data *outputs,
1175 struct rte_bbdev_enc_op *ref_op)
1178 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1179 for (i = 0; i < n; ++i) {
1180 if (turbo_enc->code_block_mode == 0) {
1181 ops[i]->turbo_enc.tb_params.ea =
1182 turbo_enc->tb_params.ea;
1183 ops[i]->turbo_enc.tb_params.eb =
1184 turbo_enc->tb_params.eb;
1185 ops[i]->turbo_enc.tb_params.k_pos =
1186 turbo_enc->tb_params.k_pos;
1187 ops[i]->turbo_enc.tb_params.k_neg =
1188 turbo_enc->tb_params.k_neg;
1189 ops[i]->turbo_enc.tb_params.c =
1190 turbo_enc->tb_params.c;
1191 ops[i]->turbo_enc.tb_params.c_neg =
1192 turbo_enc->tb_params.c_neg;
1193 ops[i]->turbo_enc.tb_params.cab =
1194 turbo_enc->tb_params.cab;
1195 ops[i]->turbo_enc.tb_params.ncb_pos =
1196 turbo_enc->tb_params.ncb_pos;
1197 ops[i]->turbo_enc.tb_params.ncb_neg =
1198 turbo_enc->tb_params.ncb_neg;
1199 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1201 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1202 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1203 ops[i]->turbo_enc.cb_params.ncb =
1204 turbo_enc->cb_params.ncb;
1206 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1207 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1208 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1210 ops[i]->turbo_enc.output = outputs[start_idx + i];
1211 ops[i]->turbo_enc.input = inputs[start_idx + i];
1216 /* Returns a random number drawn from a normal distribution
1217 * with mean of 0 and variance of 1
1218 * Marsaglia algorithm
1223 double S, Z, U1, U2, u, v, fac;
1226 U1 = (double)rand() / RAND_MAX;
1227 U2 = (double)rand() / RAND_MAX;
1231 } while (S >= 1 || S == 0);
1232 fac = sqrt(-2. * log(S) / S);
1233 Z = (n % 2) ? u * fac : v * fac;
1237 static inline double
1238 maxstar(double A, double B)
1240 if (fabs(A - B) > 5)
1241 return RTE_MAX(A, B);
1243 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B)));
1247 * Generate Qm LLRS for Qm==8
1248 * Modulation, AWGN and LLR estimation from max log development
1251 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1256 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1257 /* 5.1.4 of TS38.211 */
1258 const double symbols_I[256] = {
1259 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5,
1260 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11,
1261 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13,
1262 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15,
1263 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3,
1264 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1,
1265 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13,
1266 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9,
1267 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5,
1268 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5,
1269 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3,
1270 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13,
1271 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9,
1272 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1273 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3,
1274 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5,
1275 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11,
1276 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13,
1277 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9,
1278 -13, -13, -15, -15, -13, -13, -15, -15};
1279 const double symbols_Q[256] = {
1280 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1281 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13,
1282 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1283 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13,
1284 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5,
1285 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13,
1286 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13,
1287 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5,
1288 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1289 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5,
1290 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11,
1291 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15,
1292 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1,
1293 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9,
1294 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1,
1295 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9,
1296 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15,
1297 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7,
1298 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15,
1299 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15};
1300 /* Average constellation point energy */
1302 for (k = 0; k < qm; k++)
1303 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1304 /* 5.1.4 of TS38.211 */
1305 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) *
1306 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6]))));
1307 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) *
1308 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7]))));
1310 I += sqrt(N0 / 2) * randn(0);
1311 Q += sqrt(N0 / 2) * randn(1);
1313 * Calculate the log of the probability that each of
1314 * the constellation points was transmitted
1316 for (m = 0; m < qam; m++)
1317 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1318 + pow(Q - symbols_Q[m], 2.0)) / N0;
1319 /* Calculate an LLR for each of the k_64QAM bits in the set */
1320 for (k = 0; k < qm; k++) {
1323 /* For each constellation point */
1324 for (m = 0; m < qam; m++) {
1325 if ((m >> (qm - k - 1)) & 1)
1326 p1 = maxstar(p1, log_syml_prob[m]);
1328 p0 = maxstar(p0, log_syml_prob[m]);
1330 /* Calculate the LLR */
1332 llr_ *= (1 << ldpc_llr_decimals);
1336 if (llr_ < -llr_max)
1338 llrs[qm * i + k] = (int8_t) llr_;
1344 * Generate Qm LLRS for Qm==6
1345 * Modulation, AWGN and LLR estimation from max log development
1348 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1353 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1354 /* 5.1.4 of TS38.211 */
1355 const double symbols_I[64] = {
1356 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1357 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7,
1358 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7,
1359 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1,
1360 -5, -5, -7, -7, -5, -5, -7, -7};
1361 const double symbols_Q[64] = {
1362 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7,
1363 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1,
1364 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1,
1365 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7,
1366 -3, -1, -3, -1, -5, -7, -5, -7};
1367 /* Average constellation point energy */
1369 for (k = 0; k < qm; k++)
1370 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1371 /* 5.1.4 of TS38.211 */
1372 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4])));
1373 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5])));
1375 I += sqrt(N0 / 2) * randn(0);
1376 Q += sqrt(N0 / 2) * randn(1);
1378 * Calculate the log of the probability that each of
1379 * the constellation points was transmitted
1381 for (m = 0; m < qam; m++)
1382 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1383 + pow(Q - symbols_Q[m], 2.0)) / N0;
1384 /* Calculate an LLR for each of the k_64QAM bits in the set */
1385 for (k = 0; k < qm; k++) {
1388 /* For each constellation point */
1389 for (m = 0; m < qam; m++) {
1390 if ((m >> (qm - k - 1)) & 1)
1391 p1 = maxstar(p1, log_syml_prob[m]);
1393 p0 = maxstar(p0, log_syml_prob[m]);
1395 /* Calculate the LLR */
1397 llr_ *= (1 << ldpc_llr_decimals);
1401 if (llr_ < -llr_max)
1403 llrs[qm * i + k] = (int8_t) llr_;
1408 * Generate Qm LLRS for Qm==4
1409 * Modulation, AWGN and LLR estimation from max log development
1412 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max)
1417 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam];
1418 /* 5.1.4 of TS38.211 */
1419 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3,
1420 -1, -1, -3, -3, -1, -1, -3, -3};
1421 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3,
1422 1, 3, 1, 3, -1, -3, -1, -3};
1423 /* Average constellation point energy */
1425 for (k = 0; k < qm; k++)
1426 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0;
1427 /* 5.1.4 of TS38.211 */
1428 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2]));
1429 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3]));
1431 I += sqrt(N0 / 2) * randn(0);
1432 Q += sqrt(N0 / 2) * randn(1);
1434 * Calculate the log of the probability that each of
1435 * the constellation points was transmitted
1437 for (m = 0; m < qam; m++)
1438 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0)
1439 + pow(Q - symbols_Q[m], 2.0)) / N0;
1440 /* Calculate an LLR for each of the k_64QAM bits in the set */
1441 for (k = 0; k < qm; k++) {
1444 /* For each constellation point */
1445 for (m = 0; m < qam; m++) {
1446 if ((m >> (qm - k - 1)) & 1)
1447 p1 = maxstar(p1, log_syml_prob[m]);
1449 p0 = maxstar(p0, log_syml_prob[m]);
1451 /* Calculate the LLR */
1453 llr_ *= (1 << ldpc_llr_decimals);
1457 if (llr_ < -llr_max)
1459 llrs[qm * i + k] = (int8_t) llr_;
1464 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max)
1467 double coeff = 2.0 * sqrt(N0);
1469 /* Ignore in vectors rare quasi null LLRs not to be saturated */
1470 if (llrs[j] < 8 && llrs[j] > -8)
1473 /* Note don't change sign here */
1475 b1 = ((llrs[j] > 0 ? 2.0 : -2.0)
1477 b = b1 * (1 << ldpc_llr_decimals);
1483 llrs[j] = (int8_t) b;
1486 /* Generate LLR for a given SNR */
1488 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs,
1489 struct rte_bbdev_dec_op *ref_op)
1493 uint32_t i, j, e, range;
1496 e = ref_op->ldpc_dec.cb_params.e;
1497 qm = ref_op->ldpc_dec.q_m;
1498 llr_max = (1 << (ldpc_llr_size - 1)) - 1;
1500 N0 = 1.0 / pow(10.0, get_snr() / 10.0);
1502 for (i = 0; i < n; ++i) {
1504 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0);
1506 for (j = 0; j < range; ++j)
1507 gen_qm8_llr(llrs, j, N0, llr_max);
1508 } else if (qm == 6) {
1509 for (j = 0; j < range; ++j)
1510 gen_qm6_llr(llrs, j, N0, llr_max);
1511 } else if (qm == 4) {
1512 for (j = 0; j < range; ++j)
1513 gen_qm4_llr(llrs, j, N0, llr_max);
1515 for (j = 0; j < e; ++j)
1516 gen_qm2_llr(llrs, j, N0, llr_max);
1522 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1523 unsigned int start_idx,
1524 struct rte_bbdev_op_data *inputs,
1525 struct rte_bbdev_op_data *hard_outputs,
1526 struct rte_bbdev_op_data *soft_outputs,
1527 struct rte_bbdev_op_data *harq_inputs,
1528 struct rte_bbdev_op_data *harq_outputs,
1529 struct rte_bbdev_dec_op *ref_op)
1532 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1534 for (i = 0; i < n; ++i) {
1535 if (ldpc_dec->code_block_mode == 0) {
1536 ops[i]->ldpc_dec.tb_params.ea =
1537 ldpc_dec->tb_params.ea;
1538 ops[i]->ldpc_dec.tb_params.eb =
1539 ldpc_dec->tb_params.eb;
1540 ops[i]->ldpc_dec.tb_params.c =
1541 ldpc_dec->tb_params.c;
1542 ops[i]->ldpc_dec.tb_params.cab =
1543 ldpc_dec->tb_params.cab;
1544 ops[i]->ldpc_dec.tb_params.r =
1545 ldpc_dec->tb_params.r;
1547 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1550 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1551 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1552 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1553 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1554 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1555 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1556 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1557 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1558 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1560 if (hard_outputs != NULL)
1561 ops[i]->ldpc_dec.hard_output =
1562 hard_outputs[start_idx + i];
1564 ops[i]->ldpc_dec.input =
1565 inputs[start_idx + i];
1566 if (soft_outputs != NULL)
1567 ops[i]->ldpc_dec.soft_output =
1568 soft_outputs[start_idx + i];
1569 if (harq_inputs != NULL)
1570 ops[i]->ldpc_dec.harq_combined_input =
1571 harq_inputs[start_idx + i];
1572 if (harq_outputs != NULL)
1573 ops[i]->ldpc_dec.harq_combined_output =
1574 harq_outputs[start_idx + i];
1580 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1581 unsigned int start_idx,
1582 struct rte_bbdev_op_data *inputs,
1583 struct rte_bbdev_op_data *outputs,
1584 struct rte_bbdev_enc_op *ref_op)
1587 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1588 for (i = 0; i < n; ++i) {
1589 if (ldpc_enc->code_block_mode == 0) {
1590 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1591 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1592 ops[i]->ldpc_enc.tb_params.cab =
1593 ldpc_enc->tb_params.cab;
1594 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1595 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1597 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1599 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1600 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1601 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1602 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1603 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1604 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1605 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1606 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1607 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1608 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1613 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1614 unsigned int order_idx, const int expected_status)
1616 int status = op->status;
1617 /* ignore parity mismatch false alarms for long iterations */
1618 if (get_iter_max() >= 10) {
1619 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1620 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1621 printf("WARNING: Ignore Syndrome Check mismatch\n");
1622 status -= (1 << RTE_BBDEV_SYNDROME_ERROR);
1624 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) &&
1625 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) {
1626 printf("WARNING: Ignore Syndrome Check mismatch\n");
1627 status += (1 << RTE_BBDEV_SYNDROME_ERROR);
1631 TEST_ASSERT(status == expected_status,
1632 "op_status (%d) != expected_status (%d)",
1633 op->status, expected_status);
1635 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1636 "Ordering error, expected %p, got %p",
1637 (void *)(uintptr_t)order_idx, op->opaque_data);
1639 return TEST_SUCCESS;
1643 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1644 unsigned int order_idx, const int expected_status)
1646 TEST_ASSERT(op->status == expected_status,
1647 "op_status (%d) != expected_status (%d)",
1648 op->status, expected_status);
1650 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE)
1651 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1652 "Ordering error, expected %p, got %p",
1653 (void *)(uintptr_t)order_idx, op->opaque_data);
1655 return TEST_SUCCESS;
1659 validate_op_chain(struct rte_bbdev_op_data *op,
1660 struct op_data_entries *orig_op)
1663 struct rte_mbuf *m = op->data;
1664 uint8_t nb_dst_segments = orig_op->nb_segments;
1665 uint32_t total_data_size = 0;
1667 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1668 "Number of segments differ in original (%u) and filled (%u) op",
1669 nb_dst_segments, m->nb_segs);
1671 /* Validate each mbuf segment length */
1672 for (i = 0; i < nb_dst_segments; ++i) {
1673 /* Apply offset to the first mbuf segment */
1674 uint16_t offset = (i == 0) ? op->offset : 0;
1675 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1676 total_data_size += orig_op->segments[i].length;
1678 TEST_ASSERT(orig_op->segments[i].length == data_len,
1679 "Length of segment differ in original (%u) and filled (%u) op",
1680 orig_op->segments[i].length, data_len);
1681 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1682 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1684 "Output buffers (CB=%u) are not equal", i);
1688 /* Validate total mbuf pkt length */
1689 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1690 TEST_ASSERT(total_data_size == pkt_len,
1691 "Length of data differ in original (%u) and filled (%u) op",
1692 total_data_size, pkt_len);
1694 return TEST_SUCCESS;
1698 * Compute K0 for a given configuration for HARQ output length computation
1699 * As per definition in 3GPP 38.212 Table 5.4.2.1-2
1701 static inline uint16_t
1702 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
1706 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1709 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
1710 else if (rv_index == 2)
1711 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
1713 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
1715 /* LBRM case - includes a division by N */
1717 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
1719 else if (rv_index == 2)
1720 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
1723 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
1727 /* HARQ output length including the Filler bits */
1728 static inline uint16_t
1729 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld)
1732 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index;
1733 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv);
1734 /* Compute RM out size and number of rows */
1735 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1736 * ops_ld->z_c - ops_ld->n_filler;
1737 uint16_t deRmOutSize = RTE_MIN(
1738 k0 + ops_ld->cb_params.e +
1739 ((k0 > parity_offset) ?
1740 0 : ops_ld->n_filler),
1742 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1)
1744 uint16_t harq_output_len = numRows * ops_ld->z_c;
1745 return harq_output_len;
1749 validate_op_harq_chain(struct rte_bbdev_op_data *op,
1750 struct op_data_entries *orig_op,
1751 struct rte_bbdev_op_ldpc_dec *ops_ld)
1755 struct rte_mbuf *m = op->data;
1756 uint8_t nb_dst_segments = orig_op->nb_segments;
1757 uint32_t total_data_size = 0;
1758 int8_t *harq_orig, *harq_out, abs_harq_origin;
1759 uint32_t byte_error = 0, cum_error = 0, error;
1760 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1;
1761 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1;
1762 uint16_t parity_offset;
1764 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1765 "Number of segments differ in original (%u) and filled (%u) op",
1766 nb_dst_segments, m->nb_segs);
1768 /* Validate each mbuf segment length */
1769 for (i = 0; i < nb_dst_segments; ++i) {
1770 /* Apply offset to the first mbuf segment */
1771 uint16_t offset = (i == 0) ? op->offset : 0;
1772 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1773 total_data_size += orig_op->segments[i].length;
1775 TEST_ASSERT(orig_op->segments[i].length <
1776 (uint32_t)(data_len + 64),
1777 "Length of segment differ in original (%u) and filled (%u) op",
1778 orig_op->segments[i].length, data_len);
1779 harq_orig = (int8_t *) orig_op->segments[i].addr;
1780 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset);
1782 if (!(ldpc_cap_flags &
1783 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS
1784 ) || (ops_ld->op_flags &
1785 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1786 data_len -= ops_ld->z_c;
1787 parity_offset = data_len;
1789 /* Compute RM out size and number of rows */
1790 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8)
1791 * ops_ld->z_c - ops_ld->n_filler;
1792 uint16_t deRmOutSize = compute_harq_len(ops_ld) -
1794 if (data_len > deRmOutSize)
1795 data_len = deRmOutSize;
1796 if (data_len > orig_op->segments[i].length)
1797 data_len = orig_op->segments[i].length;
1800 * HARQ output can have minor differences
1801 * due to integer representation and related scaling
1803 for (j = 0, jj = 0; j < data_len; j++, jj++) {
1804 if (j == parity_offset) {
1805 /* Special Handling of the filler bits */
1806 for (k = 0; k < ops_ld->n_filler; k++) {
1808 llr_max_pre_scaling) {
1809 printf("HARQ Filler issue %d: %d %d\n",
1817 if (!(ops_ld->op_flags &
1818 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1819 if (ldpc_llr_decimals > 1)
1820 harq_out[jj] = (harq_out[jj] + 1)
1821 >> (ldpc_llr_decimals - 1);
1822 /* Saturated to S7 */
1823 if (harq_orig[j] > llr_max)
1824 harq_orig[j] = llr_max;
1825 if (harq_orig[j] < -llr_max)
1826 harq_orig[j] = -llr_max;
1828 if (harq_orig[j] != harq_out[jj]) {
1829 error = (harq_orig[j] > harq_out[jj]) ?
1830 harq_orig[j] - harq_out[jj] :
1831 harq_out[jj] - harq_orig[j];
1832 abs_harq_origin = harq_orig[j] > 0 ?
1835 /* Residual quantization error */
1836 if ((error > 8 && (abs_harq_origin <
1839 printf("HARQ mismatch %d: exp %d act %d => %d\n",
1841 harq_out[jj], error);
1851 TEST_ASSERT(byte_error <= 1,
1852 "HARQ output mismatch (%d) %d",
1853 byte_error, cum_error);
1855 /* Validate total mbuf pkt length */
1856 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1857 TEST_ASSERT(total_data_size < pkt_len + 64,
1858 "Length of data differ in original (%u) and filled (%u) op",
1859 total_data_size, pkt_len);
1861 return TEST_SUCCESS;
1865 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1866 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1870 struct op_data_entries *hard_data_orig =
1871 &test_vector.entries[DATA_HARD_OUTPUT];
1872 struct op_data_entries *soft_data_orig =
1873 &test_vector.entries[DATA_SOFT_OUTPUT];
1874 struct rte_bbdev_op_turbo_dec *ops_td;
1875 struct rte_bbdev_op_data *hard_output;
1876 struct rte_bbdev_op_data *soft_output;
1877 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
1879 for (i = 0; i < n; ++i) {
1880 ops_td = &ops[i]->turbo_dec;
1881 hard_output = &ops_td->hard_output;
1882 soft_output = &ops_td->soft_output;
1884 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1885 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1886 "Returned iter_count (%d) > expected iter_count (%d)",
1887 ops_td->iter_count, ref_td->iter_count);
1888 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1889 TEST_ASSERT_SUCCESS(ret,
1890 "Checking status and ordering for decoder failed");
1892 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1894 "Hard output buffers (CB=%u) are not equal",
1897 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
1898 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1900 "Soft output buffers (CB=%u) are not equal",
1904 return TEST_SUCCESS;
1907 /* Check Number of code blocks errors */
1909 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n)
1912 struct op_data_entries *hard_data_orig =
1913 &test_vector.entries[DATA_HARD_OUTPUT];
1914 struct rte_bbdev_op_ldpc_dec *ops_td;
1915 struct rte_bbdev_op_data *hard_output;
1919 for (i = 0; i < n; ++i) {
1920 ops_td = &ops[i]->ldpc_dec;
1921 hard_output = &ops_td->hard_output;
1922 m = hard_output->data;
1923 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0),
1924 hard_data_orig->segments[0].addr,
1925 hard_data_orig->segments[0].length))
1932 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1933 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1937 struct op_data_entries *hard_data_orig =
1938 &test_vector.entries[DATA_HARD_OUTPUT];
1939 struct op_data_entries *soft_data_orig =
1940 &test_vector.entries[DATA_SOFT_OUTPUT];
1941 struct op_data_entries *harq_data_orig =
1942 &test_vector.entries[DATA_HARQ_OUTPUT];
1943 struct rte_bbdev_op_ldpc_dec *ops_td;
1944 struct rte_bbdev_op_data *hard_output;
1945 struct rte_bbdev_op_data *harq_output;
1946 struct rte_bbdev_op_data *soft_output;
1947 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
1949 for (i = 0; i < n; ++i) {
1950 ops_td = &ops[i]->ldpc_dec;
1951 hard_output = &ops_td->hard_output;
1952 harq_output = &ops_td->harq_combined_output;
1953 soft_output = &ops_td->soft_output;
1955 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1956 TEST_ASSERT_SUCCESS(ret,
1957 "Checking status and ordering for decoder failed");
1958 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1959 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1960 "Returned iter_count (%d) > expected iter_count (%d)",
1961 ops_td->iter_count, ref_td->iter_count);
1963 * We can ignore output data when the decoding failed to
1964 * converge or for loop-back cases
1966 if (!check_bit(ops[i]->ldpc_dec.op_flags,
1967 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK
1969 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR
1971 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1973 "Hard output buffers (CB=%u) are not equal",
1976 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
1977 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1979 "Soft output buffers (CB=%u) are not equal",
1981 if (ref_op->ldpc_dec.op_flags &
1982 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
1983 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
1984 harq_data_orig, ops_td),
1985 "HARQ output buffers (CB=%u) are not equal",
1988 if (ref_op->ldpc_dec.op_flags &
1989 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)
1990 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output,
1991 harq_data_orig, ops_td),
1992 "HARQ output buffers (CB=%u) are not equal",
1997 return TEST_SUCCESS;
2002 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2003 struct rte_bbdev_enc_op *ref_op)
2007 struct op_data_entries *hard_data_orig =
2008 &test_vector.entries[DATA_HARD_OUTPUT];
2010 for (i = 0; i < n; ++i) {
2011 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2012 TEST_ASSERT_SUCCESS(ret,
2013 "Checking status and ordering for encoder failed");
2014 TEST_ASSERT_SUCCESS(validate_op_chain(
2015 &ops[i]->turbo_enc.output,
2017 "Output buffers (CB=%u) are not equal",
2021 return TEST_SUCCESS;
2025 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
2026 struct rte_bbdev_enc_op *ref_op)
2030 struct op_data_entries *hard_data_orig =
2031 &test_vector.entries[DATA_HARD_OUTPUT];
2033 for (i = 0; i < n; ++i) {
2034 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
2035 TEST_ASSERT_SUCCESS(ret,
2036 "Checking status and ordering for encoder failed");
2037 TEST_ASSERT_SUCCESS(validate_op_chain(
2038 &ops[i]->ldpc_enc.output,
2040 "Output buffers (CB=%u) are not equal",
2044 return TEST_SUCCESS;
2048 create_reference_dec_op(struct rte_bbdev_dec_op *op)
2051 struct op_data_entries *entry;
2053 op->turbo_dec = test_vector.turbo_dec;
2054 entry = &test_vector.entries[DATA_INPUT];
2055 for (i = 0; i < entry->nb_segments; ++i)
2056 op->turbo_dec.input.length +=
2057 entry->segments[i].length;
2061 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
2064 struct op_data_entries *entry;
2066 op->ldpc_dec = test_vector.ldpc_dec;
2067 entry = &test_vector.entries[DATA_INPUT];
2068 for (i = 0; i < entry->nb_segments; ++i)
2069 op->ldpc_dec.input.length +=
2070 entry->segments[i].length;
2071 if (test_vector.ldpc_dec.op_flags &
2072 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
2073 entry = &test_vector.entries[DATA_HARQ_INPUT];
2074 for (i = 0; i < entry->nb_segments; ++i)
2075 op->ldpc_dec.harq_combined_input.length +=
2076 entry->segments[i].length;
2082 create_reference_enc_op(struct rte_bbdev_enc_op *op)
2085 struct op_data_entries *entry;
2087 op->turbo_enc = test_vector.turbo_enc;
2088 entry = &test_vector.entries[DATA_INPUT];
2089 for (i = 0; i < entry->nb_segments; ++i)
2090 op->turbo_enc.input.length +=
2091 entry->segments[i].length;
2095 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
2098 struct op_data_entries *entry;
2100 op->ldpc_enc = test_vector.ldpc_enc;
2101 entry = &test_vector.entries[DATA_INPUT];
2102 for (i = 0; i < entry->nb_segments; ++i)
2103 op->ldpc_enc.input.length +=
2104 entry->segments[i].length;
2108 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
2111 uint32_t c, r, tb_size = 0;
2113 if (op->turbo_dec.code_block_mode) {
2114 tb_size = op->turbo_dec.tb_params.k_neg;
2116 c = op->turbo_dec.tb_params.c;
2117 r = op->turbo_dec.tb_params.r;
2118 for (i = 0; i < c-r; i++)
2119 tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
2120 op->turbo_dec.tb_params.k_neg :
2121 op->turbo_dec.tb_params.k_pos;
2127 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
2130 uint32_t c, r, tb_size = 0;
2131 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
2133 if (op->ldpc_dec.code_block_mode) {
2134 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
2136 c = op->ldpc_dec.tb_params.c;
2137 r = op->ldpc_dec.tb_params.r;
2138 for (i = 0; i < c-r; i++)
2139 tb_size += sys_cols * op->ldpc_dec.z_c
2140 - op->ldpc_dec.n_filler;
2146 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
2149 uint32_t c, r, tb_size = 0;
2151 if (op->turbo_enc.code_block_mode) {
2152 tb_size = op->turbo_enc.tb_params.k_neg;
2154 c = op->turbo_enc.tb_params.c;
2155 r = op->turbo_enc.tb_params.r;
2156 for (i = 0; i < c-r; i++)
2157 tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
2158 op->turbo_enc.tb_params.k_neg :
2159 op->turbo_enc.tb_params.k_pos;
2165 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
2168 uint32_t c, r, tb_size = 0;
2169 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
2171 if (op->turbo_enc.code_block_mode) {
2172 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
2174 c = op->turbo_enc.tb_params.c;
2175 r = op->turbo_enc.tb_params.r;
2176 for (i = 0; i < c-r; i++)
2177 tb_size += sys_cols * op->ldpc_enc.z_c
2178 - op->ldpc_enc.n_filler;
2185 init_test_op_params(struct test_op_params *op_params,
2186 enum rte_bbdev_op_type op_type, const int expected_status,
2187 const int vector_mask, struct rte_mempool *ops_mp,
2188 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
2191 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2192 op_type == RTE_BBDEV_OP_LDPC_DEC)
2193 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
2194 &op_params->ref_dec_op, 1);
2196 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
2197 &op_params->ref_enc_op, 1);
2199 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
2201 op_params->mp = ops_mp;
2202 op_params->burst_sz = burst_sz;
2203 op_params->num_to_process = num_to_process;
2204 op_params->num_lcores = num_lcores;
2205 op_params->vector_mask = vector_mask;
2206 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
2207 op_type == RTE_BBDEV_OP_LDPC_DEC)
2208 op_params->ref_dec_op->status = expected_status;
2209 else if (op_type == RTE_BBDEV_OP_TURBO_ENC
2210 || op_type == RTE_BBDEV_OP_LDPC_ENC)
2211 op_params->ref_enc_op->status = expected_status;
2216 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
2217 struct test_op_params *op_params)
2219 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
2221 struct active_device *ad;
2222 unsigned int burst_sz = get_burst_sz();
2223 enum rte_bbdev_op_type op_type = test_vector.op_type;
2224 const struct rte_bbdev_op_cap *capabilities = NULL;
2226 ad = &active_devs[dev_id];
2228 /* Check if device supports op_type */
2229 if (!is_avail_op(ad, test_vector.op_type))
2230 return TEST_SUCCESS;
2232 struct rte_bbdev_info info;
2233 rte_bbdev_info_get(ad->dev_id, &info);
2234 socket_id = GET_SOCKET(info.socket_id);
2236 f_ret = create_mempools(ad, socket_id, op_type,
2238 if (f_ret != TEST_SUCCESS) {
2239 printf("Couldn't create mempools");
2242 if (op_type == RTE_BBDEV_OP_NONE)
2243 op_type = RTE_BBDEV_OP_TURBO_ENC;
2245 f_ret = init_test_op_params(op_params, test_vector.op_type,
2246 test_vector.expected_status,
2252 if (f_ret != TEST_SUCCESS) {
2253 printf("Couldn't init test op params");
2258 /* Find capabilities */
2259 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
2260 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
2261 if (cap->type == test_vector.op_type) {
2267 TEST_ASSERT_NOT_NULL(capabilities,
2268 "Couldn't find capabilities");
2270 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2271 create_reference_dec_op(op_params->ref_dec_op);
2272 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2273 create_reference_enc_op(op_params->ref_enc_op);
2274 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2275 create_reference_ldpc_enc_op(op_params->ref_enc_op);
2276 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2277 create_reference_ldpc_dec_op(op_params->ref_dec_op);
2279 for (i = 0; i < ad->nb_queues; ++i) {
2280 f_ret = fill_queue_buffers(op_params,
2282 ad->hard_out_mbuf_pool,
2283 ad->soft_out_mbuf_pool,
2284 ad->harq_in_mbuf_pool,
2285 ad->harq_out_mbuf_pool,
2288 info.drv.min_alignment,
2290 if (f_ret != TEST_SUCCESS) {
2291 printf("Couldn't init queue buffers");
2296 /* Run test case function */
2297 t_ret = test_case_func(ad, op_params);
2299 /* Free active device resources and return */
2300 free_buffers(ad, op_params);
2304 free_buffers(ad, op_params);
2308 /* Run given test function per active device per supported op type
2312 run_test_case(test_case_function *test_case_func)
2317 /* Alloc op_params */
2318 struct test_op_params *op_params = rte_zmalloc(NULL,
2319 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
2320 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
2321 RTE_ALIGN(sizeof(struct test_op_params),
2322 RTE_CACHE_LINE_SIZE));
2324 /* For each device run test case function */
2325 for (dev = 0; dev < nb_active_devs; ++dev)
2326 ret |= run_test_case_on_device(test_case_func, dev, op_params);
2328 rte_free(op_params);
2334 /* Push back the HARQ output from DDR to host */
2336 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2337 struct rte_bbdev_dec_op **ops,
2341 int save_status, ret;
2342 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2343 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2344 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2345 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK;
2346 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2347 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2348 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2349 for (j = 0; j < n; ++j) {
2350 if ((loopback && mem_out) || hc_out) {
2351 save_status = ops[j]->status;
2352 ops[j]->ldpc_dec.op_flags =
2353 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2354 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2356 ops[j]->ldpc_dec.op_flags +=
2357 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2358 ops[j]->ldpc_dec.harq_combined_input.offset =
2360 ops[j]->ldpc_dec.harq_combined_output.offset = 0;
2361 harq_offset += HARQ_INCR;
2363 ops[j]->ldpc_dec.harq_combined_input.length =
2364 ops[j]->ldpc_dec.harq_combined_output.length;
2365 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2369 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2372 ops[j]->ldpc_dec.op_flags = flags;
2373 ops[j]->status = save_status;
2379 * Push back the HARQ output from HW DDR to Host
2380 * Preload HARQ memory input and adjust HARQ offset
2383 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id,
2384 struct rte_bbdev_dec_op **ops, const uint16_t n,
2389 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024;
2390 struct rte_bbdev_op_data save_hc_in, save_hc_out;
2391 struct rte_bbdev_dec_op *ops_deq[MAX_BURST];
2392 uint32_t flags = ops[0]->ldpc_dec.op_flags;
2393 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE;
2394 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
2395 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2396 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
2397 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2398 for (j = 0; j < n; ++j) {
2399 if ((mem_in || hc_in) && preload) {
2400 save_hc_in = ops[j]->ldpc_dec.harq_combined_input;
2401 save_hc_out = ops[j]->ldpc_dec.harq_combined_output;
2402 ops[j]->ldpc_dec.op_flags =
2403 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK +
2404 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
2406 ops[j]->ldpc_dec.op_flags +=
2407 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION;
2408 ops[j]->ldpc_dec.harq_combined_output.offset =
2410 ops[j]->ldpc_dec.harq_combined_input.offset = 0;
2411 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2415 ret = rte_bbdev_dequeue_ldpc_dec_ops(
2416 dev_id, queue_id, &ops_deq[j], 1);
2417 ops[j]->ldpc_dec.op_flags = flags;
2418 ops[j]->ldpc_dec.harq_combined_input = save_hc_in;
2419 ops[j]->ldpc_dec.harq_combined_output = save_hc_out;
2421 /* Adjust HARQ offset when we reach external DDR */
2422 if (mem_in || hc_in)
2423 ops[j]->ldpc_dec.harq_combined_input.offset
2425 if (mem_out || hc_out)
2426 ops[j]->ldpc_dec.harq_combined_output.offset
2428 harq_offset += HARQ_INCR;
2433 dequeue_event_callback(uint16_t dev_id,
2434 enum rte_bbdev_event_type event, void *cb_arg,
2439 uint64_t total_time;
2440 uint16_t deq, burst_sz, num_ops;
2441 uint16_t queue_id = *(uint16_t *) ret_param;
2442 struct rte_bbdev_info info;
2444 struct thread_params *tp = cb_arg;
2446 /* Find matching thread params using queue_id */
2447 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
2448 if (tp->queue_id == queue_id)
2451 if (i == MAX_QUEUES) {
2452 printf("%s: Queue_id from interrupt details was not found!\n",
2457 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
2458 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2460 "Dequeue interrupt handler called for incorrect event!\n");
2464 burst_sz = rte_atomic16_read(&tp->burst_sz);
2465 num_ops = tp->op_params->num_to_process;
2467 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2468 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2470 rte_atomic16_read(&tp->nb_dequeued)],
2472 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2473 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2475 rte_atomic16_read(&tp->nb_dequeued)],
2477 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2478 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2480 rte_atomic16_read(&tp->nb_dequeued)],
2482 else /*RTE_BBDEV_OP_TURBO_ENC*/
2483 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2485 rte_atomic16_read(&tp->nb_dequeued)],
2488 if (deq < burst_sz) {
2490 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
2492 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2496 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
2497 rte_atomic16_add(&tp->nb_dequeued, deq);
2501 total_time = rte_rdtsc_precise() - tp->start_time;
2503 rte_bbdev_info_get(dev_id, &info);
2507 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
2508 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2509 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
2510 tp->op_params->vector_mask);
2511 /* get the max of iter_count for all dequeued ops */
2512 for (i = 0; i < num_ops; ++i)
2513 tp->iter_count = RTE_MAX(
2514 tp->dec_ops[i]->turbo_dec.iter_count,
2516 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2517 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
2518 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2519 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
2520 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2521 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
2522 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2523 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
2524 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
2525 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
2526 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2527 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
2528 tp->op_params->vector_mask);
2529 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
2533 printf("Buffers validation failed\n");
2534 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2537 switch (test_vector.op_type) {
2538 case RTE_BBDEV_OP_TURBO_DEC:
2539 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
2541 case RTE_BBDEV_OP_TURBO_ENC:
2542 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
2544 case RTE_BBDEV_OP_LDPC_DEC:
2545 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
2547 case RTE_BBDEV_OP_LDPC_ENC:
2548 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
2550 case RTE_BBDEV_OP_NONE:
2554 printf("Unknown op type: %d\n", test_vector.op_type);
2555 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
2559 tp->ops_per_sec += ((double)num_ops) /
2560 ((double)total_time / (double)rte_get_tsc_hz());
2561 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
2562 ((double)total_time / (double)rte_get_tsc_hz());
2564 rte_atomic16_add(&tp->nb_dequeued, deq);
2568 throughput_intr_lcore_ldpc_dec(void *arg)
2570 struct thread_params *tp = arg;
2571 unsigned int enqueued;
2572 const uint16_t queue_id = tp->queue_id;
2573 const uint16_t burst_sz = tp->op_params->burst_sz;
2574 const uint16_t num_to_process = tp->op_params->num_to_process;
2575 struct rte_bbdev_dec_op *ops[num_to_process];
2576 struct test_buffers *bufs = NULL;
2577 struct rte_bbdev_info info;
2579 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2580 uint16_t num_to_enq, enq;
2582 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
2583 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
2584 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
2585 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
2587 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2588 "BURST_SIZE should be <= %u", MAX_BURST);
2590 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2591 "Failed to enable interrupts for dev: %u, queue_id: %u",
2592 tp->dev_id, queue_id);
2594 rte_bbdev_info_get(tp->dev_id, &info);
2596 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2597 "NUM_OPS cannot exceed %u for this device",
2598 info.drv.queue_size_lim);
2600 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2602 rte_atomic16_clear(&tp->processing_status);
2603 rte_atomic16_clear(&tp->nb_dequeued);
2605 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2608 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2610 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2612 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2613 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
2614 bufs->hard_outputs, bufs->soft_outputs,
2615 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2617 /* Set counter to validate the ordering */
2618 for (j = 0; j < num_to_process; ++j)
2619 ops[j]->opaque_data = (void *)(uintptr_t)j;
2621 for (j = 0; j < TEST_REPETITIONS; ++j) {
2622 for (i = 0; i < num_to_process; ++i) {
2625 ops[i]->ldpc_dec.hard_output.data);
2626 if (hc_out || loopback)
2628 ops[i]->ldpc_dec.harq_combined_output.data);
2631 tp->start_time = rte_rdtsc_precise();
2632 for (enqueued = 0; enqueued < num_to_process;) {
2633 num_to_enq = burst_sz;
2635 if (unlikely(num_to_process - enqueued < num_to_enq))
2636 num_to_enq = num_to_process - enqueued;
2640 enq += rte_bbdev_enqueue_ldpc_dec_ops(
2642 queue_id, &ops[enqueued],
2644 } while (unlikely(num_to_enq != enq));
2647 /* Write to thread burst_sz current number of enqueued
2648 * descriptors. It ensures that proper number of
2649 * descriptors will be dequeued in callback
2650 * function - needed for last batch in case where
2651 * the number of operations is not a multiple of
2654 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2656 /* Wait until processing of previous batch is
2659 while (rte_atomic16_read(&tp->nb_dequeued) !=
2663 if (j != TEST_REPETITIONS - 1)
2664 rte_atomic16_clear(&tp->nb_dequeued);
2667 return TEST_SUCCESS;
2671 throughput_intr_lcore_dec(void *arg)
2673 struct thread_params *tp = arg;
2674 unsigned int enqueued;
2675 const uint16_t queue_id = tp->queue_id;
2676 const uint16_t burst_sz = tp->op_params->burst_sz;
2677 const uint16_t num_to_process = tp->op_params->num_to_process;
2678 struct rte_bbdev_dec_op *ops[num_to_process];
2679 struct test_buffers *bufs = NULL;
2680 struct rte_bbdev_info info;
2682 uint16_t num_to_enq, enq;
2684 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2685 "BURST_SIZE should be <= %u", MAX_BURST);
2687 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2688 "Failed to enable interrupts for dev: %u, queue_id: %u",
2689 tp->dev_id, queue_id);
2691 rte_bbdev_info_get(tp->dev_id, &info);
2693 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2694 "NUM_OPS cannot exceed %u for this device",
2695 info.drv.queue_size_lim);
2697 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2699 rte_atomic16_clear(&tp->processing_status);
2700 rte_atomic16_clear(&tp->nb_dequeued);
2702 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2705 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
2707 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2709 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2710 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
2711 bufs->hard_outputs, bufs->soft_outputs,
2712 tp->op_params->ref_dec_op);
2714 /* Set counter to validate the ordering */
2715 for (j = 0; j < num_to_process; ++j)
2716 ops[j]->opaque_data = (void *)(uintptr_t)j;
2718 for (j = 0; j < TEST_REPETITIONS; ++j) {
2719 for (i = 0; i < num_to_process; ++i)
2720 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
2722 tp->start_time = rte_rdtsc_precise();
2723 for (enqueued = 0; enqueued < num_to_process;) {
2724 num_to_enq = burst_sz;
2726 if (unlikely(num_to_process - enqueued < num_to_enq))
2727 num_to_enq = num_to_process - enqueued;
2731 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2732 queue_id, &ops[enqueued],
2734 } while (unlikely(num_to_enq != enq));
2737 /* Write to thread burst_sz current number of enqueued
2738 * descriptors. It ensures that proper number of
2739 * descriptors will be dequeued in callback
2740 * function - needed for last batch in case where
2741 * the number of operations is not a multiple of
2744 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2746 /* Wait until processing of previous batch is
2749 while (rte_atomic16_read(&tp->nb_dequeued) !=
2753 if (j != TEST_REPETITIONS - 1)
2754 rte_atomic16_clear(&tp->nb_dequeued);
2757 return TEST_SUCCESS;
2761 throughput_intr_lcore_enc(void *arg)
2763 struct thread_params *tp = arg;
2764 unsigned int enqueued;
2765 const uint16_t queue_id = tp->queue_id;
2766 const uint16_t burst_sz = tp->op_params->burst_sz;
2767 const uint16_t num_to_process = tp->op_params->num_to_process;
2768 struct rte_bbdev_enc_op *ops[num_to_process];
2769 struct test_buffers *bufs = NULL;
2770 struct rte_bbdev_info info;
2772 uint16_t num_to_enq, enq;
2774 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2775 "BURST_SIZE should be <= %u", MAX_BURST);
2777 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2778 "Failed to enable interrupts for dev: %u, queue_id: %u",
2779 tp->dev_id, queue_id);
2781 rte_bbdev_info_get(tp->dev_id, &info);
2783 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2784 "NUM_OPS cannot exceed %u for this device",
2785 info.drv.queue_size_lim);
2787 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2789 rte_atomic16_clear(&tp->processing_status);
2790 rte_atomic16_clear(&tp->nb_dequeued);
2792 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2795 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2797 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2799 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2800 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
2801 bufs->hard_outputs, tp->op_params->ref_enc_op);
2803 /* Set counter to validate the ordering */
2804 for (j = 0; j < num_to_process; ++j)
2805 ops[j]->opaque_data = (void *)(uintptr_t)j;
2807 for (j = 0; j < TEST_REPETITIONS; ++j) {
2808 for (i = 0; i < num_to_process; ++i)
2809 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2811 tp->start_time = rte_rdtsc_precise();
2812 for (enqueued = 0; enqueued < num_to_process;) {
2813 num_to_enq = burst_sz;
2815 if (unlikely(num_to_process - enqueued < num_to_enq))
2816 num_to_enq = num_to_process - enqueued;
2820 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2821 queue_id, &ops[enqueued],
2823 } while (unlikely(enq != num_to_enq));
2826 /* Write to thread burst_sz current number of enqueued
2827 * descriptors. It ensures that proper number of
2828 * descriptors will be dequeued in callback
2829 * function - needed for last batch in case where
2830 * the number of operations is not a multiple of
2833 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2835 /* Wait until processing of previous batch is
2838 while (rte_atomic16_read(&tp->nb_dequeued) !=
2842 if (j != TEST_REPETITIONS - 1)
2843 rte_atomic16_clear(&tp->nb_dequeued);
2846 return TEST_SUCCESS;
2851 throughput_intr_lcore_ldpc_enc(void *arg)
2853 struct thread_params *tp = arg;
2854 unsigned int enqueued;
2855 const uint16_t queue_id = tp->queue_id;
2856 const uint16_t burst_sz = tp->op_params->burst_sz;
2857 const uint16_t num_to_process = tp->op_params->num_to_process;
2858 struct rte_bbdev_enc_op *ops[num_to_process];
2859 struct test_buffers *bufs = NULL;
2860 struct rte_bbdev_info info;
2862 uint16_t num_to_enq, enq;
2864 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2865 "BURST_SIZE should be <= %u", MAX_BURST);
2867 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
2868 "Failed to enable interrupts for dev: %u, queue_id: %u",
2869 tp->dev_id, queue_id);
2871 rte_bbdev_info_get(tp->dev_id, &info);
2873 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
2874 "NUM_OPS cannot exceed %u for this device",
2875 info.drv.queue_size_lim);
2877 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2879 rte_atomic16_clear(&tp->processing_status);
2880 rte_atomic16_clear(&tp->nb_dequeued);
2882 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2885 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
2887 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2889 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2890 copy_reference_ldpc_enc_op(ops, num_to_process, 0,
2891 bufs->inputs, bufs->hard_outputs,
2892 tp->op_params->ref_enc_op);
2894 /* Set counter to validate the ordering */
2895 for (j = 0; j < num_to_process; ++j)
2896 ops[j]->opaque_data = (void *)(uintptr_t)j;
2898 for (j = 0; j < TEST_REPETITIONS; ++j) {
2899 for (i = 0; i < num_to_process; ++i)
2900 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
2902 tp->start_time = rte_rdtsc_precise();
2903 for (enqueued = 0; enqueued < num_to_process;) {
2904 num_to_enq = burst_sz;
2906 if (unlikely(num_to_process - enqueued < num_to_enq))
2907 num_to_enq = num_to_process - enqueued;
2911 enq += rte_bbdev_enqueue_ldpc_enc_ops(
2913 queue_id, &ops[enqueued],
2915 } while (unlikely(enq != num_to_enq));
2918 /* Write to thread burst_sz current number of enqueued
2919 * descriptors. It ensures that proper number of
2920 * descriptors will be dequeued in callback
2921 * function - needed for last batch in case where
2922 * the number of operations is not a multiple of
2925 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2927 /* Wait until processing of previous batch is
2930 while (rte_atomic16_read(&tp->nb_dequeued) !=
2934 if (j != TEST_REPETITIONS - 1)
2935 rte_atomic16_clear(&tp->nb_dequeued);
2938 return TEST_SUCCESS;
2942 throughput_pmd_lcore_dec(void *arg)
2944 struct thread_params *tp = arg;
2946 uint64_t total_time = 0, start_time;
2947 const uint16_t queue_id = tp->queue_id;
2948 const uint16_t burst_sz = tp->op_params->burst_sz;
2949 const uint16_t num_ops = tp->op_params->num_to_process;
2950 struct rte_bbdev_dec_op *ops_enq[num_ops];
2951 struct rte_bbdev_dec_op *ops_deq[num_ops];
2952 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2953 struct test_buffers *bufs = NULL;
2955 struct rte_bbdev_info info;
2956 uint16_t num_to_enq;
2958 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2959 "BURST_SIZE should be <= %u", MAX_BURST);
2961 rte_bbdev_info_get(tp->dev_id, &info);
2963 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2964 "NUM_OPS cannot exceed %u for this device",
2965 info.drv.queue_size_lim);
2967 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2969 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2972 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2973 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2975 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2976 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2977 bufs->hard_outputs, bufs->soft_outputs, ref_op);
2979 /* Set counter to validate the ordering */
2980 for (j = 0; j < num_ops; ++j)
2981 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2983 for (i = 0; i < TEST_REPETITIONS; ++i) {
2985 for (j = 0; j < num_ops; ++j)
2986 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
2988 start_time = rte_rdtsc_precise();
2990 for (enq = 0, deq = 0; enq < num_ops;) {
2991 num_to_enq = burst_sz;
2993 if (unlikely(num_ops - enq < num_to_enq))
2994 num_to_enq = num_ops - enq;
2996 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2997 queue_id, &ops_enq[enq], num_to_enq);
2999 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3000 queue_id, &ops_deq[deq], enq - deq);
3003 /* dequeue the remaining */
3005 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
3006 queue_id, &ops_deq[deq], enq - deq);
3009 total_time += rte_rdtsc_precise() - start_time;
3013 /* get the max of iter_count for all dequeued ops */
3014 for (i = 0; i < num_ops; ++i) {
3015 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
3019 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3020 ret = validate_dec_op(ops_deq, num_ops, ref_op,
3021 tp->op_params->vector_mask);
3022 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3025 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3027 double tb_len_bits = calc_dec_TB_size(ref_op);
3029 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3030 ((double)total_time / (double)rte_get_tsc_hz());
3031 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3032 1000000.0) / ((double)total_time /
3033 (double)rte_get_tsc_hz());
3035 return TEST_SUCCESS;
3039 bler_pmd_lcore_ldpc_dec(void *arg)
3041 struct thread_params *tp = arg;
3043 uint64_t total_time = 0, start_time;
3044 const uint16_t queue_id = tp->queue_id;
3045 const uint16_t burst_sz = tp->op_params->burst_sz;
3046 const uint16_t num_ops = tp->op_params->num_to_process;
3047 struct rte_bbdev_dec_op *ops_enq[num_ops];
3048 struct rte_bbdev_dec_op *ops_deq[num_ops];
3049 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3050 struct test_buffers *bufs = NULL;
3052 float parity_bler = 0;
3053 struct rte_bbdev_info info;
3054 uint16_t num_to_enq;
3055 bool extDdr = check_bit(ldpc_cap_flags,
3056 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3057 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3058 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3059 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3060 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3062 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3063 "BURST_SIZE should be <= %u", MAX_BURST);
3065 rte_bbdev_info_get(tp->dev_id, &info);
3067 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3068 "NUM_OPS cannot exceed %u for this device",
3069 info.drv.queue_size_lim);
3071 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3073 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3076 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3077 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3079 /* For BLER tests we need to enable early termination */
3080 if (!check_bit(ref_op->ldpc_dec.op_flags,
3081 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3082 ref_op->ldpc_dec.op_flags +=
3083 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3084 ref_op->ldpc_dec.iter_max = get_iter_max();
3085 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3087 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3088 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3089 bufs->hard_outputs, bufs->soft_outputs,
3090 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3091 generate_llr_input(num_ops, bufs->inputs, ref_op);
3093 /* Set counter to validate the ordering */
3094 for (j = 0; j < num_ops; ++j)
3095 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3097 for (i = 0; i < 1; ++i) { /* Could add more iterations */
3098 for (j = 0; j < num_ops; ++j) {
3101 ops_enq[j]->ldpc_dec.hard_output.data);
3102 if (hc_out || loopback)
3104 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3107 bool preload = i == (TEST_REPETITIONS - 1);
3108 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3111 start_time = rte_rdtsc_precise();
3113 for (enq = 0, deq = 0; enq < num_ops;) {
3114 num_to_enq = burst_sz;
3116 if (unlikely(num_ops - enq < num_to_enq))
3117 num_to_enq = num_ops - enq;
3119 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3120 queue_id, &ops_enq[enq], num_to_enq);
3122 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3123 queue_id, &ops_deq[deq], enq - deq);
3126 /* dequeue the remaining */
3128 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3129 queue_id, &ops_deq[deq], enq - deq);
3132 total_time += rte_rdtsc_precise() - start_time;
3136 tp->iter_average = 0;
3137 /* get the max of iter_count for all dequeued ops */
3138 for (i = 0; i < num_ops; ++i) {
3139 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3141 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count;
3142 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR))
3146 parity_bler /= num_ops; /* This one is based on SYND */
3147 tp->iter_average /= num_ops;
3148 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops;
3150 if (test_vector.op_type != RTE_BBDEV_OP_NONE
3154 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3155 tp->op_params->vector_mask);
3156 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3159 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3161 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3162 tp->ops_per_sec = ((double)num_ops * 1) /
3163 ((double)total_time / (double)rte_get_tsc_hz());
3164 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) /
3165 1000000.0) / ((double)total_time /
3166 (double)rte_get_tsc_hz());
3168 return TEST_SUCCESS;
3172 throughput_pmd_lcore_ldpc_dec(void *arg)
3174 struct thread_params *tp = arg;
3176 uint64_t total_time = 0, start_time;
3177 const uint16_t queue_id = tp->queue_id;
3178 const uint16_t burst_sz = tp->op_params->burst_sz;
3179 const uint16_t num_ops = tp->op_params->num_to_process;
3180 struct rte_bbdev_dec_op *ops_enq[num_ops];
3181 struct rte_bbdev_dec_op *ops_deq[num_ops];
3182 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
3183 struct test_buffers *bufs = NULL;
3185 struct rte_bbdev_info info;
3186 uint16_t num_to_enq;
3187 bool extDdr = check_bit(ldpc_cap_flags,
3188 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE);
3189 bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
3190 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
3191 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
3192 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
3194 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3195 "BURST_SIZE should be <= %u", MAX_BURST);
3197 rte_bbdev_info_get(tp->dev_id, &info);
3199 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3200 "NUM_OPS cannot exceed %u for this device",
3201 info.drv.queue_size_lim);
3203 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3205 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3208 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
3209 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
3211 /* For throughput tests we need to disable early termination */
3212 if (check_bit(ref_op->ldpc_dec.op_flags,
3213 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3214 ref_op->ldpc_dec.op_flags -=
3215 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3216 ref_op->ldpc_dec.iter_max = get_iter_max();
3217 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3219 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3220 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
3221 bufs->hard_outputs, bufs->soft_outputs,
3222 bufs->harq_inputs, bufs->harq_outputs, ref_op);
3224 /* Set counter to validate the ordering */
3225 for (j = 0; j < num_ops; ++j)
3226 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3228 for (i = 0; i < TEST_REPETITIONS; ++i) {
3229 for (j = 0; j < num_ops; ++j) {
3232 ops_enq[j]->ldpc_dec.hard_output.data);
3233 if (hc_out || loopback)
3235 ops_enq[j]->ldpc_dec.harq_combined_output.data);
3238 bool preload = i == (TEST_REPETITIONS - 1);
3239 preload_harq_ddr(tp->dev_id, queue_id, ops_enq,
3242 start_time = rte_rdtsc_precise();
3244 for (enq = 0, deq = 0; enq < num_ops;) {
3245 num_to_enq = burst_sz;
3247 if (unlikely(num_ops - enq < num_to_enq))
3248 num_to_enq = num_ops - enq;
3250 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
3251 queue_id, &ops_enq[enq], num_to_enq);
3253 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3254 queue_id, &ops_deq[deq], enq - deq);
3257 /* dequeue the remaining */
3259 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
3260 queue_id, &ops_deq[deq], enq - deq);
3263 total_time += rte_rdtsc_precise() - start_time;
3267 /* get the max of iter_count for all dequeued ops */
3268 for (i = 0; i < num_ops; ++i) {
3269 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
3273 /* Read loopback is not thread safe */
3274 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops);
3277 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3278 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
3279 tp->op_params->vector_mask);
3280 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3283 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
3285 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
3287 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3288 ((double)total_time / (double)rte_get_tsc_hz());
3289 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
3290 1000000.0) / ((double)total_time /
3291 (double)rte_get_tsc_hz());
3293 return TEST_SUCCESS;
3297 throughput_pmd_lcore_enc(void *arg)
3299 struct thread_params *tp = arg;
3301 uint64_t total_time = 0, start_time;
3302 const uint16_t queue_id = tp->queue_id;
3303 const uint16_t burst_sz = tp->op_params->burst_sz;
3304 const uint16_t num_ops = tp->op_params->num_to_process;
3305 struct rte_bbdev_enc_op *ops_enq[num_ops];
3306 struct rte_bbdev_enc_op *ops_deq[num_ops];
3307 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3308 struct test_buffers *bufs = NULL;
3310 struct rte_bbdev_info info;
3311 uint16_t num_to_enq;
3313 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3314 "BURST_SIZE should be <= %u", MAX_BURST);
3316 rte_bbdev_info_get(tp->dev_id, &info);
3318 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3319 "NUM_OPS cannot exceed %u for this device",
3320 info.drv.queue_size_lim);
3322 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3324 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3327 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3329 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3331 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3332 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3333 bufs->hard_outputs, ref_op);
3335 /* Set counter to validate the ordering */
3336 for (j = 0; j < num_ops; ++j)
3337 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3339 for (i = 0; i < TEST_REPETITIONS; ++i) {
3341 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3342 for (j = 0; j < num_ops; ++j)
3343 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3345 start_time = rte_rdtsc_precise();
3347 for (enq = 0, deq = 0; enq < num_ops;) {
3348 num_to_enq = burst_sz;
3350 if (unlikely(num_ops - enq < num_to_enq))
3351 num_to_enq = num_ops - enq;
3353 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
3354 queue_id, &ops_enq[enq], num_to_enq);
3356 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3357 queue_id, &ops_deq[deq], enq - deq);
3360 /* dequeue the remaining */
3362 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
3363 queue_id, &ops_deq[deq], enq - deq);
3366 total_time += rte_rdtsc_precise() - start_time;
3369 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3370 ret = validate_enc_op(ops_deq, num_ops, ref_op);
3371 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3374 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3376 double tb_len_bits = calc_enc_TB_size(ref_op);
3378 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3379 ((double)total_time / (double)rte_get_tsc_hz());
3380 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3381 / 1000000.0) / ((double)total_time /
3382 (double)rte_get_tsc_hz());
3384 return TEST_SUCCESS;
3388 throughput_pmd_lcore_ldpc_enc(void *arg)
3390 struct thread_params *tp = arg;
3392 uint64_t total_time = 0, start_time;
3393 const uint16_t queue_id = tp->queue_id;
3394 const uint16_t burst_sz = tp->op_params->burst_sz;
3395 const uint16_t num_ops = tp->op_params->num_to_process;
3396 struct rte_bbdev_enc_op *ops_enq[num_ops];
3397 struct rte_bbdev_enc_op *ops_deq[num_ops];
3398 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
3399 struct test_buffers *bufs = NULL;
3401 struct rte_bbdev_info info;
3402 uint16_t num_to_enq;
3404 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3405 "BURST_SIZE should be <= %u", MAX_BURST);
3407 rte_bbdev_info_get(tp->dev_id, &info);
3409 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
3410 "NUM_OPS cannot exceed %u for this device",
3411 info.drv.queue_size_lim);
3413 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3415 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
3418 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
3420 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
3422 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3423 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
3424 bufs->hard_outputs, ref_op);
3426 /* Set counter to validate the ordering */
3427 for (j = 0; j < num_ops; ++j)
3428 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3430 for (i = 0; i < TEST_REPETITIONS; ++i) {
3432 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3433 for (j = 0; j < num_ops; ++j)
3434 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
3436 start_time = rte_rdtsc_precise();
3438 for (enq = 0, deq = 0; enq < num_ops;) {
3439 num_to_enq = burst_sz;
3441 if (unlikely(num_ops - enq < num_to_enq))
3442 num_to_enq = num_ops - enq;
3444 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
3445 queue_id, &ops_enq[enq], num_to_enq);
3447 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3448 queue_id, &ops_deq[deq], enq - deq);
3451 /* dequeue the remaining */
3453 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
3454 queue_id, &ops_deq[deq], enq - deq);
3457 total_time += rte_rdtsc_precise() - start_time;
3460 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3461 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
3462 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3465 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
3467 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
3469 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
3470 ((double)total_time / (double)rte_get_tsc_hz());
3471 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
3472 / 1000000.0) / ((double)total_time /
3473 (double)rte_get_tsc_hz());
3475 return TEST_SUCCESS;
3479 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
3481 unsigned int iter = 0;
3482 double total_mops = 0, total_mbps = 0;
3484 for (iter = 0; iter < used_cores; iter++) {
3486 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
3487 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
3488 t_params[iter].mbps);
3489 total_mops += t_params[iter].ops_per_sec;
3490 total_mbps += t_params[iter].mbps;
3493 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
3494 used_cores, total_mops, total_mbps);
3497 /* Aggregate the performance results over the number of cores used */
3499 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
3501 unsigned int core_idx = 0;
3502 double total_mops = 0, total_mbps = 0;
3503 uint8_t iter_count = 0;
3505 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3507 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
3508 t_params[core_idx].lcore_id,
3509 t_params[core_idx].ops_per_sec,
3510 t_params[core_idx].mbps,
3511 t_params[core_idx].iter_count);
3512 total_mops += t_params[core_idx].ops_per_sec;
3513 total_mbps += t_params[core_idx].mbps;
3514 iter_count = RTE_MAX(iter_count,
3515 t_params[core_idx].iter_count);
3518 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
3519 used_cores, total_mops, total_mbps, iter_count);
3522 /* Aggregate the performance results over the number of cores used */
3524 print_dec_bler(struct thread_params *t_params, unsigned int used_cores)
3526 unsigned int core_idx = 0;
3527 double total_mbps = 0, total_bler = 0, total_iter = 0;
3528 double snr = get_snr();
3530 for (core_idx = 0; core_idx < used_cores; core_idx++) {
3531 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n",
3532 t_params[core_idx].lcore_id,
3533 t_params[core_idx].bler * 100,
3534 t_params[core_idx].iter_average,
3535 t_params[core_idx].mbps,
3536 get_vector_filename());
3537 total_mbps += t_params[core_idx].mbps;
3538 total_bler += t_params[core_idx].bler;
3539 total_iter += t_params[core_idx].iter_average;
3541 total_bler /= used_cores;
3542 total_iter /= used_cores;
3544 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n",
3545 snr, total_bler * 100, total_iter, get_iter_max(),
3546 total_mbps, get_vector_filename());
3550 * Test function that determines BLER wireless performance
3553 bler_test(struct active_device *ad,
3554 struct test_op_params *op_params)
3557 unsigned int lcore_id, used_cores = 0;
3558 struct thread_params *t_params;
3559 struct rte_bbdev_info info;
3560 lcore_function_t *bler_function;
3561 uint16_t num_lcores;
3562 const char *op_type_str;
3564 rte_bbdev_info_get(ad->dev_id, &info);
3566 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3567 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3568 test_vector.op_type);
3570 printf("+ ------------------------------------------------------- +\n");
3571 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3572 info.dev_name, ad->nb_queues, op_params->burst_sz,
3573 op_params->num_to_process, op_params->num_lcores,
3575 intr_enabled ? "Interrupt mode" : "PMD mode",
3576 (double)rte_get_tsc_hz() / 1000000000.0);
3578 /* Set number of lcores */
3579 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3581 : op_params->num_lcores;
3583 /* Allocate memory for thread parameters structure */
3584 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3585 RTE_CACHE_LINE_SIZE);
3586 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3587 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3588 RTE_CACHE_LINE_SIZE));
3590 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3591 bler_function = bler_pmd_lcore_ldpc_dec;
3593 return TEST_SKIPPED;
3595 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3597 /* Master core is set at first entry */
3598 t_params[0].dev_id = ad->dev_id;
3599 t_params[0].lcore_id = rte_lcore_id();
3600 t_params[0].op_params = op_params;
3601 t_params[0].queue_id = ad->queue_ids[used_cores++];
3602 t_params[0].iter_count = 0;
3604 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3605 if (used_cores >= num_lcores)
3608 t_params[used_cores].dev_id = ad->dev_id;
3609 t_params[used_cores].lcore_id = lcore_id;
3610 t_params[used_cores].op_params = op_params;
3611 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3612 t_params[used_cores].iter_count = 0;
3614 rte_eal_remote_launch(bler_function,
3615 &t_params[used_cores++], lcore_id);
3618 rte_atomic16_set(&op_params->sync, SYNC_START);
3619 ret = bler_function(&t_params[0]);
3621 /* Master core is always used */
3622 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3623 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3625 print_dec_bler(t_params, num_lcores);
3627 /* Return if test failed */
3633 /* Function to print something here*/
3639 * Test function that determines how long an enqueue + dequeue of a burst
3640 * takes on available lcores.
3643 throughput_test(struct active_device *ad,
3644 struct test_op_params *op_params)
3647 unsigned int lcore_id, used_cores = 0;
3648 struct thread_params *t_params, *tp;
3649 struct rte_bbdev_info info;
3650 lcore_function_t *throughput_function;
3651 uint16_t num_lcores;
3652 const char *op_type_str;
3654 rte_bbdev_info_get(ad->dev_id, &info);
3656 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
3657 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
3658 test_vector.op_type);
3660 printf("+ ------------------------------------------------------- +\n");
3661 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
3662 info.dev_name, ad->nb_queues, op_params->burst_sz,
3663 op_params->num_to_process, op_params->num_lcores,
3665 intr_enabled ? "Interrupt mode" : "PMD mode",
3666 (double)rte_get_tsc_hz() / 1000000000.0);
3668 /* Set number of lcores */
3669 num_lcores = (ad->nb_queues < (op_params->num_lcores))
3671 : op_params->num_lcores;
3673 /* Allocate memory for thread parameters structure */
3674 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
3675 RTE_CACHE_LINE_SIZE);
3676 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
3677 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
3678 RTE_CACHE_LINE_SIZE));
3681 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3682 throughput_function = throughput_intr_lcore_dec;
3683 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3684 throughput_function = throughput_intr_lcore_ldpc_dec;
3685 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3686 throughput_function = throughput_intr_lcore_enc;
3687 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3688 throughput_function = throughput_intr_lcore_ldpc_enc;
3690 throughput_function = throughput_intr_lcore_enc;
3692 /* Dequeue interrupt callback registration */
3693 ret = rte_bbdev_callback_register(ad->dev_id,
3694 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
3701 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
3702 throughput_function = throughput_pmd_lcore_dec;
3703 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3704 throughput_function = throughput_pmd_lcore_ldpc_dec;
3705 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
3706 throughput_function = throughput_pmd_lcore_enc;
3707 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3708 throughput_function = throughput_pmd_lcore_ldpc_enc;
3710 throughput_function = throughput_pmd_lcore_enc;
3713 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
3715 /* Master core is set at first entry */
3716 t_params[0].dev_id = ad->dev_id;
3717 t_params[0].lcore_id = rte_lcore_id();
3718 t_params[0].op_params = op_params;
3719 t_params[0].queue_id = ad->queue_ids[used_cores++];
3720 t_params[0].iter_count = 0;
3722 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3723 if (used_cores >= num_lcores)
3726 t_params[used_cores].dev_id = ad->dev_id;
3727 t_params[used_cores].lcore_id = lcore_id;
3728 t_params[used_cores].op_params = op_params;
3729 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
3730 t_params[used_cores].iter_count = 0;
3732 rte_eal_remote_launch(throughput_function,
3733 &t_params[used_cores++], lcore_id);
3736 rte_atomic16_set(&op_params->sync, SYNC_START);
3737 ret = throughput_function(&t_params[0]);
3739 /* Master core is always used */
3740 for (used_cores = 1; used_cores < num_lcores; used_cores++)
3741 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
3743 /* Return if test failed */
3749 /* Print throughput if interrupts are disabled and test passed */
3750 if (!intr_enabled) {
3751 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3752 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3753 print_dec_throughput(t_params, num_lcores);
3755 print_enc_throughput(t_params, num_lcores);
3760 /* In interrupt TC we need to wait for the interrupt callback to deqeue
3761 * all pending operations. Skip waiting for queues which reported an
3762 * error using processing_status variable.
3763 * Wait for master lcore operations.
3766 while ((rte_atomic16_read(&tp->nb_dequeued) <
3767 op_params->num_to_process) &&
3768 (rte_atomic16_read(&tp->processing_status) !=
3772 tp->ops_per_sec /= TEST_REPETITIONS;
3773 tp->mbps /= TEST_REPETITIONS;
3774 ret |= (int)rte_atomic16_read(&tp->processing_status);
3776 /* Wait for slave lcores operations */
3777 for (used_cores = 1; used_cores < num_lcores; used_cores++) {
3778 tp = &t_params[used_cores];
3780 while ((rte_atomic16_read(&tp->nb_dequeued) <
3781 op_params->num_to_process) &&
3782 (rte_atomic16_read(&tp->processing_status) !=
3786 tp->ops_per_sec /= TEST_REPETITIONS;
3787 tp->mbps /= TEST_REPETITIONS;
3788 ret |= (int)rte_atomic16_read(&tp->processing_status);
3791 /* Print throughput if test passed */
3793 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
3794 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
3795 print_dec_throughput(t_params, num_lcores);
3796 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
3797 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
3798 print_enc_throughput(t_params, num_lcores);
3806 latency_test_dec(struct rte_mempool *mempool,
3807 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3808 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3809 const uint16_t num_to_process, uint16_t burst_sz,
3810 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3812 int ret = TEST_SUCCESS;
3813 uint16_t i, j, dequeued;
3814 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3815 uint64_t start_time = 0, last_time = 0;
3817 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3818 uint16_t enq = 0, deq = 0;
3819 bool first_time = true;
3822 if (unlikely(num_to_process - dequeued < burst_sz))
3823 burst_sz = num_to_process - dequeued;
3825 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3826 TEST_ASSERT_SUCCESS(ret,
3827 "rte_bbdev_dec_op_alloc_bulk() failed");
3828 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3829 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3835 /* Set counter to validate the ordering */
3836 for (j = 0; j < burst_sz; ++j)
3837 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3839 start_time = rte_rdtsc_precise();
3841 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
3843 TEST_ASSERT(enq == burst_sz,
3844 "Error enqueueing burst, expected %u, got %u",
3849 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3850 &ops_deq[deq], burst_sz - deq);
3851 if (likely(first_time && (deq > 0))) {
3852 last_time = rte_rdtsc_precise() - start_time;
3855 } while (unlikely(burst_sz != deq));
3857 *max_time = RTE_MAX(*max_time, last_time);
3858 *min_time = RTE_MIN(*min_time, last_time);
3859 *total_time += last_time;
3861 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3862 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
3864 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3867 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3875 latency_test_ldpc_dec(struct rte_mempool *mempool,
3876 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
3877 int vector_mask, uint16_t dev_id, uint16_t queue_id,
3878 const uint16_t num_to_process, uint16_t burst_sz,
3879 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3881 int ret = TEST_SUCCESS;
3882 uint16_t i, j, dequeued;
3883 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3884 uint64_t start_time = 0, last_time = 0;
3885 bool extDdr = ldpc_cap_flags &
3886 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
3888 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3889 uint16_t enq = 0, deq = 0;
3890 bool first_time = true;
3893 if (unlikely(num_to_process - dequeued < burst_sz))
3894 burst_sz = num_to_process - dequeued;
3896 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3897 TEST_ASSERT_SUCCESS(ret,
3898 "rte_bbdev_dec_op_alloc_bulk() failed");
3900 /* For latency tests we need to disable early termination */
3901 if (check_bit(ref_op->ldpc_dec.op_flags,
3902 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
3903 ref_op->ldpc_dec.op_flags -=
3904 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
3905 ref_op->ldpc_dec.iter_max = get_iter_max();
3906 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
3908 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3909 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
3918 preload_harq_ddr(dev_id, queue_id, ops_enq,
3921 /* Set counter to validate the ordering */
3922 for (j = 0; j < burst_sz; ++j)
3923 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3925 start_time = rte_rdtsc_precise();
3927 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
3928 &ops_enq[enq], burst_sz);
3929 TEST_ASSERT(enq == burst_sz,
3930 "Error enqueueing burst, expected %u, got %u",
3935 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
3936 &ops_deq[deq], burst_sz - deq);
3937 if (likely(first_time && (deq > 0))) {
3938 last_time = rte_rdtsc_precise() - start_time;
3941 } while (unlikely(burst_sz != deq));
3943 *max_time = RTE_MAX(*max_time, last_time);
3944 *min_time = RTE_MIN(*min_time, last_time);
3945 *total_time += last_time;
3948 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
3950 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
3951 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
3953 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
3956 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3963 latency_test_enc(struct rte_mempool *mempool,
3964 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
3965 uint16_t dev_id, uint16_t queue_id,
3966 const uint16_t num_to_process, uint16_t burst_sz,
3967 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
3969 int ret = TEST_SUCCESS;
3970 uint16_t i, j, dequeued;
3971 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3972 uint64_t start_time = 0, last_time = 0;
3974 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3975 uint16_t enq = 0, deq = 0;
3976 bool first_time = true;
3979 if (unlikely(num_to_process - dequeued < burst_sz))
3980 burst_sz = num_to_process - dequeued;
3982 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3983 TEST_ASSERT_SUCCESS(ret,
3984 "rte_bbdev_enc_op_alloc_bulk() failed");
3985 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3986 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
3991 /* Set counter to validate the ordering */
3992 for (j = 0; j < burst_sz; ++j)
3993 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
3995 start_time = rte_rdtsc_precise();
3997 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
3999 TEST_ASSERT(enq == burst_sz,
4000 "Error enqueueing burst, expected %u, got %u",
4005 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4006 &ops_deq[deq], burst_sz - deq);
4007 if (likely(first_time && (deq > 0))) {
4008 last_time += rte_rdtsc_precise() - start_time;
4011 } while (unlikely(burst_sz != deq));
4013 *max_time = RTE_MAX(*max_time, last_time);
4014 *min_time = RTE_MIN(*min_time, last_time);
4015 *total_time += last_time;
4017 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4018 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4019 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4022 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4030 latency_test_ldpc_enc(struct rte_mempool *mempool,
4031 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
4032 uint16_t dev_id, uint16_t queue_id,
4033 const uint16_t num_to_process, uint16_t burst_sz,
4034 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
4036 int ret = TEST_SUCCESS;
4037 uint16_t i, j, dequeued;
4038 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4039 uint64_t start_time = 0, last_time = 0;
4041 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4042 uint16_t enq = 0, deq = 0;
4043 bool first_time = true;
4046 if (unlikely(num_to_process - dequeued < burst_sz))
4047 burst_sz = num_to_process - dequeued;
4049 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4050 TEST_ASSERT_SUCCESS(ret,
4051 "rte_bbdev_enc_op_alloc_bulk() failed");
4052 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4053 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4058 /* Set counter to validate the ordering */
4059 for (j = 0; j < burst_sz; ++j)
4060 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
4062 start_time = rte_rdtsc_precise();
4064 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4065 &ops_enq[enq], burst_sz);
4066 TEST_ASSERT(enq == burst_sz,
4067 "Error enqueueing burst, expected %u, got %u",
4072 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4073 &ops_deq[deq], burst_sz - deq);
4074 if (likely(first_time && (deq > 0))) {
4075 last_time += rte_rdtsc_precise() - start_time;
4078 } while (unlikely(burst_sz != deq));
4080 *max_time = RTE_MAX(*max_time, last_time);
4081 *min_time = RTE_MIN(*min_time, last_time);
4082 *total_time += last_time;
4084 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
4085 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
4086 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
4089 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4097 latency_test(struct active_device *ad,
4098 struct test_op_params *op_params)
4101 uint16_t burst_sz = op_params->burst_sz;
4102 const uint16_t num_to_process = op_params->num_to_process;
4103 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4104 const uint16_t queue_id = ad->queue_ids[0];
4105 struct test_buffers *bufs = NULL;
4106 struct rte_bbdev_info info;
4107 uint64_t total_time, min_time, max_time;
4108 const char *op_type_str;
4110 total_time = max_time = 0;
4111 min_time = UINT64_MAX;
4113 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4114 "BURST_SIZE should be <= %u", MAX_BURST);
4116 rte_bbdev_info_get(ad->dev_id, &info);
4117 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4119 op_type_str = rte_bbdev_op_type_str(op_type);
4120 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4122 printf("+ ------------------------------------------------------- +\n");
4123 printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4124 info.dev_name, burst_sz, num_to_process, op_type_str);
4126 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4127 iter = latency_test_dec(op_params->mp, bufs,
4128 op_params->ref_dec_op, op_params->vector_mask,
4129 ad->dev_id, queue_id, num_to_process,
4130 burst_sz, &total_time, &min_time, &max_time);
4131 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4132 iter = latency_test_enc(op_params->mp, bufs,
4133 op_params->ref_enc_op, ad->dev_id, queue_id,
4134 num_to_process, burst_sz, &total_time,
4135 &min_time, &max_time);
4136 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4137 iter = latency_test_ldpc_enc(op_params->mp, bufs,
4138 op_params->ref_enc_op, ad->dev_id, queue_id,
4139 num_to_process, burst_sz, &total_time,
4140 &min_time, &max_time);
4141 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4142 iter = latency_test_ldpc_dec(op_params->mp, bufs,
4143 op_params->ref_dec_op, op_params->vector_mask,
4144 ad->dev_id, queue_id, num_to_process,
4145 burst_sz, &total_time, &min_time, &max_time);
4147 iter = latency_test_enc(op_params->mp, bufs,
4148 op_params->ref_enc_op,
4149 ad->dev_id, queue_id,
4150 num_to_process, burst_sz, &total_time,
4151 &min_time, &max_time);
4156 printf("Operation latency:\n"
4157 "\tavg: %lg cycles, %lg us\n"
4158 "\tmin: %lg cycles, %lg us\n"
4159 "\tmax: %lg cycles, %lg us\n",
4160 (double)total_time / (double)iter,
4161 (double)(total_time * 1000000) / (double)iter /
4162 (double)rte_get_tsc_hz(), (double)min_time,
4163 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
4164 (double)max_time, (double)(max_time * 1000000) /
4165 (double)rte_get_tsc_hz());
4167 return TEST_SUCCESS;
4170 #ifdef RTE_BBDEV_OFFLOAD_COST
4172 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
4173 struct rte_bbdev_stats *stats)
4175 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
4176 struct rte_bbdev_stats *q_stats;
4178 if (queue_id >= dev->data->num_queues)
4181 q_stats = &dev->data->queues[queue_id].queue_stats;
4183 stats->enqueued_count = q_stats->enqueued_count;
4184 stats->dequeued_count = q_stats->dequeued_count;
4185 stats->enqueue_err_count = q_stats->enqueue_err_count;
4186 stats->dequeue_err_count = q_stats->dequeue_err_count;
4187 stats->acc_offload_cycles = q_stats->acc_offload_cycles;
4193 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
4194 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4195 uint16_t queue_id, const uint16_t num_to_process,
4196 uint16_t burst_sz, struct test_time_stats *time_st)
4198 int i, dequeued, ret;
4199 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4200 uint64_t enq_start_time, deq_start_time;
4201 uint64_t enq_sw_last_time, deq_last_time;
4202 struct rte_bbdev_stats stats;
4204 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4205 uint16_t enq = 0, deq = 0;
4207 if (unlikely(num_to_process - dequeued < burst_sz))
4208 burst_sz = num_to_process - dequeued;
4210 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4211 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4212 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
4218 /* Start time meas for enqueue function offload latency */
4219 enq_start_time = rte_rdtsc_precise();
4221 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
4222 &ops_enq[enq], burst_sz - enq);
4223 } while (unlikely(burst_sz != enq));
4225 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4226 TEST_ASSERT_SUCCESS(ret,
4227 "Failed to get stats for queue (%u) of device (%u)",
4230 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
4231 stats.acc_offload_cycles;
4232 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4234 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4236 time_st->enq_sw_total_time += enq_sw_last_time;
4238 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4239 stats.acc_offload_cycles);
4240 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4241 stats.acc_offload_cycles);
4242 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4244 /* give time for device to process ops */
4247 /* Start time meas for dequeue function offload latency */
4248 deq_start_time = rte_rdtsc_precise();
4249 /* Dequeue one operation */
4251 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4253 } while (unlikely(deq != 1));
4255 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4256 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4258 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4260 time_st->deq_total_time += deq_last_time;
4262 /* Dequeue remaining operations if needed*/
4263 while (burst_sz != deq)
4264 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
4265 &ops_deq[deq], burst_sz - deq);
4267 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4275 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
4276 struct test_buffers *bufs,
4277 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
4278 uint16_t queue_id, const uint16_t num_to_process,
4279 uint16_t burst_sz, struct test_time_stats *time_st)
4281 int i, dequeued, ret;
4282 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4283 uint64_t enq_start_time, deq_start_time;
4284 uint64_t enq_sw_last_time, deq_last_time;
4285 struct rte_bbdev_stats stats;
4286 bool extDdr = ldpc_cap_flags &
4287 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
4289 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4290 uint16_t enq = 0, deq = 0;
4292 if (unlikely(num_to_process - dequeued < burst_sz))
4293 burst_sz = num_to_process - dequeued;
4295 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
4296 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4297 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
4306 preload_harq_ddr(dev_id, queue_id, ops_enq,
4309 /* Start time meas for enqueue function offload latency */
4310 enq_start_time = rte_rdtsc_precise();
4312 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
4313 &ops_enq[enq], burst_sz - enq);
4314 } while (unlikely(burst_sz != enq));
4316 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4317 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4318 TEST_ASSERT_SUCCESS(ret,
4319 "Failed to get stats for queue (%u) of device (%u)",
4322 enq_sw_last_time -= stats.acc_offload_cycles;
4323 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4325 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4327 time_st->enq_sw_total_time += enq_sw_last_time;
4329 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4330 stats.acc_offload_cycles);
4331 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4332 stats.acc_offload_cycles);
4333 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4335 /* give time for device to process ops */
4338 /* Start time meas for dequeue function offload latency */
4339 deq_start_time = rte_rdtsc_precise();
4340 /* Dequeue one operation */
4342 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4344 } while (unlikely(deq != 1));
4346 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4347 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4349 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4351 time_st->deq_total_time += deq_last_time;
4353 /* Dequeue remaining operations if needed*/
4354 while (burst_sz != deq)
4355 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
4356 &ops_deq[deq], burst_sz - deq);
4359 /* Read loopback is not thread safe */
4360 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz);
4363 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
4371 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
4372 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4373 uint16_t queue_id, const uint16_t num_to_process,
4374 uint16_t burst_sz, struct test_time_stats *time_st)
4376 int i, dequeued, ret;
4377 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4378 uint64_t enq_start_time, deq_start_time;
4379 uint64_t enq_sw_last_time, deq_last_time;
4380 struct rte_bbdev_stats stats;
4382 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4383 uint16_t enq = 0, deq = 0;
4385 if (unlikely(num_to_process - dequeued < burst_sz))
4386 burst_sz = num_to_process - dequeued;
4388 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4389 TEST_ASSERT_SUCCESS(ret,
4390 "rte_bbdev_enc_op_alloc_bulk() failed");
4391 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4392 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
4397 /* Start time meas for enqueue function offload latency */
4398 enq_start_time = rte_rdtsc_precise();
4400 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
4401 &ops_enq[enq], burst_sz - enq);
4402 } while (unlikely(burst_sz != enq));
4404 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4406 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4407 TEST_ASSERT_SUCCESS(ret,
4408 "Failed to get stats for queue (%u) of device (%u)",
4410 enq_sw_last_time -= stats.acc_offload_cycles;
4411 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4413 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4415 time_st->enq_sw_total_time += enq_sw_last_time;
4417 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4418 stats.acc_offload_cycles);
4419 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4420 stats.acc_offload_cycles);
4421 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4423 /* give time for device to process ops */
4426 /* Start time meas for dequeue function offload latency */
4427 deq_start_time = rte_rdtsc_precise();
4428 /* Dequeue one operation */
4430 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4432 } while (unlikely(deq != 1));
4434 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4435 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4437 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4439 time_st->deq_total_time += deq_last_time;
4441 while (burst_sz != deq)
4442 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
4443 &ops_deq[deq], burst_sz - deq);
4445 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4453 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
4454 struct test_buffers *bufs,
4455 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
4456 uint16_t queue_id, const uint16_t num_to_process,
4457 uint16_t burst_sz, struct test_time_stats *time_st)
4459 int i, dequeued, ret;
4460 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
4461 uint64_t enq_start_time, deq_start_time;
4462 uint64_t enq_sw_last_time, deq_last_time;
4463 struct rte_bbdev_stats stats;
4465 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
4466 uint16_t enq = 0, deq = 0;
4468 if (unlikely(num_to_process - dequeued < burst_sz))
4469 burst_sz = num_to_process - dequeued;
4471 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
4472 TEST_ASSERT_SUCCESS(ret,
4473 "rte_bbdev_enc_op_alloc_bulk() failed");
4474 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
4475 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
4480 /* Start time meas for enqueue function offload latency */
4481 enq_start_time = rte_rdtsc_precise();
4483 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
4484 &ops_enq[enq], burst_sz - enq);
4485 } while (unlikely(burst_sz != enq));
4487 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time;
4488 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
4489 TEST_ASSERT_SUCCESS(ret,
4490 "Failed to get stats for queue (%u) of device (%u)",
4493 enq_sw_last_time -= stats.acc_offload_cycles;
4494 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
4496 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
4498 time_st->enq_sw_total_time += enq_sw_last_time;
4500 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
4501 stats.acc_offload_cycles);
4502 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
4503 stats.acc_offload_cycles);
4504 time_st->enq_acc_total_time += stats.acc_offload_cycles;
4506 /* give time for device to process ops */
4509 /* Start time meas for dequeue function offload latency */
4510 deq_start_time = rte_rdtsc_precise();
4511 /* Dequeue one operation */
4513 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4515 } while (unlikely(deq != 1));
4517 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4518 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
4520 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
4522 time_st->deq_total_time += deq_last_time;
4524 while (burst_sz != deq)
4525 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
4526 &ops_deq[deq], burst_sz - deq);
4528 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
4537 offload_cost_test(struct active_device *ad,
4538 struct test_op_params *op_params)
4540 #ifndef RTE_BBDEV_OFFLOAD_COST
4542 RTE_SET_USED(op_params);
4543 printf("Offload latency test is disabled.\n");
4544 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4545 return TEST_SKIPPED;
4548 uint16_t burst_sz = op_params->burst_sz;
4549 const uint16_t num_to_process = op_params->num_to_process;
4550 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4551 const uint16_t queue_id = ad->queue_ids[0];
4552 struct test_buffers *bufs = NULL;
4553 struct rte_bbdev_info info;
4554 const char *op_type_str;
4555 struct test_time_stats time_st;
4557 memset(&time_st, 0, sizeof(struct test_time_stats));
4558 time_st.enq_sw_min_time = UINT64_MAX;
4559 time_st.enq_acc_min_time = UINT64_MAX;
4560 time_st.deq_min_time = UINT64_MAX;
4562 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4563 "BURST_SIZE should be <= %u", MAX_BURST);
4565 rte_bbdev_info_get(ad->dev_id, &info);
4566 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
4568 op_type_str = rte_bbdev_op_type_str(op_type);
4569 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4571 printf("+ ------------------------------------------------------- +\n");
4572 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4573 info.dev_name, burst_sz, num_to_process, op_type_str);
4575 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
4576 iter = offload_latency_test_dec(op_params->mp, bufs,
4577 op_params->ref_dec_op, ad->dev_id, queue_id,
4578 num_to_process, burst_sz, &time_st);
4579 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
4580 iter = offload_latency_test_enc(op_params->mp, bufs,
4581 op_params->ref_enc_op, ad->dev_id, queue_id,
4582 num_to_process, burst_sz, &time_st);
4583 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4584 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
4585 op_params->ref_enc_op, ad->dev_id, queue_id,
4586 num_to_process, burst_sz, &time_st);
4587 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4588 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
4589 op_params->ref_dec_op, ad->dev_id, queue_id,
4590 num_to_process, burst_sz, &time_st);
4592 iter = offload_latency_test_enc(op_params->mp, bufs,
4593 op_params->ref_enc_op, ad->dev_id, queue_id,
4594 num_to_process, burst_sz, &time_st);
4599 printf("Enqueue driver offload cost latency:\n"
4600 "\tavg: %lg cycles, %lg us\n"
4601 "\tmin: %lg cycles, %lg us\n"
4602 "\tmax: %lg cycles, %lg us\n"
4603 "Enqueue accelerator offload cost latency:\n"
4604 "\tavg: %lg cycles, %lg us\n"
4605 "\tmin: %lg cycles, %lg us\n"
4606 "\tmax: %lg cycles, %lg us\n",
4607 (double)time_st.enq_sw_total_time / (double)iter,
4608 (double)(time_st.enq_sw_total_time * 1000000) /
4609 (double)iter / (double)rte_get_tsc_hz(),
4610 (double)time_st.enq_sw_min_time,
4611 (double)(time_st.enq_sw_min_time * 1000000) /
4612 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
4613 (double)(time_st.enq_sw_max_time * 1000000) /
4614 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
4616 (double)(time_st.enq_acc_total_time * 1000000) /
4617 (double)iter / (double)rte_get_tsc_hz(),
4618 (double)time_st.enq_acc_min_time,
4619 (double)(time_st.enq_acc_min_time * 1000000) /
4620 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
4621 (double)(time_st.enq_acc_max_time * 1000000) /
4624 printf("Dequeue offload cost latency - one op:\n"
4625 "\tavg: %lg cycles, %lg us\n"
4626 "\tmin: %lg cycles, %lg us\n"
4627 "\tmax: %lg cycles, %lg us\n",
4628 (double)time_st.deq_total_time / (double)iter,
4629 (double)(time_st.deq_total_time * 1000000) /
4630 (double)iter / (double)rte_get_tsc_hz(),
4631 (double)time_st.deq_min_time,
4632 (double)(time_st.deq_min_time * 1000000) /
4633 rte_get_tsc_hz(), (double)time_st.deq_max_time,
4634 (double)(time_st.deq_max_time * 1000000) /
4637 return TEST_SUCCESS;
4641 #ifdef RTE_BBDEV_OFFLOAD_COST
4643 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
4644 const uint16_t num_to_process, uint16_t burst_sz,
4645 uint64_t *deq_total_time, uint64_t *deq_min_time,
4646 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4649 struct rte_bbdev_dec_op *ops[MAX_BURST];
4650 uint64_t deq_start_time, deq_last_time;
4652 /* Test deq offload latency from an empty queue */
4654 for (i = 0, deq_total = 0; deq_total < num_to_process;
4655 ++i, deq_total += burst_sz) {
4656 deq_start_time = rte_rdtsc_precise();
4658 if (unlikely(num_to_process - deq_total < burst_sz))
4659 burst_sz = num_to_process - deq_total;
4660 if (op_type == RTE_BBDEV_OP_LDPC_DEC)
4661 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops,
4664 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops,
4667 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4668 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4669 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4670 *deq_total_time += deq_last_time;
4677 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
4678 const uint16_t num_to_process, uint16_t burst_sz,
4679 uint64_t *deq_total_time, uint64_t *deq_min_time,
4680 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type)
4683 struct rte_bbdev_enc_op *ops[MAX_BURST];
4684 uint64_t deq_start_time, deq_last_time;
4686 /* Test deq offload latency from an empty queue */
4687 for (i = 0, deq_total = 0; deq_total < num_to_process;
4688 ++i, deq_total += burst_sz) {
4689 deq_start_time = rte_rdtsc_precise();
4691 if (unlikely(num_to_process - deq_total < burst_sz))
4692 burst_sz = num_to_process - deq_total;
4693 if (op_type == RTE_BBDEV_OP_LDPC_ENC)
4694 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops,
4697 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops,
4700 deq_last_time = rte_rdtsc_precise() - deq_start_time;
4701 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
4702 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
4703 *deq_total_time += deq_last_time;
4712 offload_latency_empty_q_test(struct active_device *ad,
4713 struct test_op_params *op_params)
4715 #ifndef RTE_BBDEV_OFFLOAD_COST
4717 RTE_SET_USED(op_params);
4718 printf("Offload latency empty dequeue test is disabled.\n");
4719 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
4720 return TEST_SKIPPED;
4723 uint64_t deq_total_time, deq_min_time, deq_max_time;
4724 uint16_t burst_sz = op_params->burst_sz;
4725 const uint16_t num_to_process = op_params->num_to_process;
4726 const enum rte_bbdev_op_type op_type = test_vector.op_type;
4727 const uint16_t queue_id = ad->queue_ids[0];
4728 struct rte_bbdev_info info;
4729 const char *op_type_str;
4731 deq_total_time = deq_max_time = 0;
4732 deq_min_time = UINT64_MAX;
4734 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
4735 "BURST_SIZE should be <= %u", MAX_BURST);
4737 rte_bbdev_info_get(ad->dev_id, &info);
4739 op_type_str = rte_bbdev_op_type_str(op_type);
4740 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
4742 printf("+ ------------------------------------------------------- +\n");
4743 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
4744 info.dev_name, burst_sz, num_to_process, op_type_str);
4746 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
4747 op_type == RTE_BBDEV_OP_LDPC_DEC)
4748 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
4749 num_to_process, burst_sz, &deq_total_time,
4750 &deq_min_time, &deq_max_time, op_type);
4752 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
4753 num_to_process, burst_sz, &deq_total_time,
4754 &deq_min_time, &deq_max_time, op_type);
4759 printf("Empty dequeue offload:\n"
4760 "\tavg: %lg cycles, %lg us\n"
4761 "\tmin: %lg cycles, %lg us\n"
4762 "\tmax: %lg cycles, %lg us\n",
4763 (double)deq_total_time / (double)iter,
4764 (double)(deq_total_time * 1000000) / (double)iter /
4765 (double)rte_get_tsc_hz(), (double)deq_min_time,
4766 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
4767 (double)deq_max_time, (double)(deq_max_time * 1000000) /
4770 return TEST_SUCCESS;
4777 return run_test_case(bler_test);
4783 return run_test_case(throughput_test);
4787 offload_cost_tc(void)
4789 return run_test_case(offload_cost_test);
4793 offload_latency_empty_q_tc(void)
4795 return run_test_case(offload_latency_empty_q_test);
4801 return run_test_case(latency_test);
4807 return run_test_case(throughput_test);
4810 static struct unit_test_suite bbdev_bler_testsuite = {
4811 .suite_name = "BBdev BLER Tests",
4812 .setup = testsuite_setup,
4813 .teardown = testsuite_teardown,
4814 .unit_test_cases = {
4815 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc),
4816 TEST_CASES_END() /**< NULL terminate unit test array */
4820 static struct unit_test_suite bbdev_throughput_testsuite = {
4821 .suite_name = "BBdev Throughput Tests",
4822 .setup = testsuite_setup,
4823 .teardown = testsuite_teardown,
4824 .unit_test_cases = {
4825 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
4826 TEST_CASES_END() /**< NULL terminate unit test array */
4830 static struct unit_test_suite bbdev_validation_testsuite = {
4831 .suite_name = "BBdev Validation Tests",
4832 .setup = testsuite_setup,
4833 .teardown = testsuite_teardown,
4834 .unit_test_cases = {
4835 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4836 TEST_CASES_END() /**< NULL terminate unit test array */
4840 static struct unit_test_suite bbdev_latency_testsuite = {
4841 .suite_name = "BBdev Latency Tests",
4842 .setup = testsuite_setup,
4843 .teardown = testsuite_teardown,
4844 .unit_test_cases = {
4845 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
4846 TEST_CASES_END() /**< NULL terminate unit test array */
4850 static struct unit_test_suite bbdev_offload_cost_testsuite = {
4851 .suite_name = "BBdev Offload Cost Tests",
4852 .setup = testsuite_setup,
4853 .teardown = testsuite_teardown,
4854 .unit_test_cases = {
4855 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
4856 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
4857 TEST_CASES_END() /**< NULL terminate unit test array */
4861 static struct unit_test_suite bbdev_interrupt_testsuite = {
4862 .suite_name = "BBdev Interrupt Tests",
4863 .setup = interrupt_testsuite_setup,
4864 .teardown = testsuite_teardown,
4865 .unit_test_cases = {
4866 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
4867 TEST_CASES_END() /**< NULL terminate unit test array */
4871 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite);
4872 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
4873 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
4874 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
4875 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
4876 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);