1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
19 #include <rte_interrupts.h>
21 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
22 #include <fpga_lte_fec.h>
26 #include "test_bbdev_vector.h"
28 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
30 #define MAX_QUEUES RTE_MAX_LCORE
31 #define TEST_REPETITIONS 1000
33 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
34 #define FPGA_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
35 #define FPGA_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
36 #define VF_UL_QUEUE_VALUE 4
37 #define VF_DL_QUEUE_VALUE 4
38 #define UL_BANDWIDTH 3
39 #define DL_BANDWIDTH 3
40 #define UL_LOAD_BALANCE 128
41 #define DL_LOAD_BALANCE 128
42 #define FLR_TIMEOUT 610
45 #define OPS_CACHE_SIZE 256U
46 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
51 #define INVALID_QUEUE_ID -1
53 static struct test_bbdev_vector test_vector;
55 /* Switch between PMD and Interrupt for throughput TC */
56 static bool intr_enabled;
58 /* Represents tested active devices */
59 static struct active_device {
60 const char *driver_name;
62 uint16_t supported_ops;
63 uint16_t queue_ids[MAX_QUEUES];
65 struct rte_mempool *ops_mempool;
66 struct rte_mempool *in_mbuf_pool;
67 struct rte_mempool *hard_out_mbuf_pool;
68 struct rte_mempool *soft_out_mbuf_pool;
69 struct rte_mempool *harq_in_mbuf_pool;
70 struct rte_mempool *harq_out_mbuf_pool;
71 } active_devs[RTE_BBDEV_MAX_DEVS];
73 static uint8_t nb_active_devs;
75 /* Data buffers used by BBDEV ops */
77 struct rte_bbdev_op_data *inputs;
78 struct rte_bbdev_op_data *hard_outputs;
79 struct rte_bbdev_op_data *soft_outputs;
80 struct rte_bbdev_op_data *harq_inputs;
81 struct rte_bbdev_op_data *harq_outputs;
84 /* Operation parameters specific for given test case */
85 struct test_op_params {
86 struct rte_mempool *mp;
87 struct rte_bbdev_dec_op *ref_dec_op;
88 struct rte_bbdev_enc_op *ref_enc_op;
90 uint16_t num_to_process;
94 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
97 /* Contains per lcore params */
98 struct thread_params {
106 rte_atomic16_t nb_dequeued;
107 rte_atomic16_t processing_status;
108 rte_atomic16_t burst_sz;
109 struct test_op_params *op_params;
110 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
111 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
114 #ifdef RTE_BBDEV_OFFLOAD_COST
115 /* Stores time statistics */
116 struct test_time_stats {
117 /* Stores software enqueue total working time */
118 uint64_t enq_sw_total_time;
119 /* Stores minimum value of software enqueue working time */
120 uint64_t enq_sw_min_time;
121 /* Stores maximum value of software enqueue working time */
122 uint64_t enq_sw_max_time;
123 /* Stores turbo enqueue total working time */
124 uint64_t enq_acc_total_time;
125 /* Stores minimum value of accelerator enqueue working time */
126 uint64_t enq_acc_min_time;
127 /* Stores maximum value of accelerator enqueue working time */
128 uint64_t enq_acc_max_time;
129 /* Stores dequeue total working time */
130 uint64_t deq_total_time;
131 /* Stores minimum value of dequeue working time */
132 uint64_t deq_min_time;
133 /* Stores maximum value of dequeue working time */
134 uint64_t deq_max_time;
138 typedef int (test_case_function)(struct active_device *ad,
139 struct test_op_params *op_params);
142 mbuf_reset(struct rte_mbuf *m)
152 /* Read flag value 0/1 from bitmap */
154 check_bit(uint32_t bitmap, uint32_t bitmask)
156 return bitmap & bitmask;
160 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
162 ad->supported_ops |= (1 << op_type);
166 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
168 return ad->supported_ops & (1 << op_type);
172 flags_match(uint32_t flags_req, uint32_t flags_present)
174 return (flags_req & flags_present) == flags_req;
178 clear_soft_out_cap(uint32_t *op_flags)
180 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
181 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
182 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
186 check_dev_cap(const struct rte_bbdev_info *dev_info)
189 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
190 nb_harq_inputs, nb_harq_outputs;
191 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
193 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
194 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
195 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
196 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
197 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
199 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
200 if (op_cap->type != test_vector.op_type)
203 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
204 const struct rte_bbdev_op_cap_turbo_dec *cap =
205 &op_cap->cap.turbo_dec;
206 /* Ignore lack of soft output capability, just skip
207 * checking if soft output is valid.
209 if ((test_vector.turbo_dec.op_flags &
210 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
211 !(cap->capability_flags &
212 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
214 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
217 &test_vector.turbo_dec.op_flags);
220 if (!flags_match(test_vector.turbo_dec.op_flags,
221 cap->capability_flags))
223 if (nb_inputs > cap->num_buffers_src) {
224 printf("Too many inputs defined: %u, max: %u\n",
225 nb_inputs, cap->num_buffers_src);
228 if (nb_soft_outputs > cap->num_buffers_soft_out &&
229 (test_vector.turbo_dec.op_flags &
230 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
232 "Too many soft outputs defined: %u, max: %u\n",
234 cap->num_buffers_soft_out);
237 if (nb_hard_outputs > cap->num_buffers_hard_out) {
239 "Too many hard outputs defined: %u, max: %u\n",
241 cap->num_buffers_hard_out);
244 if (intr_enabled && !(cap->capability_flags &
245 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
247 "Dequeue interrupts are not supported!\n");
252 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
253 const struct rte_bbdev_op_cap_turbo_enc *cap =
254 &op_cap->cap.turbo_enc;
256 if (!flags_match(test_vector.turbo_enc.op_flags,
257 cap->capability_flags))
259 if (nb_inputs > cap->num_buffers_src) {
260 printf("Too many inputs defined: %u, max: %u\n",
261 nb_inputs, cap->num_buffers_src);
264 if (nb_hard_outputs > cap->num_buffers_dst) {
266 "Too many hard outputs defined: %u, max: %u\n",
267 nb_hard_outputs, cap->num_buffers_dst);
270 if (intr_enabled && !(cap->capability_flags &
271 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
273 "Dequeue interrupts are not supported!\n");
278 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
279 const struct rte_bbdev_op_cap_ldpc_enc *cap =
280 &op_cap->cap.ldpc_enc;
282 if (!flags_match(test_vector.ldpc_enc.op_flags,
283 cap->capability_flags)){
284 printf("Flag Mismatch\n");
287 if (nb_inputs > cap->num_buffers_src) {
288 printf("Too many inputs defined: %u, max: %u\n",
289 nb_inputs, cap->num_buffers_src);
292 if (nb_hard_outputs > cap->num_buffers_dst) {
294 "Too many hard outputs defined: %u, max: %u\n",
295 nb_hard_outputs, cap->num_buffers_dst);
298 if (intr_enabled && !(cap->capability_flags &
299 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
301 "Dequeue interrupts are not supported!\n");
306 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
307 const struct rte_bbdev_op_cap_ldpc_dec *cap =
308 &op_cap->cap.ldpc_dec;
310 if (!flags_match(test_vector.ldpc_dec.op_flags,
311 cap->capability_flags)){
312 printf("Flag Mismatch\n");
315 if (nb_inputs > cap->num_buffers_src) {
316 printf("Too many inputs defined: %u, max: %u\n",
317 nb_inputs, cap->num_buffers_src);
320 if (nb_hard_outputs > cap->num_buffers_hard_out) {
322 "Too many hard outputs defined: %u, max: %u\n",
324 cap->num_buffers_hard_out);
327 if (nb_harq_inputs > cap->num_buffers_hard_out) {
329 "Too many HARQ inputs defined: %u, max: %u\n",
331 cap->num_buffers_hard_out);
334 if (nb_harq_outputs > cap->num_buffers_hard_out) {
336 "Too many HARQ outputs defined: %u, max: %u\n",
338 cap->num_buffers_hard_out);
341 if (intr_enabled && !(cap->capability_flags &
342 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
344 "Dequeue interrupts are not supported!\n");
352 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
353 return TEST_SUCCESS; /* Special case for NULL device */
358 /* calculates optimal mempool size not smaller than the val */
360 optimal_mempool_size(unsigned int val)
362 return rte_align32pow2(val + 1) - 1;
365 /* allocates mbuf mempool for inputs and outputs */
366 static struct rte_mempool *
367 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
368 int socket_id, unsigned int mbuf_pool_size,
369 const char *op_type_str)
372 uint32_t max_seg_sz = 0;
373 char pool_name[RTE_MEMPOOL_NAMESIZE];
375 /* find max input segment size */
376 for (i = 0; i < entries->nb_segments; ++i)
377 if (entries->segments[i].length > max_seg_sz)
378 max_seg_sz = entries->segments[i].length;
380 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
382 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
383 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
384 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
388 create_mempools(struct active_device *ad, int socket_id,
389 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
391 struct rte_mempool *mp;
392 unsigned int ops_pool_size, mbuf_pool_size = 0;
393 char pool_name[RTE_MEMPOOL_NAMESIZE];
394 const char *op_type_str;
395 enum rte_bbdev_op_type op_type = org_op_type;
397 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
398 struct op_data_entries *hard_out =
399 &test_vector.entries[DATA_HARD_OUTPUT];
400 struct op_data_entries *soft_out =
401 &test_vector.entries[DATA_SOFT_OUTPUT];
402 struct op_data_entries *harq_in =
403 &test_vector.entries[DATA_HARQ_INPUT];
404 struct op_data_entries *harq_out =
405 &test_vector.entries[DATA_HARQ_OUTPUT];
407 /* allocate ops mempool */
408 ops_pool_size = optimal_mempool_size(RTE_MAX(
409 /* Ops used plus 1 reference op */
410 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
411 /* Minimal cache size plus 1 reference op */
412 (unsigned int)(1.5 * rte_lcore_count() *
413 OPS_CACHE_SIZE + 1)),
416 if (org_op_type == RTE_BBDEV_OP_NONE)
417 op_type = RTE_BBDEV_OP_TURBO_ENC;
419 op_type_str = rte_bbdev_op_type_str(op_type);
420 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
422 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
424 mp = rte_bbdev_op_pool_create(pool_name, op_type,
425 ops_pool_size, OPS_CACHE_SIZE, socket_id);
426 TEST_ASSERT_NOT_NULL(mp,
427 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
431 ad->ops_mempool = mp;
433 /* Do not create inputs and outputs mbufs for BaseBand Null Device */
434 if (org_op_type == RTE_BBDEV_OP_NONE)
438 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
439 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
440 TEST_ASSERT_NOT_NULL(mp,
441 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
445 ad->in_mbuf_pool = mp;
448 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
449 hard_out->nb_segments);
450 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
452 TEST_ASSERT_NOT_NULL(mp,
453 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
457 ad->hard_out_mbuf_pool = mp;
461 if (soft_out->nb_segments > 0) {
462 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
463 soft_out->nb_segments);
464 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
467 TEST_ASSERT_NOT_NULL(mp,
468 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
472 ad->soft_out_mbuf_pool = mp;
476 if (harq_in->nb_segments > 0) {
477 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
478 harq_in->nb_segments);
479 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
482 TEST_ASSERT_NOT_NULL(mp,
483 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
487 ad->harq_in_mbuf_pool = mp;
491 if (harq_out->nb_segments > 0) {
492 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
493 harq_out->nb_segments);
494 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
497 TEST_ASSERT_NOT_NULL(mp,
498 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
502 ad->harq_out_mbuf_pool = mp;
509 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
510 struct test_bbdev_vector *vector)
513 unsigned int queue_id;
514 struct rte_bbdev_queue_conf qconf;
515 struct active_device *ad = &active_devs[nb_active_devs];
516 unsigned int nb_queues;
517 enum rte_bbdev_op_type op_type = vector->op_type;
519 /* Configure fpga lte fec with PF & VF values
520 * if '-i' flag is set and using fpga device
522 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
523 if ((get_init_device() == true) &&
524 (!strcmp(info->drv.driver_name, FPGA_PF_DRIVER_NAME))) {
525 struct fpga_lte_fec_conf conf;
528 printf("Configure FPGA FEC Driver %s with default values\n",
529 info->drv.driver_name);
531 /* clear default configuration before initialization */
532 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
535 * true if PF is used for data plane
538 conf.pf_mode_en = true;
540 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
541 /* Number of UL queues per VF (fpga supports 8 VFs) */
542 conf.vf_ul_queues_number[i] = VF_UL_QUEUE_VALUE;
543 /* Number of DL queues per VF (fpga supports 8 VFs) */
544 conf.vf_dl_queues_number[i] = VF_DL_QUEUE_VALUE;
547 /* UL bandwidth. Needed for schedule algorithm */
548 conf.ul_bandwidth = UL_BANDWIDTH;
550 conf.dl_bandwidth = DL_BANDWIDTH;
552 /* UL & DL load Balance Factor to 64 */
553 conf.ul_load_balance = UL_LOAD_BALANCE;
554 conf.dl_load_balance = DL_LOAD_BALANCE;
556 /**< FLR timeout value */
557 conf.flr_time_out = FLR_TIMEOUT;
559 /* setup FPGA PF with configuration information */
560 ret = fpga_lte_fec_configure(info->dev_name, &conf);
561 TEST_ASSERT_SUCCESS(ret,
562 "Failed to configure 4G FPGA PF for bbdev %s",
566 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
567 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
570 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
572 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
573 dev_id, nb_queues, info->socket_id, ret);
577 /* configure interrupts if needed */
579 ret = rte_bbdev_intr_enable(dev_id);
581 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
587 /* setup device queues */
588 qconf.socket = info->socket_id;
589 qconf.queue_size = info->drv.default_queue_conf.queue_size;
591 qconf.deferred_start = 0;
592 qconf.op_type = op_type;
594 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
595 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
598 "Allocated all queues (id=%u) at prio%u on dev%u\n",
599 queue_id, qconf.priority, dev_id);
601 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
605 printf("All queues on dev %u allocated: %u\n",
609 ad->queue_ids[queue_id] = queue_id;
611 TEST_ASSERT(queue_id != 0,
612 "ERROR Failed to configure any queues on dev %u",
614 ad->nb_queues = queue_id;
616 set_avail_op(ad, op_type);
622 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
623 struct test_bbdev_vector *vector)
627 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
628 active_devs[nb_active_devs].dev_id = dev_id;
630 ret = add_bbdev_dev(dev_id, info, vector);
631 if (ret == TEST_SUCCESS)
637 populate_active_devices(void)
641 uint8_t nb_devs_added = 0;
642 struct rte_bbdev_info info;
644 RTE_BBDEV_FOREACH(dev_id) {
645 rte_bbdev_info_get(dev_id, &info);
647 if (check_dev_cap(&info)) {
649 "Device %d (%s) does not support specified capabilities\n",
650 dev_id, info.dev_name);
654 ret = add_active_device(dev_id, &info, &test_vector);
656 printf("Adding active bbdev %s skipped\n",
663 return nb_devs_added;
667 read_test_vector(void)
671 memset(&test_vector, 0, sizeof(test_vector));
672 printf("Test vector file = %s\n", get_vector_filename());
673 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
674 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
675 get_vector_filename());
681 testsuite_setup(void)
683 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
685 if (populate_active_devices() == 0) {
686 printf("No suitable devices found!\n");
694 interrupt_testsuite_setup(void)
696 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
698 /* Enable interrupts */
701 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
702 if (populate_active_devices() == 0 ||
703 test_vector.op_type == RTE_BBDEV_OP_NONE) {
704 intr_enabled = false;
705 printf("No suitable devices found!\n");
713 testsuite_teardown(void)
717 /* Unconfigure devices */
718 RTE_BBDEV_FOREACH(dev_id)
719 rte_bbdev_close(dev_id);
721 /* Clear active devices structs. */
722 memset(active_devs, 0, sizeof(active_devs));
731 for (i = 0; i < nb_active_devs; i++) {
732 dev_id = active_devs[i].dev_id;
733 /* reset bbdev stats */
734 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
735 "Failed to reset stats of bbdev %u", dev_id);
736 /* start the device */
737 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
738 "Failed to start bbdev %u", dev_id);
748 struct rte_bbdev_stats stats;
750 for (i = 0; i < nb_active_devs; i++) {
751 dev_id = active_devs[i].dev_id;
752 /* read stats and print */
753 rte_bbdev_stats_get(dev_id, &stats);
754 /* Stop the device */
755 rte_bbdev_stop(dev_id);
760 init_op_data_objs(struct rte_bbdev_op_data *bufs,
761 struct op_data_entries *ref_entries,
762 struct rte_mempool *mbuf_pool, const uint16_t n,
763 enum op_data_type op_type, uint16_t min_alignment)
768 for (i = 0; i < n; ++i) {
770 struct op_data_buf *seg = &ref_entries->segments[0];
771 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
772 TEST_ASSERT_NOT_NULL(m_head,
773 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
774 op_type, n * ref_entries->nb_segments,
777 TEST_ASSERT_SUCCESS(((seg->length + RTE_PKTMBUF_HEADROOM) >
778 (uint32_t)UINT16_MAX),
779 "Given data is bigger than allowed mbuf segment size");
781 bufs[i].data = m_head;
785 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
786 data = rte_pktmbuf_append(m_head, seg->length);
787 TEST_ASSERT_NOT_NULL(data,
788 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
789 seg->length, op_type);
791 TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
792 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
793 data, min_alignment);
794 rte_memcpy(data, seg->addr, seg->length);
795 bufs[i].length += seg->length;
797 for (j = 1; j < ref_entries->nb_segments; ++j) {
798 struct rte_mbuf *m_tail =
799 rte_pktmbuf_alloc(mbuf_pool);
800 TEST_ASSERT_NOT_NULL(m_tail,
801 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
803 n * ref_entries->nb_segments,
807 data = rte_pktmbuf_append(m_tail, seg->length);
808 TEST_ASSERT_NOT_NULL(data,
809 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
810 seg->length, op_type);
812 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
814 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
815 data, min_alignment);
816 rte_memcpy(data, seg->addr, seg->length);
817 bufs[i].length += seg->length;
819 ret = rte_pktmbuf_chain(m_head, m_tail);
820 TEST_ASSERT_SUCCESS(ret,
821 "Couldn't chain mbufs from %d data type mbuf pool",
826 /* allocate chained-mbuf for output buffer */
827 for (j = 1; j < ref_entries->nb_segments; ++j) {
828 struct rte_mbuf *m_tail =
829 rte_pktmbuf_alloc(mbuf_pool);
830 TEST_ASSERT_NOT_NULL(m_tail,
831 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
833 n * ref_entries->nb_segments,
836 ret = rte_pktmbuf_chain(m_head, m_tail);
837 TEST_ASSERT_SUCCESS(ret,
838 "Couldn't chain mbufs from %d data type mbuf pool",
848 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
853 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
854 if (*buffers == NULL) {
855 printf("WARNING: Failed to allocate op_data on socket %d\n",
857 /* try to allocate memory on other detected sockets */
858 for (i = 0; i < socket; i++) {
859 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
860 if (*buffers != NULL)
865 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
869 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
870 const uint16_t n, const int8_t max_llr_modulus)
872 uint16_t i, byte_idx;
874 for (i = 0; i < n; ++i) {
875 struct rte_mbuf *m = input_ops[i].data;
877 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
878 input_ops[i].offset);
879 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
881 llr[byte_idx] = round((double)max_llr_modulus *
882 llr[byte_idx] / INT8_MAX);
890 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
891 const uint16_t n, const int8_t llr_size,
892 const int8_t llr_decimals)
894 if (input_ops == NULL)
897 uint16_t i, byte_idx;
899 int16_t llr_max, llr_min, llr_tmp;
900 llr_max = (1 << (llr_size - 1)) - 1;
902 for (i = 0; i < n; ++i) {
903 struct rte_mbuf *m = input_ops[i].data;
905 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
906 input_ops[i].offset);
907 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
910 llr_tmp = llr[byte_idx];
911 if (llr_decimals == 2)
913 else if (llr_decimals == 0)
915 llr_tmp = RTE_MIN(llr_max,
916 RTE_MAX(llr_min, llr_tmp));
917 llr[byte_idx] = (int8_t) llr_tmp;
928 fill_queue_buffers(struct test_op_params *op_params,
929 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
930 struct rte_mempool *soft_out_mp,
931 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
933 const struct rte_bbdev_op_cap *capabilities,
934 uint16_t min_alignment, const int socket_id)
937 enum op_data_type type;
938 const uint16_t n = op_params->num_to_process;
940 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
948 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
949 &op_params->q_bufs[socket_id][queue_id].inputs,
950 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
951 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
952 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
953 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
956 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
957 struct op_data_entries *ref_entries =
958 &test_vector.entries[type];
959 if (ref_entries->nb_segments == 0)
962 ret = allocate_buffers_on_socket(queue_ops[type],
963 n * sizeof(struct rte_bbdev_op_data),
965 TEST_ASSERT_SUCCESS(ret,
966 "Couldn't allocate memory for rte_bbdev_op_data structs");
968 ret = init_op_data_objs(*queue_ops[type], ref_entries,
969 mbuf_pools[type], n, type, min_alignment);
970 TEST_ASSERT_SUCCESS(ret,
971 "Couldn't init rte_bbdev_op_data structs");
974 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
975 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
976 capabilities->cap.turbo_dec.max_llr_modulus);
978 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
979 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
980 capabilities->cap.ldpc_dec.llr_size,
981 capabilities->cap.ldpc_dec.llr_decimals);
982 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
983 capabilities->cap.ldpc_dec.llr_size,
984 capabilities->cap.ldpc_dec.llr_decimals);
991 free_buffers(struct active_device *ad, struct test_op_params *op_params)
995 rte_mempool_free(ad->ops_mempool);
996 rte_mempool_free(ad->in_mbuf_pool);
997 rte_mempool_free(ad->hard_out_mbuf_pool);
998 rte_mempool_free(ad->soft_out_mbuf_pool);
999 rte_mempool_free(ad->harq_in_mbuf_pool);
1000 rte_mempool_free(ad->harq_out_mbuf_pool);
1002 for (i = 0; i < rte_lcore_count(); ++i) {
1003 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1004 rte_free(op_params->q_bufs[j][i].inputs);
1005 rte_free(op_params->q_bufs[j][i].hard_outputs);
1006 rte_free(op_params->q_bufs[j][i].soft_outputs);
1007 rte_free(op_params->q_bufs[j][i].harq_inputs);
1008 rte_free(op_params->q_bufs[j][i].harq_outputs);
1014 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1015 unsigned int start_idx,
1016 struct rte_bbdev_op_data *inputs,
1017 struct rte_bbdev_op_data *hard_outputs,
1018 struct rte_bbdev_op_data *soft_outputs,
1019 struct rte_bbdev_dec_op *ref_op)
1022 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1024 for (i = 0; i < n; ++i) {
1025 if (turbo_dec->code_block_mode == 0) {
1026 ops[i]->turbo_dec.tb_params.ea =
1027 turbo_dec->tb_params.ea;
1028 ops[i]->turbo_dec.tb_params.eb =
1029 turbo_dec->tb_params.eb;
1030 ops[i]->turbo_dec.tb_params.k_pos =
1031 turbo_dec->tb_params.k_pos;
1032 ops[i]->turbo_dec.tb_params.k_neg =
1033 turbo_dec->tb_params.k_neg;
1034 ops[i]->turbo_dec.tb_params.c =
1035 turbo_dec->tb_params.c;
1036 ops[i]->turbo_dec.tb_params.c_neg =
1037 turbo_dec->tb_params.c_neg;
1038 ops[i]->turbo_dec.tb_params.cab =
1039 turbo_dec->tb_params.cab;
1040 ops[i]->turbo_dec.tb_params.r =
1041 turbo_dec->tb_params.r;
1043 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1044 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1047 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1048 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1049 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1050 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1051 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1052 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1053 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1055 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1056 ops[i]->turbo_dec.input = inputs[start_idx + i];
1057 if (soft_outputs != NULL)
1058 ops[i]->turbo_dec.soft_output =
1059 soft_outputs[start_idx + i];
1064 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1065 unsigned int start_idx,
1066 struct rte_bbdev_op_data *inputs,
1067 struct rte_bbdev_op_data *outputs,
1068 struct rte_bbdev_enc_op *ref_op)
1071 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1072 for (i = 0; i < n; ++i) {
1073 if (turbo_enc->code_block_mode == 0) {
1074 ops[i]->turbo_enc.tb_params.ea =
1075 turbo_enc->tb_params.ea;
1076 ops[i]->turbo_enc.tb_params.eb =
1077 turbo_enc->tb_params.eb;
1078 ops[i]->turbo_enc.tb_params.k_pos =
1079 turbo_enc->tb_params.k_pos;
1080 ops[i]->turbo_enc.tb_params.k_neg =
1081 turbo_enc->tb_params.k_neg;
1082 ops[i]->turbo_enc.tb_params.c =
1083 turbo_enc->tb_params.c;
1084 ops[i]->turbo_enc.tb_params.c_neg =
1085 turbo_enc->tb_params.c_neg;
1086 ops[i]->turbo_enc.tb_params.cab =
1087 turbo_enc->tb_params.cab;
1088 ops[i]->turbo_enc.tb_params.ncb_pos =
1089 turbo_enc->tb_params.ncb_pos;
1090 ops[i]->turbo_enc.tb_params.ncb_neg =
1091 turbo_enc->tb_params.ncb_neg;
1092 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1094 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1095 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1096 ops[i]->turbo_enc.cb_params.ncb =
1097 turbo_enc->cb_params.ncb;
1099 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1100 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1101 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1103 ops[i]->turbo_enc.output = outputs[start_idx + i];
1104 ops[i]->turbo_enc.input = inputs[start_idx + i];
1109 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1110 unsigned int start_idx,
1111 struct rte_bbdev_op_data *inputs,
1112 struct rte_bbdev_op_data *hard_outputs,
1113 struct rte_bbdev_op_data *soft_outputs,
1114 struct rte_bbdev_op_data *harq_inputs,
1115 struct rte_bbdev_op_data *harq_outputs,
1116 struct rte_bbdev_dec_op *ref_op)
1119 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1121 for (i = 0; i < n; ++i) {
1122 if (ldpc_dec->code_block_mode == 0) {
1123 ops[i]->ldpc_dec.tb_params.ea =
1124 ldpc_dec->tb_params.ea;
1125 ops[i]->ldpc_dec.tb_params.eb =
1126 ldpc_dec->tb_params.eb;
1127 ops[i]->ldpc_dec.tb_params.c =
1128 ldpc_dec->tb_params.c;
1129 ops[i]->ldpc_dec.tb_params.cab =
1130 ldpc_dec->tb_params.cab;
1131 ops[i]->ldpc_dec.tb_params.r =
1132 ldpc_dec->tb_params.r;
1134 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1137 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1138 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1139 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1140 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1141 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1142 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1143 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1144 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1145 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1147 ops[i]->ldpc_dec.hard_output = hard_outputs[start_idx + i];
1148 ops[i]->ldpc_dec.input = inputs[start_idx + i];
1149 if (soft_outputs != NULL)
1150 ops[i]->ldpc_dec.soft_output =
1151 soft_outputs[start_idx + i];
1152 if (harq_inputs != NULL)
1153 ops[i]->ldpc_dec.harq_combined_input =
1154 harq_inputs[start_idx + i];
1155 if (harq_outputs != NULL)
1156 ops[i]->ldpc_dec.harq_combined_output =
1157 harq_outputs[start_idx + i];
1163 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1164 unsigned int start_idx,
1165 struct rte_bbdev_op_data *inputs,
1166 struct rte_bbdev_op_data *outputs,
1167 struct rte_bbdev_enc_op *ref_op)
1170 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1171 for (i = 0; i < n; ++i) {
1172 if (ldpc_enc->code_block_mode == 0) {
1173 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1174 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1175 ops[i]->ldpc_enc.tb_params.cab =
1176 ldpc_enc->tb_params.cab;
1177 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1178 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1180 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1182 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1183 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1184 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1185 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1186 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1187 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1188 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1189 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1190 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1191 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1196 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1197 unsigned int order_idx, const int expected_status)
1199 TEST_ASSERT(op->status == expected_status,
1200 "op_status (%d) != expected_status (%d)",
1201 op->status, expected_status);
1203 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1204 "Ordering error, expected %p, got %p",
1205 (void *)(uintptr_t)order_idx, op->opaque_data);
1207 return TEST_SUCCESS;
1211 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1212 unsigned int order_idx, const int expected_status)
1214 TEST_ASSERT(op->status == expected_status,
1215 "op_status (%d) != expected_status (%d)",
1216 op->status, expected_status);
1218 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1219 "Ordering error, expected %p, got %p",
1220 (void *)(uintptr_t)order_idx, op->opaque_data);
1222 return TEST_SUCCESS;
1226 validate_op_chain(struct rte_bbdev_op_data *op,
1227 struct op_data_entries *orig_op)
1230 struct rte_mbuf *m = op->data;
1231 uint8_t nb_dst_segments = orig_op->nb_segments;
1232 uint32_t total_data_size = 0;
1234 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1235 "Number of segments differ in original (%u) and filled (%u) op",
1236 nb_dst_segments, m->nb_segs);
1238 /* Validate each mbuf segment length */
1239 for (i = 0; i < nb_dst_segments; ++i) {
1240 /* Apply offset to the first mbuf segment */
1241 uint16_t offset = (i == 0) ? op->offset : 0;
1242 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1243 total_data_size += orig_op->segments[i].length;
1245 TEST_ASSERT(orig_op->segments[i].length == data_len,
1246 "Length of segment differ in original (%u) and filled (%u) op",
1247 orig_op->segments[i].length, data_len);
1248 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1249 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1251 "Output buffers (CB=%u) are not equal", i);
1255 /* Validate total mbuf pkt length */
1256 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1257 TEST_ASSERT(total_data_size == pkt_len,
1258 "Length of data differ in original (%u) and filled (%u) op",
1259 total_data_size, pkt_len);
1261 return TEST_SUCCESS;
1265 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1266 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1270 struct op_data_entries *hard_data_orig =
1271 &test_vector.entries[DATA_HARD_OUTPUT];
1272 struct op_data_entries *soft_data_orig =
1273 &test_vector.entries[DATA_SOFT_OUTPUT];
1274 struct rte_bbdev_op_turbo_dec *ops_td;
1275 struct rte_bbdev_op_data *hard_output;
1276 struct rte_bbdev_op_data *soft_output;
1277 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
1279 for (i = 0; i < n; ++i) {
1280 ops_td = &ops[i]->turbo_dec;
1281 hard_output = &ops_td->hard_output;
1282 soft_output = &ops_td->soft_output;
1284 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1285 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1286 "Returned iter_count (%d) > expected iter_count (%d)",
1287 ops_td->iter_count, ref_td->iter_count);
1288 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1289 TEST_ASSERT_SUCCESS(ret,
1290 "Checking status and ordering for decoder failed");
1292 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1294 "Hard output buffers (CB=%u) are not equal",
1297 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
1298 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1300 "Soft output buffers (CB=%u) are not equal",
1304 return TEST_SUCCESS;
1309 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1310 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1314 struct op_data_entries *hard_data_orig =
1315 &test_vector.entries[DATA_HARD_OUTPUT];
1316 struct op_data_entries *soft_data_orig =
1317 &test_vector.entries[DATA_SOFT_OUTPUT];
1318 struct op_data_entries *harq_data_orig =
1319 &test_vector.entries[DATA_HARQ_OUTPUT];
1320 struct rte_bbdev_op_ldpc_dec *ops_td;
1321 struct rte_bbdev_op_data *hard_output;
1322 struct rte_bbdev_op_data *harq_output;
1323 struct rte_bbdev_op_data *soft_output;
1324 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
1326 for (i = 0; i < n; ++i) {
1327 ops_td = &ops[i]->ldpc_dec;
1328 hard_output = &ops_td->hard_output;
1329 harq_output = &ops_td->harq_combined_output;
1330 soft_output = &ops_td->soft_output;
1332 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1333 TEST_ASSERT_SUCCESS(ret,
1334 "Checking status and ordering for decoder failed");
1335 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1336 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1337 "Returned iter_count (%d) > expected iter_count (%d)",
1338 ops_td->iter_count, ref_td->iter_count);
1339 /* We can ignore data when the decoding failed to converge */
1340 if ((ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) == 0)
1341 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1343 "Hard output buffers (CB=%u) are not equal",
1346 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
1347 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1349 "Soft output buffers (CB=%u) are not equal",
1351 if (ref_op->ldpc_dec.op_flags &
1352 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
1353 ldpc_input_llr_scaling(harq_output, 1, 8, 0);
1354 TEST_ASSERT_SUCCESS(validate_op_chain(harq_output,
1356 "HARQ output buffers (CB=%u) are not equal",
1361 return TEST_SUCCESS;
1366 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
1367 struct rte_bbdev_enc_op *ref_op)
1371 struct op_data_entries *hard_data_orig =
1372 &test_vector.entries[DATA_HARD_OUTPUT];
1374 for (i = 0; i < n; ++i) {
1375 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
1376 TEST_ASSERT_SUCCESS(ret,
1377 "Checking status and ordering for encoder failed");
1378 TEST_ASSERT_SUCCESS(validate_op_chain(
1379 &ops[i]->turbo_enc.output,
1381 "Output buffers (CB=%u) are not equal",
1385 return TEST_SUCCESS;
1389 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
1390 struct rte_bbdev_enc_op *ref_op)
1394 struct op_data_entries *hard_data_orig =
1395 &test_vector.entries[DATA_HARD_OUTPUT];
1397 for (i = 0; i < n; ++i) {
1398 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
1399 TEST_ASSERT_SUCCESS(ret,
1400 "Checking status and ordering for encoder failed");
1401 TEST_ASSERT_SUCCESS(validate_op_chain(
1402 &ops[i]->ldpc_enc.output,
1404 "Output buffers (CB=%u) are not equal",
1408 return TEST_SUCCESS;
1412 create_reference_dec_op(struct rte_bbdev_dec_op *op)
1415 struct op_data_entries *entry;
1417 op->turbo_dec = test_vector.turbo_dec;
1418 entry = &test_vector.entries[DATA_INPUT];
1419 for (i = 0; i < entry->nb_segments; ++i)
1420 op->turbo_dec.input.length +=
1421 entry->segments[i].length;
1425 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
1428 struct op_data_entries *entry;
1430 op->ldpc_dec = test_vector.ldpc_dec;
1431 entry = &test_vector.entries[DATA_INPUT];
1432 for (i = 0; i < entry->nb_segments; ++i)
1433 op->ldpc_dec.input.length +=
1434 entry->segments[i].length;
1435 if (test_vector.ldpc_dec.op_flags &
1436 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
1437 entry = &test_vector.entries[DATA_HARQ_INPUT];
1438 for (i = 0; i < entry->nb_segments; ++i)
1439 op->ldpc_dec.harq_combined_input.length +=
1440 entry->segments[i].length;
1446 create_reference_enc_op(struct rte_bbdev_enc_op *op)
1449 struct op_data_entries *entry;
1451 op->turbo_enc = test_vector.turbo_enc;
1452 entry = &test_vector.entries[DATA_INPUT];
1453 for (i = 0; i < entry->nb_segments; ++i)
1454 op->turbo_enc.input.length +=
1455 entry->segments[i].length;
1459 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
1462 struct op_data_entries *entry;
1464 op->ldpc_enc = test_vector.ldpc_enc;
1465 entry = &test_vector.entries[DATA_INPUT];
1466 for (i = 0; i < entry->nb_segments; ++i)
1467 op->ldpc_enc.input.length +=
1468 entry->segments[i].length;
1472 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
1475 uint32_t c, r, tb_size = 0;
1477 if (op->turbo_dec.code_block_mode) {
1478 tb_size = op->turbo_dec.tb_params.k_neg;
1480 c = op->turbo_dec.tb_params.c;
1481 r = op->turbo_dec.tb_params.r;
1482 for (i = 0; i < c-r; i++)
1483 tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
1484 op->turbo_dec.tb_params.k_neg :
1485 op->turbo_dec.tb_params.k_pos;
1491 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
1494 uint32_t c, r, tb_size = 0;
1495 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
1497 if (op->ldpc_dec.code_block_mode) {
1498 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
1500 c = op->ldpc_dec.tb_params.c;
1501 r = op->ldpc_dec.tb_params.r;
1502 for (i = 0; i < c-r; i++)
1503 tb_size += sys_cols * op->ldpc_dec.z_c
1504 - op->ldpc_dec.n_filler;
1510 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
1513 uint32_t c, r, tb_size = 0;
1515 if (op->turbo_enc.code_block_mode) {
1516 tb_size = op->turbo_enc.tb_params.k_neg;
1518 c = op->turbo_enc.tb_params.c;
1519 r = op->turbo_enc.tb_params.r;
1520 for (i = 0; i < c-r; i++)
1521 tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
1522 op->turbo_enc.tb_params.k_neg :
1523 op->turbo_enc.tb_params.k_pos;
1529 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
1532 uint32_t c, r, tb_size = 0;
1533 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
1535 if (op->turbo_enc.code_block_mode) {
1536 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
1538 c = op->turbo_enc.tb_params.c;
1539 r = op->turbo_enc.tb_params.r;
1540 for (i = 0; i < c-r; i++)
1541 tb_size += sys_cols * op->ldpc_enc.z_c
1542 - op->ldpc_enc.n_filler;
1549 init_test_op_params(struct test_op_params *op_params,
1550 enum rte_bbdev_op_type op_type, const int expected_status,
1551 const int vector_mask, struct rte_mempool *ops_mp,
1552 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
1555 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
1556 op_type == RTE_BBDEV_OP_LDPC_DEC)
1557 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
1558 &op_params->ref_dec_op, 1);
1560 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
1561 &op_params->ref_enc_op, 1);
1563 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
1565 op_params->mp = ops_mp;
1566 op_params->burst_sz = burst_sz;
1567 op_params->num_to_process = num_to_process;
1568 op_params->num_lcores = num_lcores;
1569 op_params->vector_mask = vector_mask;
1570 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
1571 op_type == RTE_BBDEV_OP_LDPC_DEC)
1572 op_params->ref_dec_op->status = expected_status;
1573 else if (op_type == RTE_BBDEV_OP_TURBO_ENC
1574 || op_type == RTE_BBDEV_OP_LDPC_ENC)
1575 op_params->ref_enc_op->status = expected_status;
1580 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
1581 struct test_op_params *op_params)
1583 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
1585 struct active_device *ad;
1586 unsigned int burst_sz = get_burst_sz();
1587 enum rte_bbdev_op_type op_type = test_vector.op_type;
1588 const struct rte_bbdev_op_cap *capabilities = NULL;
1590 ad = &active_devs[dev_id];
1592 /* Check if device supports op_type */
1593 if (!is_avail_op(ad, test_vector.op_type))
1594 return TEST_SUCCESS;
1596 struct rte_bbdev_info info;
1597 rte_bbdev_info_get(ad->dev_id, &info);
1598 socket_id = GET_SOCKET(info.socket_id);
1600 f_ret = create_mempools(ad, socket_id, op_type,
1602 if (f_ret != TEST_SUCCESS) {
1603 printf("Couldn't create mempools");
1606 if (op_type == RTE_BBDEV_OP_NONE)
1607 op_type = RTE_BBDEV_OP_TURBO_ENC;
1609 f_ret = init_test_op_params(op_params, test_vector.op_type,
1610 test_vector.expected_status,
1616 if (f_ret != TEST_SUCCESS) {
1617 printf("Couldn't init test op params");
1622 /* Find capabilities */
1623 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
1624 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
1625 if (cap->type == test_vector.op_type) {
1631 TEST_ASSERT_NOT_NULL(capabilities,
1632 "Couldn't find capabilities");
1634 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1635 create_reference_dec_op(op_params->ref_dec_op);
1636 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1637 create_reference_enc_op(op_params->ref_enc_op);
1638 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
1639 create_reference_ldpc_enc_op(op_params->ref_enc_op);
1640 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
1641 create_reference_ldpc_dec_op(op_params->ref_dec_op);
1643 for (i = 0; i < ad->nb_queues; ++i) {
1644 f_ret = fill_queue_buffers(op_params,
1646 ad->hard_out_mbuf_pool,
1647 ad->soft_out_mbuf_pool,
1648 ad->harq_in_mbuf_pool,
1649 ad->harq_out_mbuf_pool,
1652 info.drv.min_alignment,
1654 if (f_ret != TEST_SUCCESS) {
1655 printf("Couldn't init queue buffers");
1660 /* Run test case function */
1661 t_ret = test_case_func(ad, op_params);
1663 /* Free active device resources and return */
1664 free_buffers(ad, op_params);
1668 free_buffers(ad, op_params);
1672 /* Run given test function per active device per supported op type
1676 run_test_case(test_case_function *test_case_func)
1681 /* Alloc op_params */
1682 struct test_op_params *op_params = rte_zmalloc(NULL,
1683 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1684 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1685 RTE_ALIGN(sizeof(struct test_op_params),
1686 RTE_CACHE_LINE_SIZE));
1688 /* For each device run test case function */
1689 for (dev = 0; dev < nb_active_devs; ++dev)
1690 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1692 rte_free(op_params);
1698 dequeue_event_callback(uint16_t dev_id,
1699 enum rte_bbdev_event_type event, void *cb_arg,
1704 uint64_t total_time;
1705 uint16_t deq, burst_sz, num_ops;
1706 uint16_t queue_id = *(uint16_t *) ret_param;
1707 struct rte_bbdev_info info;
1709 struct thread_params *tp = cb_arg;
1711 /* Find matching thread params using queue_id */
1712 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1713 if (tp->queue_id == queue_id)
1716 if (i == MAX_QUEUES) {
1717 printf("%s: Queue_id from interrupt details was not found!\n",
1722 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1723 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1725 "Dequeue interrupt handler called for incorrect event!\n");
1729 burst_sz = rte_atomic16_read(&tp->burst_sz);
1730 num_ops = tp->op_params->num_to_process;
1732 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
1733 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
1734 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1736 rte_atomic16_read(&tp->nb_dequeued)],
1739 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1741 rte_atomic16_read(&tp->nb_dequeued)],
1744 if (deq < burst_sz) {
1746 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1748 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1752 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
1753 rte_atomic16_add(&tp->nb_dequeued, deq);
1757 total_time = rte_rdtsc_precise() - tp->start_time;
1759 rte_bbdev_info_get(dev_id, &info);
1763 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1764 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1765 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
1766 tp->op_params->vector_mask);
1767 /* get the max of iter_count for all dequeued ops */
1768 for (i = 0; i < num_ops; ++i)
1769 tp->iter_count = RTE_MAX(
1770 tp->dec_ops[i]->turbo_dec.iter_count,
1772 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
1773 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
1774 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1775 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
1776 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
1777 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
1778 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1779 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
1780 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
1781 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1782 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1783 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
1784 tp->op_params->vector_mask);
1785 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
1789 printf("Buffers validation failed\n");
1790 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1793 switch (test_vector.op_type) {
1794 case RTE_BBDEV_OP_TURBO_DEC:
1795 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
1797 case RTE_BBDEV_OP_TURBO_ENC:
1798 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
1800 case RTE_BBDEV_OP_LDPC_DEC:
1801 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
1803 case RTE_BBDEV_OP_LDPC_ENC:
1804 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
1806 case RTE_BBDEV_OP_NONE:
1810 printf("Unknown op type: %d\n", test_vector.op_type);
1811 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1815 tp->ops_per_sec += ((double)num_ops) /
1816 ((double)total_time / (double)rte_get_tsc_hz());
1817 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
1818 ((double)total_time / (double)rte_get_tsc_hz());
1820 rte_atomic16_add(&tp->nb_dequeued, deq);
1824 throughput_intr_lcore_dec(void *arg)
1826 struct thread_params *tp = arg;
1827 unsigned int enqueued;
1828 const uint16_t queue_id = tp->queue_id;
1829 const uint16_t burst_sz = tp->op_params->burst_sz;
1830 const uint16_t num_to_process = tp->op_params->num_to_process;
1831 struct rte_bbdev_dec_op *ops[num_to_process];
1832 struct test_buffers *bufs = NULL;
1833 struct rte_bbdev_info info;
1835 uint16_t num_to_enq, enq;
1837 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1838 "BURST_SIZE should be <= %u", MAX_BURST);
1840 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1841 "Failed to enable interrupts for dev: %u, queue_id: %u",
1842 tp->dev_id, queue_id);
1844 rte_bbdev_info_get(tp->dev_id, &info);
1846 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
1847 "NUM_OPS cannot exceed %u for this device",
1848 info.drv.queue_size_lim);
1850 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1852 rte_atomic16_clear(&tp->processing_status);
1853 rte_atomic16_clear(&tp->nb_dequeued);
1855 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1858 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1860 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1862 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1863 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
1864 bufs->hard_outputs, bufs->soft_outputs,
1865 tp->op_params->ref_dec_op);
1867 /* Set counter to validate the ordering */
1868 for (j = 0; j < num_to_process; ++j)
1869 ops[j]->opaque_data = (void *)(uintptr_t)j;
1871 for (j = 0; j < TEST_REPETITIONS; ++j) {
1872 for (i = 0; i < num_to_process; ++i)
1873 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
1875 tp->start_time = rte_rdtsc_precise();
1876 for (enqueued = 0; enqueued < num_to_process;) {
1877 num_to_enq = burst_sz;
1879 if (unlikely(num_to_process - enqueued < num_to_enq))
1880 num_to_enq = num_to_process - enqueued;
1884 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1885 queue_id, &ops[enqueued],
1887 } while (unlikely(num_to_enq != enq));
1890 /* Write to thread burst_sz current number of enqueued
1891 * descriptors. It ensures that proper number of
1892 * descriptors will be dequeued in callback
1893 * function - needed for last batch in case where
1894 * the number of operations is not a multiple of
1897 rte_atomic16_set(&tp->burst_sz, num_to_enq);
1899 /* Wait until processing of previous batch is
1902 while (rte_atomic16_read(&tp->nb_dequeued) !=
1906 if (j != TEST_REPETITIONS - 1)
1907 rte_atomic16_clear(&tp->nb_dequeued);
1910 return TEST_SUCCESS;
1914 throughput_intr_lcore_enc(void *arg)
1916 struct thread_params *tp = arg;
1917 unsigned int enqueued;
1918 const uint16_t queue_id = tp->queue_id;
1919 const uint16_t burst_sz = tp->op_params->burst_sz;
1920 const uint16_t num_to_process = tp->op_params->num_to_process;
1921 struct rte_bbdev_enc_op *ops[num_to_process];
1922 struct test_buffers *bufs = NULL;
1923 struct rte_bbdev_info info;
1925 uint16_t num_to_enq, enq;
1927 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1928 "BURST_SIZE should be <= %u", MAX_BURST);
1930 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1931 "Failed to enable interrupts for dev: %u, queue_id: %u",
1932 tp->dev_id, queue_id);
1934 rte_bbdev_info_get(tp->dev_id, &info);
1936 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
1937 "NUM_OPS cannot exceed %u for this device",
1938 info.drv.queue_size_lim);
1940 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1942 rte_atomic16_clear(&tp->processing_status);
1943 rte_atomic16_clear(&tp->nb_dequeued);
1945 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1948 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1950 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1952 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1953 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
1954 bufs->hard_outputs, tp->op_params->ref_enc_op);
1956 /* Set counter to validate the ordering */
1957 for (j = 0; j < num_to_process; ++j)
1958 ops[j]->opaque_data = (void *)(uintptr_t)j;
1960 for (j = 0; j < TEST_REPETITIONS; ++j) {
1961 for (i = 0; i < num_to_process; ++i)
1962 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
1964 tp->start_time = rte_rdtsc_precise();
1965 for (enqueued = 0; enqueued < num_to_process;) {
1966 num_to_enq = burst_sz;
1968 if (unlikely(num_to_process - enqueued < num_to_enq))
1969 num_to_enq = num_to_process - enqueued;
1973 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1974 queue_id, &ops[enqueued],
1976 } while (unlikely(enq != num_to_enq));
1979 /* Write to thread burst_sz current number of enqueued
1980 * descriptors. It ensures that proper number of
1981 * descriptors will be dequeued in callback
1982 * function - needed for last batch in case where
1983 * the number of operations is not a multiple of
1986 rte_atomic16_set(&tp->burst_sz, num_to_enq);
1988 /* Wait until processing of previous batch is
1991 while (rte_atomic16_read(&tp->nb_dequeued) !=
1995 if (j != TEST_REPETITIONS - 1)
1996 rte_atomic16_clear(&tp->nb_dequeued);
1999 return TEST_SUCCESS;
2003 throughput_pmd_lcore_dec(void *arg)
2005 struct thread_params *tp = arg;
2007 uint64_t total_time = 0, start_time;
2008 const uint16_t queue_id = tp->queue_id;
2009 const uint16_t burst_sz = tp->op_params->burst_sz;
2010 const uint16_t num_ops = tp->op_params->num_to_process;
2011 struct rte_bbdev_dec_op *ops_enq[num_ops];
2012 struct rte_bbdev_dec_op *ops_deq[num_ops];
2013 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2014 struct test_buffers *bufs = NULL;
2016 struct rte_bbdev_info info;
2017 uint16_t num_to_enq;
2019 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2020 "BURST_SIZE should be <= %u", MAX_BURST);
2022 rte_bbdev_info_get(tp->dev_id, &info);
2024 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2025 "NUM_OPS cannot exceed %u for this device",
2026 info.drv.queue_size_lim);
2028 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2030 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2033 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2034 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2036 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2037 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2038 bufs->hard_outputs, bufs->soft_outputs, ref_op);
2040 /* Set counter to validate the ordering */
2041 for (j = 0; j < num_ops; ++j)
2042 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2044 for (i = 0; i < TEST_REPETITIONS; ++i) {
2046 for (j = 0; j < num_ops; ++j)
2047 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
2049 start_time = rte_rdtsc_precise();
2051 for (enq = 0, deq = 0; enq < num_ops;) {
2052 num_to_enq = burst_sz;
2054 if (unlikely(num_ops - enq < num_to_enq))
2055 num_to_enq = num_ops - enq;
2057 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2058 queue_id, &ops_enq[enq], num_to_enq);
2060 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
2061 queue_id, &ops_deq[deq], enq - deq);
2064 /* dequeue the remaining */
2066 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
2067 queue_id, &ops_deq[deq], enq - deq);
2070 total_time += rte_rdtsc_precise() - start_time;
2074 /* get the max of iter_count for all dequeued ops */
2075 for (i = 0; i < num_ops; ++i) {
2076 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
2080 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2081 ret = validate_dec_op(ops_deq, num_ops, ref_op,
2082 tp->op_params->vector_mask);
2083 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2086 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
2088 double tb_len_bits = calc_dec_TB_size(ref_op);
2090 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2091 ((double)total_time / (double)rte_get_tsc_hz());
2092 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
2093 1000000.0) / ((double)total_time /
2094 (double)rte_get_tsc_hz());
2096 return TEST_SUCCESS;
2100 throughput_pmd_lcore_ldpc_dec(void *arg)
2102 struct thread_params *tp = arg;
2104 uint64_t total_time = 0, start_time;
2105 const uint16_t queue_id = tp->queue_id;
2106 const uint16_t burst_sz = tp->op_params->burst_sz;
2107 const uint16_t num_ops = tp->op_params->num_to_process;
2108 struct rte_bbdev_dec_op *ops_enq[num_ops];
2109 struct rte_bbdev_dec_op *ops_deq[num_ops];
2110 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2111 struct test_buffers *bufs = NULL;
2113 struct rte_bbdev_info info;
2114 uint16_t num_to_enq;
2116 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2117 "BURST_SIZE should be <= %u", MAX_BURST);
2119 rte_bbdev_info_get(tp->dev_id, &info);
2121 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2122 "NUM_OPS cannot exceed %u for this device",
2123 info.drv.queue_size_lim);
2125 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2127 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2130 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2131 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2133 /* For throughput tests we need to disable early termination */
2134 if (check_bit(ref_op->ldpc_dec.op_flags,
2135 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
2136 ref_op->ldpc_dec.op_flags -=
2137 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
2138 ref_op->ldpc_dec.iter_max = 6;
2139 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
2141 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2142 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2143 bufs->hard_outputs, bufs->soft_outputs,
2144 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2146 /* Set counter to validate the ordering */
2147 for (j = 0; j < num_ops; ++j)
2148 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2150 for (i = 0; i < TEST_REPETITIONS; ++i) {
2151 for (j = 0; j < num_ops; ++j) {
2152 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data);
2153 if (check_bit(ref_op->ldpc_dec.op_flags,
2154 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
2156 ops_enq[j]->ldpc_dec.harq_combined_output.data);
2159 start_time = rte_rdtsc_precise();
2161 for (enq = 0, deq = 0; enq < num_ops;) {
2162 num_to_enq = burst_sz;
2164 if (unlikely(num_ops - enq < num_to_enq))
2165 num_to_enq = num_ops - enq;
2167 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
2168 queue_id, &ops_enq[enq], num_to_enq);
2170 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
2171 queue_id, &ops_deq[deq], enq - deq);
2174 /* dequeue the remaining */
2176 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
2177 queue_id, &ops_deq[deq], enq - deq);
2180 total_time += rte_rdtsc_precise() - start_time;
2184 /* get the max of iter_count for all dequeued ops */
2185 for (i = 0; i < num_ops; ++i) {
2186 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
2190 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2191 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
2192 tp->op_params->vector_mask);
2193 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2196 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
2198 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
2200 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2201 ((double)total_time / (double)rte_get_tsc_hz());
2202 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
2203 1000000.0) / ((double)total_time /
2204 (double)rte_get_tsc_hz());
2206 return TEST_SUCCESS;
2210 throughput_pmd_lcore_enc(void *arg)
2212 struct thread_params *tp = arg;
2214 uint64_t total_time = 0, start_time;
2215 const uint16_t queue_id = tp->queue_id;
2216 const uint16_t burst_sz = tp->op_params->burst_sz;
2217 const uint16_t num_ops = tp->op_params->num_to_process;
2218 struct rte_bbdev_enc_op *ops_enq[num_ops];
2219 struct rte_bbdev_enc_op *ops_deq[num_ops];
2220 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2221 struct test_buffers *bufs = NULL;
2223 struct rte_bbdev_info info;
2224 uint16_t num_to_enq;
2226 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2227 "BURST_SIZE should be <= %u", MAX_BURST);
2229 rte_bbdev_info_get(tp->dev_id, &info);
2231 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2232 "NUM_OPS cannot exceed %u for this device",
2233 info.drv.queue_size_lim);
2235 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2237 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2240 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
2242 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2244 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2245 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
2246 bufs->hard_outputs, ref_op);
2248 /* Set counter to validate the ordering */
2249 for (j = 0; j < num_ops; ++j)
2250 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2252 for (i = 0; i < TEST_REPETITIONS; ++i) {
2254 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2255 for (j = 0; j < num_ops; ++j)
2256 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
2258 start_time = rte_rdtsc_precise();
2260 for (enq = 0, deq = 0; enq < num_ops;) {
2261 num_to_enq = burst_sz;
2263 if (unlikely(num_ops - enq < num_to_enq))
2264 num_to_enq = num_ops - enq;
2266 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2267 queue_id, &ops_enq[enq], num_to_enq);
2269 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
2270 queue_id, &ops_deq[deq], enq - deq);
2273 /* dequeue the remaining */
2275 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
2276 queue_id, &ops_deq[deq], enq - deq);
2279 total_time += rte_rdtsc_precise() - start_time;
2282 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2283 ret = validate_enc_op(ops_deq, num_ops, ref_op);
2284 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2287 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
2289 double tb_len_bits = calc_enc_TB_size(ref_op);
2291 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2292 ((double)total_time / (double)rte_get_tsc_hz());
2293 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
2294 / 1000000.0) / ((double)total_time /
2295 (double)rte_get_tsc_hz());
2297 return TEST_SUCCESS;
2301 throughput_pmd_lcore_ldpc_enc(void *arg)
2303 struct thread_params *tp = arg;
2305 uint64_t total_time = 0, start_time;
2306 const uint16_t queue_id = tp->queue_id;
2307 const uint16_t burst_sz = tp->op_params->burst_sz;
2308 const uint16_t num_ops = tp->op_params->num_to_process;
2309 struct rte_bbdev_enc_op *ops_enq[num_ops];
2310 struct rte_bbdev_enc_op *ops_deq[num_ops];
2311 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2312 struct test_buffers *bufs = NULL;
2314 struct rte_bbdev_info info;
2315 uint16_t num_to_enq;
2317 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2318 "BURST_SIZE should be <= %u", MAX_BURST);
2320 rte_bbdev_info_get(tp->dev_id, &info);
2322 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2323 "NUM_OPS cannot exceed %u for this device",
2324 info.drv.queue_size_lim);
2326 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2328 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2331 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
2333 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2335 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2336 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
2337 bufs->hard_outputs, ref_op);
2339 /* Set counter to validate the ordering */
2340 for (j = 0; j < num_ops; ++j)
2341 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2343 for (i = 0; i < TEST_REPETITIONS; ++i) {
2345 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2346 for (j = 0; j < num_ops; ++j)
2347 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
2349 start_time = rte_rdtsc_precise();
2351 for (enq = 0, deq = 0; enq < num_ops;) {
2352 num_to_enq = burst_sz;
2354 if (unlikely(num_ops - enq < num_to_enq))
2355 num_to_enq = num_ops - enq;
2357 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
2358 queue_id, &ops_enq[enq], num_to_enq);
2360 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
2361 queue_id, &ops_deq[deq], enq - deq);
2364 /* dequeue the remaining */
2366 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
2367 queue_id, &ops_deq[deq], enq - deq);
2370 total_time += rte_rdtsc_precise() - start_time;
2373 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2374 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
2375 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2378 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
2380 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
2382 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2383 ((double)total_time / (double)rte_get_tsc_hz());
2384 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
2385 / 1000000.0) / ((double)total_time /
2386 (double)rte_get_tsc_hz());
2388 return TEST_SUCCESS;
2392 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
2394 unsigned int iter = 0;
2395 double total_mops = 0, total_mbps = 0;
2397 for (iter = 0; iter < used_cores; iter++) {
2399 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
2400 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
2401 t_params[iter].mbps);
2402 total_mops += t_params[iter].ops_per_sec;
2403 total_mbps += t_params[iter].mbps;
2406 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
2407 used_cores, total_mops, total_mbps);
2411 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
2413 unsigned int iter = 0;
2414 double total_mops = 0, total_mbps = 0;
2415 uint8_t iter_count = 0;
2417 for (iter = 0; iter < used_cores; iter++) {
2419 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
2420 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
2421 t_params[iter].mbps, t_params[iter].iter_count);
2422 total_mops += t_params[iter].ops_per_sec;
2423 total_mbps += t_params[iter].mbps;
2424 iter_count = RTE_MAX(iter_count, t_params[iter].iter_count);
2427 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
2428 used_cores, total_mops, total_mbps, iter_count);
2432 * Test function that determines how long an enqueue + dequeue of a burst
2433 * takes on available lcores.
2436 throughput_test(struct active_device *ad,
2437 struct test_op_params *op_params)
2440 unsigned int lcore_id, used_cores = 0;
2441 struct thread_params *t_params, *tp;
2442 struct rte_bbdev_info info;
2443 lcore_function_t *throughput_function;
2444 uint16_t num_lcores;
2445 const char *op_type_str;
2447 rte_bbdev_info_get(ad->dev_id, &info);
2449 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
2450 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
2451 test_vector.op_type);
2453 printf("+ ------------------------------------------------------- +\n");
2454 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
2455 info.dev_name, ad->nb_queues, op_params->burst_sz,
2456 op_params->num_to_process, op_params->num_lcores,
2458 intr_enabled ? "Interrupt mode" : "PMD mode",
2459 (double)rte_get_tsc_hz() / 1000000000.0);
2461 /* Set number of lcores */
2462 num_lcores = (ad->nb_queues < (op_params->num_lcores))
2464 : op_params->num_lcores;
2466 /* Allocate memory for thread parameters structure */
2467 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
2468 RTE_CACHE_LINE_SIZE);
2469 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
2470 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
2471 RTE_CACHE_LINE_SIZE));
2474 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2475 throughput_function = throughput_intr_lcore_dec;
2476 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2477 throughput_function = throughput_intr_lcore_dec;
2478 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2479 throughput_function = throughput_intr_lcore_enc;
2480 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2481 throughput_function = throughput_intr_lcore_enc;
2483 throughput_function = throughput_intr_lcore_enc;
2485 /* Dequeue interrupt callback registration */
2486 ret = rte_bbdev_callback_register(ad->dev_id,
2487 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
2494 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2495 throughput_function = throughput_pmd_lcore_dec;
2496 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2497 throughput_function = throughput_pmd_lcore_ldpc_dec;
2498 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2499 throughput_function = throughput_pmd_lcore_enc;
2500 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2501 throughput_function = throughput_pmd_lcore_ldpc_enc;
2503 throughput_function = throughput_pmd_lcore_enc;
2506 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
2508 /* Master core is set at first entry */
2509 t_params[0].dev_id = ad->dev_id;
2510 t_params[0].lcore_id = rte_lcore_id();
2511 t_params[0].op_params = op_params;
2512 t_params[0].queue_id = ad->queue_ids[used_cores++];
2513 t_params[0].iter_count = 0;
2515 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2516 if (used_cores >= num_lcores)
2519 t_params[used_cores].dev_id = ad->dev_id;
2520 t_params[used_cores].lcore_id = lcore_id;
2521 t_params[used_cores].op_params = op_params;
2522 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
2523 t_params[used_cores].iter_count = 0;
2525 rte_eal_remote_launch(throughput_function,
2526 &t_params[used_cores++], lcore_id);
2529 rte_atomic16_set(&op_params->sync, SYNC_START);
2530 ret = throughput_function(&t_params[0]);
2532 /* Master core is always used */
2533 for (used_cores = 1; used_cores < num_lcores; used_cores++)
2534 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
2536 /* Return if test failed */
2542 /* Print throughput if interrupts are disabled and test passed */
2543 if (!intr_enabled) {
2544 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
2545 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2546 print_dec_throughput(t_params, num_lcores);
2548 print_enc_throughput(t_params, num_lcores);
2553 /* In interrupt TC we need to wait for the interrupt callback to deqeue
2554 * all pending operations. Skip waiting for queues which reported an
2555 * error using processing_status variable.
2556 * Wait for master lcore operations.
2559 while ((rte_atomic16_read(&tp->nb_dequeued) <
2560 op_params->num_to_process) &&
2561 (rte_atomic16_read(&tp->processing_status) !=
2565 tp->ops_per_sec /= TEST_REPETITIONS;
2566 tp->mbps /= TEST_REPETITIONS;
2567 ret |= (int)rte_atomic16_read(&tp->processing_status);
2569 /* Wait for slave lcores operations */
2570 for (used_cores = 1; used_cores < num_lcores; used_cores++) {
2571 tp = &t_params[used_cores];
2573 while ((rte_atomic16_read(&tp->nb_dequeued) <
2574 op_params->num_to_process) &&
2575 (rte_atomic16_read(&tp->processing_status) !=
2579 tp->ops_per_sec /= TEST_REPETITIONS;
2580 tp->mbps /= TEST_REPETITIONS;
2581 ret |= (int)rte_atomic16_read(&tp->processing_status);
2584 /* Print throughput if test passed */
2586 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
2587 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2588 print_dec_throughput(t_params, num_lcores);
2589 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
2590 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2591 print_enc_throughput(t_params, num_lcores);
2599 latency_test_dec(struct rte_mempool *mempool,
2600 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
2601 int vector_mask, uint16_t dev_id, uint16_t queue_id,
2602 const uint16_t num_to_process, uint16_t burst_sz,
2603 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2605 int ret = TEST_SUCCESS;
2606 uint16_t i, j, dequeued;
2607 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2608 uint64_t start_time = 0, last_time = 0;
2610 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2611 uint16_t enq = 0, deq = 0;
2612 bool first_time = true;
2615 if (unlikely(num_to_process - dequeued < burst_sz))
2616 burst_sz = num_to_process - dequeued;
2618 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
2619 TEST_ASSERT_SUCCESS(ret,
2620 "rte_bbdev_dec_op_alloc_bulk() failed");
2621 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2622 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
2628 /* Set counter to validate the ordering */
2629 for (j = 0; j < burst_sz; ++j)
2630 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2632 start_time = rte_rdtsc_precise();
2634 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
2636 TEST_ASSERT(enq == burst_sz,
2637 "Error enqueueing burst, expected %u, got %u",
2642 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2643 &ops_deq[deq], burst_sz - deq);
2644 if (likely(first_time && (deq > 0))) {
2645 last_time = rte_rdtsc_precise() - start_time;
2648 } while (unlikely(burst_sz != deq));
2650 *max_time = RTE_MAX(*max_time, last_time);
2651 *min_time = RTE_MIN(*min_time, last_time);
2652 *total_time += last_time;
2654 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2655 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
2657 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2660 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2668 latency_test_ldpc_dec(struct rte_mempool *mempool,
2669 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
2670 int vector_mask, uint16_t dev_id, uint16_t queue_id,
2671 const uint16_t num_to_process, uint16_t burst_sz,
2672 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2674 int ret = TEST_SUCCESS;
2675 uint16_t i, j, dequeued;
2676 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2677 uint64_t start_time = 0, last_time = 0;
2679 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2680 uint16_t enq = 0, deq = 0;
2681 bool first_time = true;
2684 if (unlikely(num_to_process - dequeued < burst_sz))
2685 burst_sz = num_to_process - dequeued;
2687 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
2688 TEST_ASSERT_SUCCESS(ret,
2689 "rte_bbdev_dec_op_alloc_bulk() failed");
2690 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2691 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
2699 /* Set counter to validate the ordering */
2700 for (j = 0; j < burst_sz; ++j)
2701 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2703 start_time = rte_rdtsc_precise();
2705 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2706 &ops_enq[enq], burst_sz);
2707 TEST_ASSERT(enq == burst_sz,
2708 "Error enqueueing burst, expected %u, got %u",
2713 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2714 &ops_deq[deq], burst_sz - deq);
2715 if (likely(first_time && (deq > 0))) {
2716 last_time = rte_rdtsc_precise() - start_time;
2719 } while (unlikely(burst_sz != deq));
2721 *max_time = RTE_MAX(*max_time, last_time);
2722 *min_time = RTE_MIN(*min_time, last_time);
2723 *total_time += last_time;
2725 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2726 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
2728 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2731 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2739 latency_test_enc(struct rte_mempool *mempool,
2740 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
2741 uint16_t dev_id, uint16_t queue_id,
2742 const uint16_t num_to_process, uint16_t burst_sz,
2743 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2745 int ret = TEST_SUCCESS;
2746 uint16_t i, j, dequeued;
2747 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2748 uint64_t start_time = 0, last_time = 0;
2750 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2751 uint16_t enq = 0, deq = 0;
2752 bool first_time = true;
2755 if (unlikely(num_to_process - dequeued < burst_sz))
2756 burst_sz = num_to_process - dequeued;
2758 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2759 TEST_ASSERT_SUCCESS(ret,
2760 "rte_bbdev_enc_op_alloc_bulk() failed");
2761 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2762 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
2767 /* Set counter to validate the ordering */
2768 for (j = 0; j < burst_sz; ++j)
2769 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2771 start_time = rte_rdtsc_precise();
2773 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
2775 TEST_ASSERT(enq == burst_sz,
2776 "Error enqueueing burst, expected %u, got %u",
2781 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2782 &ops_deq[deq], burst_sz - deq);
2783 if (likely(first_time && (deq > 0))) {
2784 last_time += rte_rdtsc_precise() - start_time;
2787 } while (unlikely(burst_sz != deq));
2789 *max_time = RTE_MAX(*max_time, last_time);
2790 *min_time = RTE_MIN(*min_time, last_time);
2791 *total_time += last_time;
2793 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2794 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
2795 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2798 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2806 latency_test_ldpc_enc(struct rte_mempool *mempool,
2807 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
2808 uint16_t dev_id, uint16_t queue_id,
2809 const uint16_t num_to_process, uint16_t burst_sz,
2810 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2812 int ret = TEST_SUCCESS;
2813 uint16_t i, j, dequeued;
2814 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2815 uint64_t start_time = 0, last_time = 0;
2817 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2818 uint16_t enq = 0, deq = 0;
2819 bool first_time = true;
2822 if (unlikely(num_to_process - dequeued < burst_sz))
2823 burst_sz = num_to_process - dequeued;
2825 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2827 TEST_ASSERT_SUCCESS(ret,
2828 "rte_bbdev_enc_op_alloc_bulk() failed");
2829 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2830 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
2835 /* Set counter to validate the ordering */
2836 for (j = 0; j < burst_sz; ++j)
2837 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2839 start_time = rte_rdtsc_precise();
2842 * printf("Latency Debug %d\n",
2843 * ops_enq[0]->ldpc_enc.cb_params.z_c); REMOVEME
2846 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
2847 &ops_enq[enq], burst_sz);
2848 TEST_ASSERT(enq == burst_sz,
2849 "Error enqueueing burst, expected %u, got %u",
2854 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2855 &ops_deq[deq], burst_sz - deq);
2856 if (likely(first_time && (deq > 0))) {
2857 last_time += rte_rdtsc_precise() - start_time;
2860 } while (unlikely(burst_sz != deq));
2862 *max_time = RTE_MAX(*max_time, last_time);
2863 *min_time = RTE_MIN(*min_time, last_time);
2864 *total_time += last_time;
2866 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2867 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
2868 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2872 * printf("Ready to free - deq %d num_to_process %d\n", FIXME
2873 * deq, num_to_process);
2874 * printf("cache %d\n", ops_enq[0]->mempool->cache_size);
2876 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2884 latency_test(struct active_device *ad,
2885 struct test_op_params *op_params)
2888 uint16_t burst_sz = op_params->burst_sz;
2889 const uint16_t num_to_process = op_params->num_to_process;
2890 const enum rte_bbdev_op_type op_type = test_vector.op_type;
2891 const uint16_t queue_id = ad->queue_ids[0];
2892 struct test_buffers *bufs = NULL;
2893 struct rte_bbdev_info info;
2894 uint64_t total_time, min_time, max_time;
2895 const char *op_type_str;
2897 total_time = max_time = 0;
2898 min_time = UINT64_MAX;
2900 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2901 "BURST_SIZE should be <= %u", MAX_BURST);
2903 rte_bbdev_info_get(ad->dev_id, &info);
2904 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2906 op_type_str = rte_bbdev_op_type_str(op_type);
2907 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2909 printf("+ ------------------------------------------------------- +\n");
2910 printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
2911 info.dev_name, burst_sz, num_to_process, op_type_str);
2913 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2914 iter = latency_test_dec(op_params->mp, bufs,
2915 op_params->ref_dec_op, op_params->vector_mask,
2916 ad->dev_id, queue_id, num_to_process,
2917 burst_sz, &total_time, &min_time, &max_time);
2918 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
2919 iter = latency_test_enc(op_params->mp, bufs,
2920 op_params->ref_enc_op, ad->dev_id, queue_id,
2921 num_to_process, burst_sz, &total_time,
2922 &min_time, &max_time);
2923 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
2924 iter = latency_test_ldpc_enc(op_params->mp, bufs,
2925 op_params->ref_enc_op, ad->dev_id, queue_id,
2926 num_to_process, burst_sz, &total_time,
2927 &min_time, &max_time);
2928 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
2929 iter = latency_test_ldpc_dec(op_params->mp, bufs,
2930 op_params->ref_dec_op, op_params->vector_mask,
2931 ad->dev_id, queue_id, num_to_process,
2932 burst_sz, &total_time, &min_time, &max_time);
2934 iter = latency_test_enc(op_params->mp, bufs,
2935 op_params->ref_enc_op,
2936 ad->dev_id, queue_id,
2937 num_to_process, burst_sz, &total_time,
2938 &min_time, &max_time);
2943 printf("Operation latency:\n"
2944 "\tavg: %lg cycles, %lg us\n"
2945 "\tmin: %lg cycles, %lg us\n"
2946 "\tmax: %lg cycles, %lg us\n",
2947 (double)total_time / (double)iter,
2948 (double)(total_time * 1000000) / (double)iter /
2949 (double)rte_get_tsc_hz(), (double)min_time,
2950 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
2951 (double)max_time, (double)(max_time * 1000000) /
2952 (double)rte_get_tsc_hz());
2954 return TEST_SUCCESS;
2957 #ifdef RTE_BBDEV_OFFLOAD_COST
2959 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
2960 struct rte_bbdev_stats *stats)
2962 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
2963 struct rte_bbdev_stats *q_stats;
2965 if (queue_id >= dev->data->num_queues)
2968 q_stats = &dev->data->queues[queue_id].queue_stats;
2970 stats->enqueued_count = q_stats->enqueued_count;
2971 stats->dequeued_count = q_stats->dequeued_count;
2972 stats->enqueue_err_count = q_stats->enqueue_err_count;
2973 stats->dequeue_err_count = q_stats->dequeue_err_count;
2974 stats->acc_offload_cycles = q_stats->acc_offload_cycles;
2980 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
2981 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
2982 uint16_t queue_id, const uint16_t num_to_process,
2983 uint16_t burst_sz, struct test_time_stats *time_st)
2985 int i, dequeued, ret;
2986 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2987 uint64_t enq_start_time, deq_start_time;
2988 uint64_t enq_sw_last_time, deq_last_time;
2989 struct rte_bbdev_stats stats;
2991 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2992 uint16_t enq = 0, deq = 0;
2994 if (unlikely(num_to_process - dequeued < burst_sz))
2995 burst_sz = num_to_process - dequeued;
2997 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
2998 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2999 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3005 /* Start time meas for enqueue function offload latency */
3006 enq_start_time = rte_rdtsc_precise();
3008 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
3009 &ops_enq[enq], burst_sz - enq);
3010 } while (unlikely(burst_sz != enq));
3012 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3013 TEST_ASSERT_SUCCESS(ret,
3014 "Failed to get stats for queue (%u) of device (%u)",
3017 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3018 stats.acc_offload_cycles;
3019 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3021 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3023 time_st->enq_sw_total_time += enq_sw_last_time;
3025 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3026 stats.acc_offload_cycles);
3027 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3028 stats.acc_offload_cycles);
3029 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3031 /* give time for device to process ops */
3034 /* Start time meas for dequeue function offload latency */
3035 deq_start_time = rte_rdtsc_precise();
3036 /* Dequeue one operation */
3038 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3040 } while (unlikely(deq != 1));
3042 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3043 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3045 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3047 time_st->deq_total_time += deq_last_time;
3049 /* Dequeue remaining operations if needed*/
3050 while (burst_sz != deq)
3051 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3052 &ops_deq[deq], burst_sz - deq);
3054 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3062 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
3063 struct test_buffers *bufs,
3064 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
3065 uint16_t queue_id, const uint16_t num_to_process,
3066 uint16_t burst_sz, struct test_time_stats *time_st)
3068 int i, dequeued, ret;
3069 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3070 uint64_t enq_start_time, deq_start_time;
3071 uint64_t enq_sw_last_time, deq_last_time;
3072 struct rte_bbdev_stats stats;
3074 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3075 uint16_t enq = 0, deq = 0;
3077 if (unlikely(num_to_process - dequeued < burst_sz))
3078 burst_sz = num_to_process - dequeued;
3080 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3081 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3082 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
3090 /* Start time meas for enqueue function offload latency */
3091 enq_start_time = rte_rdtsc_precise();
3093 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
3094 &ops_enq[enq], burst_sz - enq);
3095 } while (unlikely(burst_sz != enq));
3097 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3098 TEST_ASSERT_SUCCESS(ret,
3099 "Failed to get stats for queue (%u) of device (%u)",
3102 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3103 stats.acc_offload_cycles;
3104 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3106 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3108 time_st->enq_sw_total_time += enq_sw_last_time;
3110 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3111 stats.acc_offload_cycles);
3112 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3113 stats.acc_offload_cycles);
3114 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3116 /* give time for device to process ops */
3119 /* Start time meas for dequeue function offload latency */
3120 deq_start_time = rte_rdtsc_precise();
3121 /* Dequeue one operation */
3123 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
3125 } while (unlikely(deq != 1));
3127 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3128 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3130 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3132 time_st->deq_total_time += deq_last_time;
3134 /* Dequeue remaining operations if needed*/
3135 while (burst_sz != deq)
3136 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3137 &ops_deq[deq], burst_sz - deq);
3139 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3147 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
3148 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
3149 uint16_t queue_id, const uint16_t num_to_process,
3150 uint16_t burst_sz, struct test_time_stats *time_st)
3152 int i, dequeued, ret;
3153 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3154 uint64_t enq_start_time, deq_start_time;
3155 uint64_t enq_sw_last_time, deq_last_time;
3156 struct rte_bbdev_stats stats;
3158 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3159 uint16_t enq = 0, deq = 0;
3161 if (unlikely(num_to_process - dequeued < burst_sz))
3162 burst_sz = num_to_process - dequeued;
3164 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3165 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
3166 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3167 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
3172 /* Start time meas for enqueue function offload latency */
3173 enq_start_time = rte_rdtsc_precise();
3175 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
3176 &ops_enq[enq], burst_sz - enq);
3177 } while (unlikely(burst_sz != enq));
3179 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3180 TEST_ASSERT_SUCCESS(ret,
3181 "Failed to get stats for queue (%u) of device (%u)",
3184 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3185 stats.acc_offload_cycles;
3186 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3188 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3190 time_st->enq_sw_total_time += enq_sw_last_time;
3192 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3193 stats.acc_offload_cycles);
3194 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3195 stats.acc_offload_cycles);
3196 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3198 /* give time for device to process ops */
3201 /* Start time meas for dequeue function offload latency */
3202 deq_start_time = rte_rdtsc_precise();
3203 /* Dequeue one operation */
3205 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
3207 } while (unlikely(deq != 1));
3209 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3210 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3212 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3214 time_st->deq_total_time += deq_last_time;
3216 while (burst_sz != deq)
3217 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
3218 &ops_deq[deq], burst_sz - deq);
3220 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
3228 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
3229 struct test_buffers *bufs,
3230 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
3231 uint16_t queue_id, const uint16_t num_to_process,
3232 uint16_t burst_sz, struct test_time_stats *time_st)
3234 int i, dequeued, ret;
3235 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3236 uint64_t enq_start_time, deq_start_time;
3237 uint64_t enq_sw_last_time, deq_last_time;
3238 struct rte_bbdev_stats stats;
3240 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3241 uint16_t enq = 0, deq = 0;
3243 if (unlikely(num_to_process - dequeued < burst_sz))
3244 burst_sz = num_to_process - dequeued;
3246 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3247 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
3248 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3249 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
3254 /* Start time meas for enqueue function offload latency */
3255 enq_start_time = rte_rdtsc_precise();
3257 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
3258 &ops_enq[enq], burst_sz - enq);
3259 } while (unlikely(burst_sz != enq));
3261 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3262 TEST_ASSERT_SUCCESS(ret,
3263 "Failed to get stats for queue (%u) of device (%u)",
3266 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3267 stats.acc_offload_cycles;
3268 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3270 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3272 time_st->enq_sw_total_time += enq_sw_last_time;
3274 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3275 stats.acc_offload_cycles);
3276 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3277 stats.acc_offload_cycles);
3278 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3280 /* give time for device to process ops */
3283 /* Start time meas for dequeue function offload latency */
3284 deq_start_time = rte_rdtsc_precise();
3285 /* Dequeue one operation */
3287 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
3289 } while (unlikely(deq != 1));
3291 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3292 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3294 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3296 time_st->deq_total_time += deq_last_time;
3298 while (burst_sz != deq)
3299 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
3300 &ops_deq[deq], burst_sz - deq);
3302 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
3311 offload_cost_test(struct active_device *ad,
3312 struct test_op_params *op_params)
3314 #ifndef RTE_BBDEV_OFFLOAD_COST
3316 RTE_SET_USED(op_params);
3317 printf("Offload latency test is disabled.\n");
3318 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
3319 return TEST_SKIPPED;
3322 uint16_t burst_sz = op_params->burst_sz;
3323 const uint16_t num_to_process = op_params->num_to_process;
3324 const enum rte_bbdev_op_type op_type = test_vector.op_type;
3325 const uint16_t queue_id = ad->queue_ids[0];
3326 struct test_buffers *bufs = NULL;
3327 struct rte_bbdev_info info;
3328 const char *op_type_str;
3329 struct test_time_stats time_st;
3331 memset(&time_st, 0, sizeof(struct test_time_stats));
3332 time_st.enq_sw_min_time = UINT64_MAX;
3333 time_st.enq_acc_min_time = UINT64_MAX;
3334 time_st.deq_min_time = UINT64_MAX;
3336 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3337 "BURST_SIZE should be <= %u", MAX_BURST);
3339 rte_bbdev_info_get(ad->dev_id, &info);
3340 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3342 op_type_str = rte_bbdev_op_type_str(op_type);
3343 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
3345 printf("+ ------------------------------------------------------- +\n");
3346 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
3347 info.dev_name, burst_sz, num_to_process, op_type_str);
3349 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
3350 iter = offload_latency_test_dec(op_params->mp, bufs,
3351 op_params->ref_dec_op, ad->dev_id, queue_id,
3352 num_to_process, burst_sz, &time_st);
3353 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
3354 iter = offload_latency_test_enc(op_params->mp, bufs,
3355 op_params->ref_enc_op, ad->dev_id, queue_id,
3356 num_to_process, burst_sz, &time_st);
3357 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
3358 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
3359 op_params->ref_enc_op, ad->dev_id, queue_id,
3360 num_to_process, burst_sz, &time_st);
3361 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
3362 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
3363 op_params->ref_dec_op, ad->dev_id, queue_id,
3364 num_to_process, burst_sz, &time_st);
3366 iter = offload_latency_test_enc(op_params->mp, bufs,
3367 op_params->ref_enc_op, ad->dev_id, queue_id,
3368 num_to_process, burst_sz, &time_st);
3373 printf("Enqueue driver offload cost latency:\n"
3374 "\tavg: %lg cycles, %lg us\n"
3375 "\tmin: %lg cycles, %lg us\n"
3376 "\tmax: %lg cycles, %lg us\n"
3377 "Enqueue accelerator offload cost latency:\n"
3378 "\tavg: %lg cycles, %lg us\n"
3379 "\tmin: %lg cycles, %lg us\n"
3380 "\tmax: %lg cycles, %lg us\n",
3381 (double)time_st.enq_sw_total_time / (double)iter,
3382 (double)(time_st.enq_sw_total_time * 1000000) /
3383 (double)iter / (double)rte_get_tsc_hz(),
3384 (double)time_st.enq_sw_min_time,
3385 (double)(time_st.enq_sw_min_time * 1000000) /
3386 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
3387 (double)(time_st.enq_sw_max_time * 1000000) /
3388 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
3390 (double)(time_st.enq_acc_total_time * 1000000) /
3391 (double)iter / (double)rte_get_tsc_hz(),
3392 (double)time_st.enq_acc_min_time,
3393 (double)(time_st.enq_acc_min_time * 1000000) /
3394 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
3395 (double)(time_st.enq_acc_max_time * 1000000) /
3398 printf("Dequeue offload cost latency - one op:\n"
3399 "\tavg: %lg cycles, %lg us\n"
3400 "\tmin: %lg cycles, %lg us\n"
3401 "\tmax: %lg cycles, %lg us\n",
3402 (double)time_st.deq_total_time / (double)iter,
3403 (double)(time_st.deq_total_time * 1000000) /
3404 (double)iter / (double)rte_get_tsc_hz(),
3405 (double)time_st.deq_min_time,
3406 (double)(time_st.deq_min_time * 1000000) /
3407 rte_get_tsc_hz(), (double)time_st.deq_max_time,
3408 (double)(time_st.deq_max_time * 1000000) /
3411 return TEST_SUCCESS;
3415 #ifdef RTE_BBDEV_OFFLOAD_COST
3417 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
3418 const uint16_t num_to_process, uint16_t burst_sz,
3419 uint64_t *deq_total_time, uint64_t *deq_min_time,
3420 uint64_t *deq_max_time)
3423 struct rte_bbdev_dec_op *ops[MAX_BURST];
3424 uint64_t deq_start_time, deq_last_time;
3426 /* Test deq offload latency from an empty queue */
3428 for (i = 0, deq_total = 0; deq_total < num_to_process;
3429 ++i, deq_total += burst_sz) {
3430 deq_start_time = rte_rdtsc_precise();
3432 if (unlikely(num_to_process - deq_total < burst_sz))
3433 burst_sz = num_to_process - deq_total;
3434 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
3436 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3437 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
3438 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
3439 *deq_total_time += deq_last_time;
3446 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
3447 const uint16_t num_to_process, uint16_t burst_sz,
3448 uint64_t *deq_total_time, uint64_t *deq_min_time,
3449 uint64_t *deq_max_time)
3452 struct rte_bbdev_enc_op *ops[MAX_BURST];
3453 uint64_t deq_start_time, deq_last_time;
3455 /* Test deq offload latency from an empty queue */
3456 for (i = 0, deq_total = 0; deq_total < num_to_process;
3457 ++i, deq_total += burst_sz) {
3458 deq_start_time = rte_rdtsc_precise();
3460 if (unlikely(num_to_process - deq_total < burst_sz))
3461 burst_sz = num_to_process - deq_total;
3462 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
3464 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3465 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
3466 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
3467 *deq_total_time += deq_last_time;
3475 offload_latency_empty_q_test(struct active_device *ad,
3476 struct test_op_params *op_params)
3478 #ifndef RTE_BBDEV_OFFLOAD_COST
3480 RTE_SET_USED(op_params);
3481 printf("Offload latency empty dequeue test is disabled.\n");
3482 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
3483 return TEST_SKIPPED;
3486 uint64_t deq_total_time, deq_min_time, deq_max_time;
3487 uint16_t burst_sz = op_params->burst_sz;
3488 const uint16_t num_to_process = op_params->num_to_process;
3489 const enum rte_bbdev_op_type op_type = test_vector.op_type;
3490 const uint16_t queue_id = ad->queue_ids[0];
3491 struct rte_bbdev_info info;
3492 const char *op_type_str;
3494 deq_total_time = deq_max_time = 0;
3495 deq_min_time = UINT64_MAX;
3497 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3498 "BURST_SIZE should be <= %u", MAX_BURST);
3500 rte_bbdev_info_get(ad->dev_id, &info);
3502 op_type_str = rte_bbdev_op_type_str(op_type);
3503 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
3505 printf("+ ------------------------------------------------------- +\n");
3506 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
3507 info.dev_name, burst_sz, num_to_process, op_type_str);
3509 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
3510 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
3511 num_to_process, burst_sz, &deq_total_time,
3512 &deq_min_time, &deq_max_time);
3514 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
3515 num_to_process, burst_sz, &deq_total_time,
3516 &deq_min_time, &deq_max_time);
3521 printf("Empty dequeue offload:\n"
3522 "\tavg: %lg cycles, %lg us\n"
3523 "\tmin: %lg cycles, %lg us\n"
3524 "\tmax: %lg cycles, %lg us\n",
3525 (double)deq_total_time / (double)iter,
3526 (double)(deq_total_time * 1000000) / (double)iter /
3527 (double)rte_get_tsc_hz(), (double)deq_min_time,
3528 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
3529 (double)deq_max_time, (double)(deq_max_time * 1000000) /
3532 return TEST_SUCCESS;
3539 return run_test_case(throughput_test);
3543 offload_cost_tc(void)
3545 return run_test_case(offload_cost_test);
3549 offload_latency_empty_q_tc(void)
3551 return run_test_case(offload_latency_empty_q_test);
3557 return run_test_case(latency_test);
3563 return run_test_case(throughput_test);
3566 static struct unit_test_suite bbdev_throughput_testsuite = {
3567 .suite_name = "BBdev Throughput Tests",
3568 .setup = testsuite_setup,
3569 .teardown = testsuite_teardown,
3570 .unit_test_cases = {
3571 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
3572 TEST_CASES_END() /**< NULL terminate unit test array */
3576 static struct unit_test_suite bbdev_validation_testsuite = {
3577 .suite_name = "BBdev Validation Tests",
3578 .setup = testsuite_setup,
3579 .teardown = testsuite_teardown,
3580 .unit_test_cases = {
3581 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
3582 TEST_CASES_END() /**< NULL terminate unit test array */
3586 static struct unit_test_suite bbdev_latency_testsuite = {
3587 .suite_name = "BBdev Latency Tests",
3588 .setup = testsuite_setup,
3589 .teardown = testsuite_teardown,
3590 .unit_test_cases = {
3591 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
3592 TEST_CASES_END() /**< NULL terminate unit test array */
3596 static struct unit_test_suite bbdev_offload_cost_testsuite = {
3597 .suite_name = "BBdev Offload Cost Tests",
3598 .setup = testsuite_setup,
3599 .teardown = testsuite_teardown,
3600 .unit_test_cases = {
3601 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
3602 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
3603 TEST_CASES_END() /**< NULL terminate unit test array */
3607 static struct unit_test_suite bbdev_interrupt_testsuite = {
3608 .suite_name = "BBdev Interrupt Tests",
3609 .setup = interrupt_testsuite_setup,
3610 .teardown = testsuite_teardown,
3611 .unit_test_cases = {
3612 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
3613 TEST_CASES_END() /**< NULL terminate unit test array */
3617 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
3618 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
3619 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
3620 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
3621 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);