1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
19 #include <rte_interrupts.h>
22 #include "test_bbdev_vector.h"
24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
26 #define MAX_QUEUES RTE_MAX_LCORE
27 #define TEST_REPETITIONS 1000
29 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
30 #include <fpga_lte_fec.h>
31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
33 #define VF_UL_4G_QUEUE_VALUE 4
34 #define VF_DL_4G_QUEUE_VALUE 4
35 #define UL_4G_BANDWIDTH 3
36 #define DL_4G_BANDWIDTH 3
37 #define UL_4G_LOAD_BALANCE 128
38 #define DL_4G_LOAD_BALANCE 128
39 #define FLR_4G_TIMEOUT 610
42 #define OPS_CACHE_SIZE 256U
43 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
48 #define INVALID_QUEUE_ID -1
50 static struct test_bbdev_vector test_vector;
52 /* Switch between PMD and Interrupt for throughput TC */
53 static bool intr_enabled;
55 /* Represents tested active devices */
56 static struct active_device {
57 const char *driver_name;
59 uint16_t supported_ops;
60 uint16_t queue_ids[MAX_QUEUES];
62 struct rte_mempool *ops_mempool;
63 struct rte_mempool *in_mbuf_pool;
64 struct rte_mempool *hard_out_mbuf_pool;
65 struct rte_mempool *soft_out_mbuf_pool;
66 struct rte_mempool *harq_in_mbuf_pool;
67 struct rte_mempool *harq_out_mbuf_pool;
68 } active_devs[RTE_BBDEV_MAX_DEVS];
70 static uint8_t nb_active_devs;
72 /* Data buffers used by BBDEV ops */
74 struct rte_bbdev_op_data *inputs;
75 struct rte_bbdev_op_data *hard_outputs;
76 struct rte_bbdev_op_data *soft_outputs;
77 struct rte_bbdev_op_data *harq_inputs;
78 struct rte_bbdev_op_data *harq_outputs;
81 /* Operation parameters specific for given test case */
82 struct test_op_params {
83 struct rte_mempool *mp;
84 struct rte_bbdev_dec_op *ref_dec_op;
85 struct rte_bbdev_enc_op *ref_enc_op;
87 uint16_t num_to_process;
91 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
94 /* Contains per lcore params */
95 struct thread_params {
103 rte_atomic16_t nb_dequeued;
104 rte_atomic16_t processing_status;
105 rte_atomic16_t burst_sz;
106 struct test_op_params *op_params;
107 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
108 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
111 #ifdef RTE_BBDEV_OFFLOAD_COST
112 /* Stores time statistics */
113 struct test_time_stats {
114 /* Stores software enqueue total working time */
115 uint64_t enq_sw_total_time;
116 /* Stores minimum value of software enqueue working time */
117 uint64_t enq_sw_min_time;
118 /* Stores maximum value of software enqueue working time */
119 uint64_t enq_sw_max_time;
120 /* Stores turbo enqueue total working time */
121 uint64_t enq_acc_total_time;
122 /* Stores minimum value of accelerator enqueue working time */
123 uint64_t enq_acc_min_time;
124 /* Stores maximum value of accelerator enqueue working time */
125 uint64_t enq_acc_max_time;
126 /* Stores dequeue total working time */
127 uint64_t deq_total_time;
128 /* Stores minimum value of dequeue working time */
129 uint64_t deq_min_time;
130 /* Stores maximum value of dequeue working time */
131 uint64_t deq_max_time;
135 typedef int (test_case_function)(struct active_device *ad,
136 struct test_op_params *op_params);
139 mbuf_reset(struct rte_mbuf *m)
149 /* Read flag value 0/1 from bitmap */
151 check_bit(uint32_t bitmap, uint32_t bitmask)
153 return bitmap & bitmask;
157 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
159 ad->supported_ops |= (1 << op_type);
163 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
165 return ad->supported_ops & (1 << op_type);
169 flags_match(uint32_t flags_req, uint32_t flags_present)
171 return (flags_req & flags_present) == flags_req;
175 clear_soft_out_cap(uint32_t *op_flags)
177 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
178 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
179 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
183 check_dev_cap(const struct rte_bbdev_info *dev_info)
186 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,
187 nb_harq_inputs, nb_harq_outputs;
188 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
190 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
191 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
192 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
193 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;
194 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;
196 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
197 if (op_cap->type != test_vector.op_type)
200 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
201 const struct rte_bbdev_op_cap_turbo_dec *cap =
202 &op_cap->cap.turbo_dec;
203 /* Ignore lack of soft output capability, just skip
204 * checking if soft output is valid.
206 if ((test_vector.turbo_dec.op_flags &
207 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
208 !(cap->capability_flags &
209 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
211 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
214 &test_vector.turbo_dec.op_flags);
217 if (!flags_match(test_vector.turbo_dec.op_flags,
218 cap->capability_flags))
220 if (nb_inputs > cap->num_buffers_src) {
221 printf("Too many inputs defined: %u, max: %u\n",
222 nb_inputs, cap->num_buffers_src);
225 if (nb_soft_outputs > cap->num_buffers_soft_out &&
226 (test_vector.turbo_dec.op_flags &
227 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
229 "Too many soft outputs defined: %u, max: %u\n",
231 cap->num_buffers_soft_out);
234 if (nb_hard_outputs > cap->num_buffers_hard_out) {
236 "Too many hard outputs defined: %u, max: %u\n",
238 cap->num_buffers_hard_out);
241 if (intr_enabled && !(cap->capability_flags &
242 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
244 "Dequeue interrupts are not supported!\n");
249 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
250 const struct rte_bbdev_op_cap_turbo_enc *cap =
251 &op_cap->cap.turbo_enc;
253 if (!flags_match(test_vector.turbo_enc.op_flags,
254 cap->capability_flags))
256 if (nb_inputs > cap->num_buffers_src) {
257 printf("Too many inputs defined: %u, max: %u\n",
258 nb_inputs, cap->num_buffers_src);
261 if (nb_hard_outputs > cap->num_buffers_dst) {
263 "Too many hard outputs defined: %u, max: %u\n",
264 nb_hard_outputs, cap->num_buffers_dst);
267 if (intr_enabled && !(cap->capability_flags &
268 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
270 "Dequeue interrupts are not supported!\n");
275 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {
276 const struct rte_bbdev_op_cap_ldpc_enc *cap =
277 &op_cap->cap.ldpc_enc;
279 if (!flags_match(test_vector.ldpc_enc.op_flags,
280 cap->capability_flags)){
281 printf("Flag Mismatch\n");
284 if (nb_inputs > cap->num_buffers_src) {
285 printf("Too many inputs defined: %u, max: %u\n",
286 nb_inputs, cap->num_buffers_src);
289 if (nb_hard_outputs > cap->num_buffers_dst) {
291 "Too many hard outputs defined: %u, max: %u\n",
292 nb_hard_outputs, cap->num_buffers_dst);
295 if (intr_enabled && !(cap->capability_flags &
296 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
298 "Dequeue interrupts are not supported!\n");
303 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {
304 const struct rte_bbdev_op_cap_ldpc_dec *cap =
305 &op_cap->cap.ldpc_dec;
307 if (!flags_match(test_vector.ldpc_dec.op_flags,
308 cap->capability_flags)){
309 printf("Flag Mismatch\n");
312 if (nb_inputs > cap->num_buffers_src) {
313 printf("Too many inputs defined: %u, max: %u\n",
314 nb_inputs, cap->num_buffers_src);
317 if (nb_hard_outputs > cap->num_buffers_hard_out) {
319 "Too many hard outputs defined: %u, max: %u\n",
321 cap->num_buffers_hard_out);
324 if (nb_harq_inputs > cap->num_buffers_hard_out) {
326 "Too many HARQ inputs defined: %u, max: %u\n",
328 cap->num_buffers_hard_out);
331 if (nb_harq_outputs > cap->num_buffers_hard_out) {
333 "Too many HARQ outputs defined: %u, max: %u\n",
335 cap->num_buffers_hard_out);
338 if (intr_enabled && !(cap->capability_flags &
339 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
341 "Dequeue interrupts are not supported!\n");
349 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
350 return TEST_SUCCESS; /* Special case for NULL device */
355 /* calculates optimal mempool size not smaller than the val */
357 optimal_mempool_size(unsigned int val)
359 return rte_align32pow2(val + 1) - 1;
362 /* allocates mbuf mempool for inputs and outputs */
363 static struct rte_mempool *
364 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
365 int socket_id, unsigned int mbuf_pool_size,
366 const char *op_type_str)
369 uint32_t max_seg_sz = 0;
370 char pool_name[RTE_MEMPOOL_NAMESIZE];
372 /* find max input segment size */
373 for (i = 0; i < entries->nb_segments; ++i)
374 if (entries->segments[i].length > max_seg_sz)
375 max_seg_sz = entries->segments[i].length;
377 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
379 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
380 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
381 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
385 create_mempools(struct active_device *ad, int socket_id,
386 enum rte_bbdev_op_type org_op_type, uint16_t num_ops)
388 struct rte_mempool *mp;
389 unsigned int ops_pool_size, mbuf_pool_size = 0;
390 char pool_name[RTE_MEMPOOL_NAMESIZE];
391 const char *op_type_str;
392 enum rte_bbdev_op_type op_type = org_op_type;
394 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
395 struct op_data_entries *hard_out =
396 &test_vector.entries[DATA_HARD_OUTPUT];
397 struct op_data_entries *soft_out =
398 &test_vector.entries[DATA_SOFT_OUTPUT];
399 struct op_data_entries *harq_in =
400 &test_vector.entries[DATA_HARQ_INPUT];
401 struct op_data_entries *harq_out =
402 &test_vector.entries[DATA_HARQ_OUTPUT];
404 /* allocate ops mempool */
405 ops_pool_size = optimal_mempool_size(RTE_MAX(
406 /* Ops used plus 1 reference op */
407 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
408 /* Minimal cache size plus 1 reference op */
409 (unsigned int)(1.5 * rte_lcore_count() *
410 OPS_CACHE_SIZE + 1)),
413 if (org_op_type == RTE_BBDEV_OP_NONE)
414 op_type = RTE_BBDEV_OP_TURBO_ENC;
416 op_type_str = rte_bbdev_op_type_str(op_type);
417 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
419 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
421 mp = rte_bbdev_op_pool_create(pool_name, op_type,
422 ops_pool_size, OPS_CACHE_SIZE, socket_id);
423 TEST_ASSERT_NOT_NULL(mp,
424 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
428 ad->ops_mempool = mp;
430 /* Do not create inputs and outputs mbufs for BaseBand Null Device */
431 if (org_op_type == RTE_BBDEV_OP_NONE)
435 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
436 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
437 TEST_ASSERT_NOT_NULL(mp,
438 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
442 ad->in_mbuf_pool = mp;
445 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
446 hard_out->nb_segments);
447 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
449 TEST_ASSERT_NOT_NULL(mp,
450 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
454 ad->hard_out_mbuf_pool = mp;
458 if (soft_out->nb_segments > 0) {
459 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
460 soft_out->nb_segments);
461 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,
464 TEST_ASSERT_NOT_NULL(mp,
465 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
469 ad->soft_out_mbuf_pool = mp;
473 if (harq_in->nb_segments > 0) {
474 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
475 harq_in->nb_segments);
476 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,
479 TEST_ASSERT_NOT_NULL(mp,
480 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",
484 ad->harq_in_mbuf_pool = mp;
488 if (harq_out->nb_segments > 0) {
489 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
490 harq_out->nb_segments);
491 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,
494 TEST_ASSERT_NOT_NULL(mp,
495 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",
499 ad->harq_out_mbuf_pool = mp;
506 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
507 struct test_bbdev_vector *vector)
510 unsigned int queue_id;
511 struct rte_bbdev_queue_conf qconf;
512 struct active_device *ad = &active_devs[nb_active_devs];
513 unsigned int nb_queues;
514 enum rte_bbdev_op_type op_type = vector->op_type;
516 /* Configure fpga lte fec with PF & VF values
517 * if '-i' flag is set and using fpga device
519 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
520 if ((get_init_device() == true) &&
521 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
522 struct fpga_lte_fec_conf conf;
525 printf("Configure FPGA LTE FEC Driver %s with default values\n",
526 info->drv.driver_name);
528 /* clear default configuration before initialization */
529 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
532 * true if PF is used for data plane
535 conf.pf_mode_en = true;
537 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
538 /* Number of UL queues per VF (fpga supports 8 VFs) */
539 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
540 /* Number of DL queues per VF (fpga supports 8 VFs) */
541 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
544 /* UL bandwidth. Needed for schedule algorithm */
545 conf.ul_bandwidth = UL_4G_BANDWIDTH;
547 conf.dl_bandwidth = DL_4G_BANDWIDTH;
549 /* UL & DL load Balance Factor to 64 */
550 conf.ul_load_balance = UL_4G_LOAD_BALANCE;
551 conf.dl_load_balance = DL_4G_LOAD_BALANCE;
553 /**< FLR timeout value */
554 conf.flr_time_out = FLR_4G_TIMEOUT;
556 /* setup FPGA PF with configuration information */
557 ret = fpga_lte_fec_configure(info->dev_name, &conf);
558 TEST_ASSERT_SUCCESS(ret,
559 "Failed to configure 4G FPGA PF for bbdev %s",
563 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
564 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
567 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
569 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
570 dev_id, nb_queues, info->socket_id, ret);
574 /* configure interrupts if needed */
576 ret = rte_bbdev_intr_enable(dev_id);
578 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
584 /* setup device queues */
585 qconf.socket = info->socket_id;
586 qconf.queue_size = info->drv.default_queue_conf.queue_size;
588 qconf.deferred_start = 0;
589 qconf.op_type = op_type;
591 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
592 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
595 "Allocated all queues (id=%u) at prio%u on dev%u\n",
596 queue_id, qconf.priority, dev_id);
598 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
602 printf("All queues on dev %u allocated: %u\n",
606 ad->queue_ids[queue_id] = queue_id;
608 TEST_ASSERT(queue_id != 0,
609 "ERROR Failed to configure any queues on dev %u",
611 ad->nb_queues = queue_id;
613 set_avail_op(ad, op_type);
619 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
620 struct test_bbdev_vector *vector)
624 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
625 active_devs[nb_active_devs].dev_id = dev_id;
627 ret = add_bbdev_dev(dev_id, info, vector);
628 if (ret == TEST_SUCCESS)
634 populate_active_devices(void)
638 uint8_t nb_devs_added = 0;
639 struct rte_bbdev_info info;
641 RTE_BBDEV_FOREACH(dev_id) {
642 rte_bbdev_info_get(dev_id, &info);
644 if (check_dev_cap(&info)) {
646 "Device %d (%s) does not support specified capabilities\n",
647 dev_id, info.dev_name);
651 ret = add_active_device(dev_id, &info, &test_vector);
653 printf("Adding active bbdev %s skipped\n",
660 return nb_devs_added;
664 read_test_vector(void)
668 memset(&test_vector, 0, sizeof(test_vector));
669 printf("Test vector file = %s\n", get_vector_filename());
670 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
671 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
672 get_vector_filename());
678 testsuite_setup(void)
680 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
682 if (populate_active_devices() == 0) {
683 printf("No suitable devices found!\n");
691 interrupt_testsuite_setup(void)
693 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
695 /* Enable interrupts */
698 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
699 if (populate_active_devices() == 0 ||
700 test_vector.op_type == RTE_BBDEV_OP_NONE) {
701 intr_enabled = false;
702 printf("No suitable devices found!\n");
710 testsuite_teardown(void)
714 /* Unconfigure devices */
715 RTE_BBDEV_FOREACH(dev_id)
716 rte_bbdev_close(dev_id);
718 /* Clear active devices structs. */
719 memset(active_devs, 0, sizeof(active_devs));
728 for (i = 0; i < nb_active_devs; i++) {
729 dev_id = active_devs[i].dev_id;
730 /* reset bbdev stats */
731 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
732 "Failed to reset stats of bbdev %u", dev_id);
733 /* start the device */
734 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
735 "Failed to start bbdev %u", dev_id);
745 struct rte_bbdev_stats stats;
747 for (i = 0; i < nb_active_devs; i++) {
748 dev_id = active_devs[i].dev_id;
749 /* read stats and print */
750 rte_bbdev_stats_get(dev_id, &stats);
751 /* Stop the device */
752 rte_bbdev_stop(dev_id);
757 init_op_data_objs(struct rte_bbdev_op_data *bufs,
758 struct op_data_entries *ref_entries,
759 struct rte_mempool *mbuf_pool, const uint16_t n,
760 enum op_data_type op_type, uint16_t min_alignment)
764 bool large_input = false;
766 for (i = 0; i < n; ++i) {
768 struct op_data_buf *seg = &ref_entries->segments[0];
769 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
770 TEST_ASSERT_NOT_NULL(m_head,
771 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
772 op_type, n * ref_entries->nb_segments,
775 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) {
777 * Special case when DPDK mbuf cannot handle
778 * the required input size
780 printf("Warning: Larger input size than DPDK mbuf %d\n",
784 bufs[i].data = m_head;
788 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
789 if ((op_type == DATA_INPUT) && large_input) {
790 /* Allocate a fake overused mbuf */
791 data = rte_malloc(NULL, seg->length, 0);
792 memcpy(data, seg->addr, seg->length);
793 m_head->buf_addr = data;
794 m_head->buf_iova = rte_malloc_virt2iova(data);
795 m_head->data_off = 0;
796 m_head->data_len = seg->length;
798 data = rte_pktmbuf_append(m_head, seg->length);
799 TEST_ASSERT_NOT_NULL(data,
800 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
801 seg->length, op_type);
803 TEST_ASSERT(data == RTE_PTR_ALIGN(
804 data, min_alignment),
805 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
806 data, min_alignment);
807 rte_memcpy(data, seg->addr, seg->length);
810 bufs[i].length += seg->length;
812 for (j = 1; j < ref_entries->nb_segments; ++j) {
813 struct rte_mbuf *m_tail =
814 rte_pktmbuf_alloc(mbuf_pool);
815 TEST_ASSERT_NOT_NULL(m_tail,
816 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
818 n * ref_entries->nb_segments,
822 data = rte_pktmbuf_append(m_tail, seg->length);
823 TEST_ASSERT_NOT_NULL(data,
824 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
825 seg->length, op_type);
827 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
829 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
830 data, min_alignment);
831 rte_memcpy(data, seg->addr, seg->length);
832 bufs[i].length += seg->length;
834 ret = rte_pktmbuf_chain(m_head, m_tail);
835 TEST_ASSERT_SUCCESS(ret,
836 "Couldn't chain mbufs from %d data type mbuf pool",
841 /* allocate chained-mbuf for output buffer */
842 for (j = 1; j < ref_entries->nb_segments; ++j) {
843 struct rte_mbuf *m_tail =
844 rte_pktmbuf_alloc(mbuf_pool);
845 TEST_ASSERT_NOT_NULL(m_tail,
846 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
848 n * ref_entries->nb_segments,
851 ret = rte_pktmbuf_chain(m_head, m_tail);
852 TEST_ASSERT_SUCCESS(ret,
853 "Couldn't chain mbufs from %d data type mbuf pool",
863 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
868 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
869 if (*buffers == NULL) {
870 printf("WARNING: Failed to allocate op_data on socket %d\n",
872 /* try to allocate memory on other detected sockets */
873 for (i = 0; i < socket; i++) {
874 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
875 if (*buffers != NULL)
880 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
884 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
885 const uint16_t n, const int8_t max_llr_modulus)
887 uint16_t i, byte_idx;
889 for (i = 0; i < n; ++i) {
890 struct rte_mbuf *m = input_ops[i].data;
892 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
893 input_ops[i].offset);
894 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
896 llr[byte_idx] = round((double)max_llr_modulus *
897 llr[byte_idx] / INT8_MAX);
905 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,
906 const uint16_t n, const int8_t llr_size,
907 const int8_t llr_decimals)
909 if (input_ops == NULL)
912 uint16_t i, byte_idx;
914 int16_t llr_max, llr_min, llr_tmp;
915 llr_max = (1 << (llr_size - 1)) - 1;
917 for (i = 0; i < n; ++i) {
918 struct rte_mbuf *m = input_ops[i].data;
920 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
921 input_ops[i].offset);
922 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);
925 llr_tmp = llr[byte_idx];
926 if (llr_decimals == 2)
928 else if (llr_decimals == 0)
930 llr_tmp = RTE_MIN(llr_max,
931 RTE_MAX(llr_min, llr_tmp));
932 llr[byte_idx] = (int8_t) llr_tmp;
943 fill_queue_buffers(struct test_op_params *op_params,
944 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
945 struct rte_mempool *soft_out_mp,
946 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp,
948 const struct rte_bbdev_op_cap *capabilities,
949 uint16_t min_alignment, const int socket_id)
952 enum op_data_type type;
953 const uint16_t n = op_params->num_to_process;
955 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
963 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
964 &op_params->q_bufs[socket_id][queue_id].inputs,
965 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
966 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
967 &op_params->q_bufs[socket_id][queue_id].harq_inputs,
968 &op_params->q_bufs[socket_id][queue_id].harq_outputs,
971 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
972 struct op_data_entries *ref_entries =
973 &test_vector.entries[type];
974 if (ref_entries->nb_segments == 0)
977 ret = allocate_buffers_on_socket(queue_ops[type],
978 n * sizeof(struct rte_bbdev_op_data),
980 TEST_ASSERT_SUCCESS(ret,
981 "Couldn't allocate memory for rte_bbdev_op_data structs");
983 ret = init_op_data_objs(*queue_ops[type], ref_entries,
984 mbuf_pools[type], n, type, min_alignment);
985 TEST_ASSERT_SUCCESS(ret,
986 "Couldn't init rte_bbdev_op_data structs");
989 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
990 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
991 capabilities->cap.turbo_dec.max_llr_modulus);
993 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
994 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,
995 capabilities->cap.ldpc_dec.llr_size,
996 capabilities->cap.ldpc_dec.llr_decimals);
997 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,
998 capabilities->cap.ldpc_dec.llr_size,
999 capabilities->cap.ldpc_dec.llr_decimals);
1006 free_buffers(struct active_device *ad, struct test_op_params *op_params)
1010 rte_mempool_free(ad->ops_mempool);
1011 rte_mempool_free(ad->in_mbuf_pool);
1012 rte_mempool_free(ad->hard_out_mbuf_pool);
1013 rte_mempool_free(ad->soft_out_mbuf_pool);
1014 rte_mempool_free(ad->harq_in_mbuf_pool);
1015 rte_mempool_free(ad->harq_out_mbuf_pool);
1017 for (i = 0; i < rte_lcore_count(); ++i) {
1018 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
1019 rte_free(op_params->q_bufs[j][i].inputs);
1020 rte_free(op_params->q_bufs[j][i].hard_outputs);
1021 rte_free(op_params->q_bufs[j][i].soft_outputs);
1022 rte_free(op_params->q_bufs[j][i].harq_inputs);
1023 rte_free(op_params->q_bufs[j][i].harq_outputs);
1029 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1030 unsigned int start_idx,
1031 struct rte_bbdev_op_data *inputs,
1032 struct rte_bbdev_op_data *hard_outputs,
1033 struct rte_bbdev_op_data *soft_outputs,
1034 struct rte_bbdev_dec_op *ref_op)
1037 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
1039 for (i = 0; i < n; ++i) {
1040 if (turbo_dec->code_block_mode == 0) {
1041 ops[i]->turbo_dec.tb_params.ea =
1042 turbo_dec->tb_params.ea;
1043 ops[i]->turbo_dec.tb_params.eb =
1044 turbo_dec->tb_params.eb;
1045 ops[i]->turbo_dec.tb_params.k_pos =
1046 turbo_dec->tb_params.k_pos;
1047 ops[i]->turbo_dec.tb_params.k_neg =
1048 turbo_dec->tb_params.k_neg;
1049 ops[i]->turbo_dec.tb_params.c =
1050 turbo_dec->tb_params.c;
1051 ops[i]->turbo_dec.tb_params.c_neg =
1052 turbo_dec->tb_params.c_neg;
1053 ops[i]->turbo_dec.tb_params.cab =
1054 turbo_dec->tb_params.cab;
1055 ops[i]->turbo_dec.tb_params.r =
1056 turbo_dec->tb_params.r;
1058 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
1059 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
1062 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
1063 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
1064 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
1065 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
1066 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
1067 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
1068 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
1070 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
1071 ops[i]->turbo_dec.input = inputs[start_idx + i];
1072 if (soft_outputs != NULL)
1073 ops[i]->turbo_dec.soft_output =
1074 soft_outputs[start_idx + i];
1079 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1080 unsigned int start_idx,
1081 struct rte_bbdev_op_data *inputs,
1082 struct rte_bbdev_op_data *outputs,
1083 struct rte_bbdev_enc_op *ref_op)
1086 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
1087 for (i = 0; i < n; ++i) {
1088 if (turbo_enc->code_block_mode == 0) {
1089 ops[i]->turbo_enc.tb_params.ea =
1090 turbo_enc->tb_params.ea;
1091 ops[i]->turbo_enc.tb_params.eb =
1092 turbo_enc->tb_params.eb;
1093 ops[i]->turbo_enc.tb_params.k_pos =
1094 turbo_enc->tb_params.k_pos;
1095 ops[i]->turbo_enc.tb_params.k_neg =
1096 turbo_enc->tb_params.k_neg;
1097 ops[i]->turbo_enc.tb_params.c =
1098 turbo_enc->tb_params.c;
1099 ops[i]->turbo_enc.tb_params.c_neg =
1100 turbo_enc->tb_params.c_neg;
1101 ops[i]->turbo_enc.tb_params.cab =
1102 turbo_enc->tb_params.cab;
1103 ops[i]->turbo_enc.tb_params.ncb_pos =
1104 turbo_enc->tb_params.ncb_pos;
1105 ops[i]->turbo_enc.tb_params.ncb_neg =
1106 turbo_enc->tb_params.ncb_neg;
1107 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
1109 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
1110 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
1111 ops[i]->turbo_enc.cb_params.ncb =
1112 turbo_enc->cb_params.ncb;
1114 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
1115 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
1116 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
1118 ops[i]->turbo_enc.output = outputs[start_idx + i];
1119 ops[i]->turbo_enc.input = inputs[start_idx + i];
1124 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
1125 unsigned int start_idx,
1126 struct rte_bbdev_op_data *inputs,
1127 struct rte_bbdev_op_data *hard_outputs,
1128 struct rte_bbdev_op_data *soft_outputs,
1129 struct rte_bbdev_op_data *harq_inputs,
1130 struct rte_bbdev_op_data *harq_outputs,
1131 struct rte_bbdev_dec_op *ref_op)
1134 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec;
1136 for (i = 0; i < n; ++i) {
1137 if (ldpc_dec->code_block_mode == 0) {
1138 ops[i]->ldpc_dec.tb_params.ea =
1139 ldpc_dec->tb_params.ea;
1140 ops[i]->ldpc_dec.tb_params.eb =
1141 ldpc_dec->tb_params.eb;
1142 ops[i]->ldpc_dec.tb_params.c =
1143 ldpc_dec->tb_params.c;
1144 ops[i]->ldpc_dec.tb_params.cab =
1145 ldpc_dec->tb_params.cab;
1146 ops[i]->ldpc_dec.tb_params.r =
1147 ldpc_dec->tb_params.r;
1149 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e;
1152 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph;
1153 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c;
1154 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m;
1155 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler;
1156 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb;
1157 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max;
1158 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index;
1159 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags;
1160 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode;
1162 ops[i]->ldpc_dec.hard_output = hard_outputs[start_idx + i];
1163 ops[i]->ldpc_dec.input = inputs[start_idx + i];
1164 if (soft_outputs != NULL)
1165 ops[i]->ldpc_dec.soft_output =
1166 soft_outputs[start_idx + i];
1167 if (harq_inputs != NULL)
1168 ops[i]->ldpc_dec.harq_combined_input =
1169 harq_inputs[start_idx + i];
1170 if (harq_outputs != NULL)
1171 ops[i]->ldpc_dec.harq_combined_output =
1172 harq_outputs[start_idx + i];
1178 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
1179 unsigned int start_idx,
1180 struct rte_bbdev_op_data *inputs,
1181 struct rte_bbdev_op_data *outputs,
1182 struct rte_bbdev_enc_op *ref_op)
1185 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
1186 for (i = 0; i < n; ++i) {
1187 if (ldpc_enc->code_block_mode == 0) {
1188 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea;
1189 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb;
1190 ops[i]->ldpc_enc.tb_params.cab =
1191 ldpc_enc->tb_params.cab;
1192 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c;
1193 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r;
1195 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e;
1197 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph;
1198 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c;
1199 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m;
1200 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler;
1201 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb;
1202 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index;
1203 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags;
1204 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode;
1205 ops[i]->ldpc_enc.output = outputs[start_idx + i];
1206 ops[i]->ldpc_enc.input = inputs[start_idx + i];
1211 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
1212 unsigned int order_idx, const int expected_status)
1214 TEST_ASSERT(op->status == expected_status,
1215 "op_status (%d) != expected_status (%d)",
1216 op->status, expected_status);
1218 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1219 "Ordering error, expected %p, got %p",
1220 (void *)(uintptr_t)order_idx, op->opaque_data);
1222 return TEST_SUCCESS;
1226 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
1227 unsigned int order_idx, const int expected_status)
1229 TEST_ASSERT(op->status == expected_status,
1230 "op_status (%d) != expected_status (%d)",
1231 op->status, expected_status);
1233 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
1234 "Ordering error, expected %p, got %p",
1235 (void *)(uintptr_t)order_idx, op->opaque_data);
1237 return TEST_SUCCESS;
1241 validate_op_chain(struct rte_bbdev_op_data *op,
1242 struct op_data_entries *orig_op)
1245 struct rte_mbuf *m = op->data;
1246 uint8_t nb_dst_segments = orig_op->nb_segments;
1247 uint32_t total_data_size = 0;
1249 TEST_ASSERT(nb_dst_segments == m->nb_segs,
1250 "Number of segments differ in original (%u) and filled (%u) op",
1251 nb_dst_segments, m->nb_segs);
1253 /* Validate each mbuf segment length */
1254 for (i = 0; i < nb_dst_segments; ++i) {
1255 /* Apply offset to the first mbuf segment */
1256 uint16_t offset = (i == 0) ? op->offset : 0;
1257 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
1258 total_data_size += orig_op->segments[i].length;
1260 TEST_ASSERT(orig_op->segments[i].length == data_len,
1261 "Length of segment differ in original (%u) and filled (%u) op",
1262 orig_op->segments[i].length, data_len);
1263 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
1264 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
1266 "Output buffers (CB=%u) are not equal", i);
1270 /* Validate total mbuf pkt length */
1271 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset;
1272 TEST_ASSERT(total_data_size == pkt_len,
1273 "Length of data differ in original (%u) and filled (%u) op",
1274 total_data_size, pkt_len);
1276 return TEST_SUCCESS;
1280 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1281 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1285 struct op_data_entries *hard_data_orig =
1286 &test_vector.entries[DATA_HARD_OUTPUT];
1287 struct op_data_entries *soft_data_orig =
1288 &test_vector.entries[DATA_SOFT_OUTPUT];
1289 struct rte_bbdev_op_turbo_dec *ops_td;
1290 struct rte_bbdev_op_data *hard_output;
1291 struct rte_bbdev_op_data *soft_output;
1292 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
1294 for (i = 0; i < n; ++i) {
1295 ops_td = &ops[i]->turbo_dec;
1296 hard_output = &ops_td->hard_output;
1297 soft_output = &ops_td->soft_output;
1299 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1300 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1301 "Returned iter_count (%d) > expected iter_count (%d)",
1302 ops_td->iter_count, ref_td->iter_count);
1303 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1304 TEST_ASSERT_SUCCESS(ret,
1305 "Checking status and ordering for decoder failed");
1307 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1309 "Hard output buffers (CB=%u) are not equal",
1312 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
1313 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1315 "Soft output buffers (CB=%u) are not equal",
1319 return TEST_SUCCESS;
1324 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
1325 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
1329 struct op_data_entries *hard_data_orig =
1330 &test_vector.entries[DATA_HARD_OUTPUT];
1331 struct op_data_entries *soft_data_orig =
1332 &test_vector.entries[DATA_SOFT_OUTPUT];
1333 struct op_data_entries *harq_data_orig =
1334 &test_vector.entries[DATA_HARQ_OUTPUT];
1335 struct rte_bbdev_op_ldpc_dec *ops_td;
1336 struct rte_bbdev_op_data *hard_output;
1337 struct rte_bbdev_op_data *harq_output;
1338 struct rte_bbdev_op_data *soft_output;
1339 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
1341 for (i = 0; i < n; ++i) {
1342 ops_td = &ops[i]->ldpc_dec;
1343 hard_output = &ops_td->hard_output;
1344 harq_output = &ops_td->harq_combined_output;
1345 soft_output = &ops_td->soft_output;
1347 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
1348 TEST_ASSERT_SUCCESS(ret,
1349 "Checking status and ordering for decoder failed");
1350 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
1351 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
1352 "Returned iter_count (%d) > expected iter_count (%d)",
1353 ops_td->iter_count, ref_td->iter_count);
1354 /* We can ignore data when the decoding failed to converge */
1355 if ((ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) == 0)
1356 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
1358 "Hard output buffers (CB=%u) are not equal",
1361 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE)
1362 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
1364 "Soft output buffers (CB=%u) are not equal",
1366 if (ref_op->ldpc_dec.op_flags &
1367 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) {
1368 ldpc_input_llr_scaling(harq_output, 1, 8, 0);
1369 TEST_ASSERT_SUCCESS(validate_op_chain(harq_output,
1371 "HARQ output buffers (CB=%u) are not equal",
1376 return TEST_SUCCESS;
1381 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
1382 struct rte_bbdev_enc_op *ref_op)
1386 struct op_data_entries *hard_data_orig =
1387 &test_vector.entries[DATA_HARD_OUTPUT];
1389 for (i = 0; i < n; ++i) {
1390 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
1391 TEST_ASSERT_SUCCESS(ret,
1392 "Checking status and ordering for encoder failed");
1393 TEST_ASSERT_SUCCESS(validate_op_chain(
1394 &ops[i]->turbo_enc.output,
1396 "Output buffers (CB=%u) are not equal",
1400 return TEST_SUCCESS;
1404 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
1405 struct rte_bbdev_enc_op *ref_op)
1409 struct op_data_entries *hard_data_orig =
1410 &test_vector.entries[DATA_HARD_OUTPUT];
1412 for (i = 0; i < n; ++i) {
1413 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
1414 TEST_ASSERT_SUCCESS(ret,
1415 "Checking status and ordering for encoder failed");
1416 TEST_ASSERT_SUCCESS(validate_op_chain(
1417 &ops[i]->ldpc_enc.output,
1419 "Output buffers (CB=%u) are not equal",
1423 return TEST_SUCCESS;
1427 create_reference_dec_op(struct rte_bbdev_dec_op *op)
1430 struct op_data_entries *entry;
1432 op->turbo_dec = test_vector.turbo_dec;
1433 entry = &test_vector.entries[DATA_INPUT];
1434 for (i = 0; i < entry->nb_segments; ++i)
1435 op->turbo_dec.input.length +=
1436 entry->segments[i].length;
1440 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op)
1443 struct op_data_entries *entry;
1445 op->ldpc_dec = test_vector.ldpc_dec;
1446 entry = &test_vector.entries[DATA_INPUT];
1447 for (i = 0; i < entry->nb_segments; ++i)
1448 op->ldpc_dec.input.length +=
1449 entry->segments[i].length;
1450 if (test_vector.ldpc_dec.op_flags &
1451 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) {
1452 entry = &test_vector.entries[DATA_HARQ_INPUT];
1453 for (i = 0; i < entry->nb_segments; ++i)
1454 op->ldpc_dec.harq_combined_input.length +=
1455 entry->segments[i].length;
1461 create_reference_enc_op(struct rte_bbdev_enc_op *op)
1464 struct op_data_entries *entry;
1466 op->turbo_enc = test_vector.turbo_enc;
1467 entry = &test_vector.entries[DATA_INPUT];
1468 for (i = 0; i < entry->nb_segments; ++i)
1469 op->turbo_enc.input.length +=
1470 entry->segments[i].length;
1474 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op)
1477 struct op_data_entries *entry;
1479 op->ldpc_enc = test_vector.ldpc_enc;
1480 entry = &test_vector.entries[DATA_INPUT];
1481 for (i = 0; i < entry->nb_segments; ++i)
1482 op->ldpc_enc.input.length +=
1483 entry->segments[i].length;
1487 calc_dec_TB_size(struct rte_bbdev_dec_op *op)
1490 uint32_t c, r, tb_size = 0;
1492 if (op->turbo_dec.code_block_mode) {
1493 tb_size = op->turbo_dec.tb_params.k_neg;
1495 c = op->turbo_dec.tb_params.c;
1496 r = op->turbo_dec.tb_params.r;
1497 for (i = 0; i < c-r; i++)
1498 tb_size += (r < op->turbo_dec.tb_params.c_neg) ?
1499 op->turbo_dec.tb_params.k_neg :
1500 op->turbo_dec.tb_params.k_pos;
1506 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op)
1509 uint32_t c, r, tb_size = 0;
1510 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10;
1512 if (op->ldpc_dec.code_block_mode) {
1513 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler;
1515 c = op->ldpc_dec.tb_params.c;
1516 r = op->ldpc_dec.tb_params.r;
1517 for (i = 0; i < c-r; i++)
1518 tb_size += sys_cols * op->ldpc_dec.z_c
1519 - op->ldpc_dec.n_filler;
1525 calc_enc_TB_size(struct rte_bbdev_enc_op *op)
1528 uint32_t c, r, tb_size = 0;
1530 if (op->turbo_enc.code_block_mode) {
1531 tb_size = op->turbo_enc.tb_params.k_neg;
1533 c = op->turbo_enc.tb_params.c;
1534 r = op->turbo_enc.tb_params.r;
1535 for (i = 0; i < c-r; i++)
1536 tb_size += (r < op->turbo_enc.tb_params.c_neg) ?
1537 op->turbo_enc.tb_params.k_neg :
1538 op->turbo_enc.tb_params.k_pos;
1544 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op)
1547 uint32_t c, r, tb_size = 0;
1548 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10;
1550 if (op->turbo_enc.code_block_mode) {
1551 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler;
1553 c = op->turbo_enc.tb_params.c;
1554 r = op->turbo_enc.tb_params.r;
1555 for (i = 0; i < c-r; i++)
1556 tb_size += sys_cols * op->ldpc_enc.z_c
1557 - op->ldpc_enc.n_filler;
1564 init_test_op_params(struct test_op_params *op_params,
1565 enum rte_bbdev_op_type op_type, const int expected_status,
1566 const int vector_mask, struct rte_mempool *ops_mp,
1567 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
1570 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
1571 op_type == RTE_BBDEV_OP_LDPC_DEC)
1572 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
1573 &op_params->ref_dec_op, 1);
1575 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
1576 &op_params->ref_enc_op, 1);
1578 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
1580 op_params->mp = ops_mp;
1581 op_params->burst_sz = burst_sz;
1582 op_params->num_to_process = num_to_process;
1583 op_params->num_lcores = num_lcores;
1584 op_params->vector_mask = vector_mask;
1585 if (op_type == RTE_BBDEV_OP_TURBO_DEC ||
1586 op_type == RTE_BBDEV_OP_LDPC_DEC)
1587 op_params->ref_dec_op->status = expected_status;
1588 else if (op_type == RTE_BBDEV_OP_TURBO_ENC
1589 || op_type == RTE_BBDEV_OP_LDPC_ENC)
1590 op_params->ref_enc_op->status = expected_status;
1595 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
1596 struct test_op_params *op_params)
1598 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
1600 struct active_device *ad;
1601 unsigned int burst_sz = get_burst_sz();
1602 enum rte_bbdev_op_type op_type = test_vector.op_type;
1603 const struct rte_bbdev_op_cap *capabilities = NULL;
1605 ad = &active_devs[dev_id];
1607 /* Check if device supports op_type */
1608 if (!is_avail_op(ad, test_vector.op_type))
1609 return TEST_SUCCESS;
1611 struct rte_bbdev_info info;
1612 rte_bbdev_info_get(ad->dev_id, &info);
1613 socket_id = GET_SOCKET(info.socket_id);
1615 f_ret = create_mempools(ad, socket_id, op_type,
1617 if (f_ret != TEST_SUCCESS) {
1618 printf("Couldn't create mempools");
1621 if (op_type == RTE_BBDEV_OP_NONE)
1622 op_type = RTE_BBDEV_OP_TURBO_ENC;
1624 f_ret = init_test_op_params(op_params, test_vector.op_type,
1625 test_vector.expected_status,
1631 if (f_ret != TEST_SUCCESS) {
1632 printf("Couldn't init test op params");
1637 /* Find capabilities */
1638 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
1639 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
1640 if (cap->type == test_vector.op_type) {
1646 TEST_ASSERT_NOT_NULL(capabilities,
1647 "Couldn't find capabilities");
1649 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1650 create_reference_dec_op(op_params->ref_dec_op);
1651 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1652 create_reference_enc_op(op_params->ref_enc_op);
1653 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
1654 create_reference_ldpc_enc_op(op_params->ref_enc_op);
1655 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
1656 create_reference_ldpc_dec_op(op_params->ref_dec_op);
1658 for (i = 0; i < ad->nb_queues; ++i) {
1659 f_ret = fill_queue_buffers(op_params,
1661 ad->hard_out_mbuf_pool,
1662 ad->soft_out_mbuf_pool,
1663 ad->harq_in_mbuf_pool,
1664 ad->harq_out_mbuf_pool,
1667 info.drv.min_alignment,
1669 if (f_ret != TEST_SUCCESS) {
1670 printf("Couldn't init queue buffers");
1675 /* Run test case function */
1676 t_ret = test_case_func(ad, op_params);
1678 /* Free active device resources and return */
1679 free_buffers(ad, op_params);
1683 free_buffers(ad, op_params);
1687 /* Run given test function per active device per supported op type
1691 run_test_case(test_case_function *test_case_func)
1696 /* Alloc op_params */
1697 struct test_op_params *op_params = rte_zmalloc(NULL,
1698 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1699 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1700 RTE_ALIGN(sizeof(struct test_op_params),
1701 RTE_CACHE_LINE_SIZE));
1703 /* For each device run test case function */
1704 for (dev = 0; dev < nb_active_devs; ++dev)
1705 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1707 rte_free(op_params);
1713 dequeue_event_callback(uint16_t dev_id,
1714 enum rte_bbdev_event_type event, void *cb_arg,
1719 uint64_t total_time;
1720 uint16_t deq, burst_sz, num_ops;
1721 uint16_t queue_id = *(uint16_t *) ret_param;
1722 struct rte_bbdev_info info;
1724 struct thread_params *tp = cb_arg;
1726 /* Find matching thread params using queue_id */
1727 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1728 if (tp->queue_id == queue_id)
1731 if (i == MAX_QUEUES) {
1732 printf("%s: Queue_id from interrupt details was not found!\n",
1737 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1738 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1740 "Dequeue interrupt handler called for incorrect event!\n");
1744 burst_sz = rte_atomic16_read(&tp->burst_sz);
1745 num_ops = tp->op_params->num_to_process;
1747 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
1748 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
1749 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1751 rte_atomic16_read(&tp->nb_dequeued)],
1754 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1756 rte_atomic16_read(&tp->nb_dequeued)],
1759 if (deq < burst_sz) {
1761 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1763 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1767 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) {
1768 rte_atomic16_add(&tp->nb_dequeued, deq);
1772 total_time = rte_rdtsc_precise() - tp->start_time;
1774 rte_bbdev_info_get(dev_id, &info);
1778 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1779 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1780 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op,
1781 tp->op_params->vector_mask);
1782 /* get the max of iter_count for all dequeued ops */
1783 for (i = 0; i < num_ops; ++i)
1784 tp->iter_count = RTE_MAX(
1785 tp->dec_ops[i]->turbo_dec.iter_count,
1787 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
1788 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) {
1789 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1790 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op);
1791 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
1792 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
1793 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1794 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
1795 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
1796 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
1797 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1798 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
1799 tp->op_params->vector_mask);
1800 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
1804 printf("Buffers validation failed\n");
1805 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1808 switch (test_vector.op_type) {
1809 case RTE_BBDEV_OP_TURBO_DEC:
1810 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
1812 case RTE_BBDEV_OP_TURBO_ENC:
1813 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
1815 case RTE_BBDEV_OP_LDPC_DEC:
1816 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
1818 case RTE_BBDEV_OP_LDPC_ENC:
1819 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
1821 case RTE_BBDEV_OP_NONE:
1825 printf("Unknown op type: %d\n", test_vector.op_type);
1826 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1830 tp->ops_per_sec += ((double)num_ops) /
1831 ((double)total_time / (double)rte_get_tsc_hz());
1832 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
1833 ((double)total_time / (double)rte_get_tsc_hz());
1835 rte_atomic16_add(&tp->nb_dequeued, deq);
1839 throughput_intr_lcore_dec(void *arg)
1841 struct thread_params *tp = arg;
1842 unsigned int enqueued;
1843 const uint16_t queue_id = tp->queue_id;
1844 const uint16_t burst_sz = tp->op_params->burst_sz;
1845 const uint16_t num_to_process = tp->op_params->num_to_process;
1846 struct rte_bbdev_dec_op *ops[num_to_process];
1847 struct test_buffers *bufs = NULL;
1848 struct rte_bbdev_info info;
1850 uint16_t num_to_enq, enq;
1852 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1853 "BURST_SIZE should be <= %u", MAX_BURST);
1855 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1856 "Failed to enable interrupts for dev: %u, queue_id: %u",
1857 tp->dev_id, queue_id);
1859 rte_bbdev_info_get(tp->dev_id, &info);
1861 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
1862 "NUM_OPS cannot exceed %u for this device",
1863 info.drv.queue_size_lim);
1865 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1867 rte_atomic16_clear(&tp->processing_status);
1868 rte_atomic16_clear(&tp->nb_dequeued);
1870 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1873 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1875 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1877 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1878 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
1879 bufs->hard_outputs, bufs->soft_outputs,
1880 tp->op_params->ref_dec_op);
1882 /* Set counter to validate the ordering */
1883 for (j = 0; j < num_to_process; ++j)
1884 ops[j]->opaque_data = (void *)(uintptr_t)j;
1886 for (j = 0; j < TEST_REPETITIONS; ++j) {
1887 for (i = 0; i < num_to_process; ++i)
1888 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
1890 tp->start_time = rte_rdtsc_precise();
1891 for (enqueued = 0; enqueued < num_to_process;) {
1892 num_to_enq = burst_sz;
1894 if (unlikely(num_to_process - enqueued < num_to_enq))
1895 num_to_enq = num_to_process - enqueued;
1899 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1900 queue_id, &ops[enqueued],
1902 } while (unlikely(num_to_enq != enq));
1905 /* Write to thread burst_sz current number of enqueued
1906 * descriptors. It ensures that proper number of
1907 * descriptors will be dequeued in callback
1908 * function - needed for last batch in case where
1909 * the number of operations is not a multiple of
1912 rte_atomic16_set(&tp->burst_sz, num_to_enq);
1914 /* Wait until processing of previous batch is
1917 while (rte_atomic16_read(&tp->nb_dequeued) !=
1921 if (j != TEST_REPETITIONS - 1)
1922 rte_atomic16_clear(&tp->nb_dequeued);
1925 return TEST_SUCCESS;
1929 throughput_intr_lcore_enc(void *arg)
1931 struct thread_params *tp = arg;
1932 unsigned int enqueued;
1933 const uint16_t queue_id = tp->queue_id;
1934 const uint16_t burst_sz = tp->op_params->burst_sz;
1935 const uint16_t num_to_process = tp->op_params->num_to_process;
1936 struct rte_bbdev_enc_op *ops[num_to_process];
1937 struct test_buffers *bufs = NULL;
1938 struct rte_bbdev_info info;
1940 uint16_t num_to_enq, enq;
1942 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1943 "BURST_SIZE should be <= %u", MAX_BURST);
1945 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1946 "Failed to enable interrupts for dev: %u, queue_id: %u",
1947 tp->dev_id, queue_id);
1949 rte_bbdev_info_get(tp->dev_id, &info);
1951 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
1952 "NUM_OPS cannot exceed %u for this device",
1953 info.drv.queue_size_lim);
1955 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1957 rte_atomic16_clear(&tp->processing_status);
1958 rte_atomic16_clear(&tp->nb_dequeued);
1960 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1963 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1965 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
1967 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1968 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
1969 bufs->hard_outputs, tp->op_params->ref_enc_op);
1971 /* Set counter to validate the ordering */
1972 for (j = 0; j < num_to_process; ++j)
1973 ops[j]->opaque_data = (void *)(uintptr_t)j;
1975 for (j = 0; j < TEST_REPETITIONS; ++j) {
1976 for (i = 0; i < num_to_process; ++i)
1977 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
1979 tp->start_time = rte_rdtsc_precise();
1980 for (enqueued = 0; enqueued < num_to_process;) {
1981 num_to_enq = burst_sz;
1983 if (unlikely(num_to_process - enqueued < num_to_enq))
1984 num_to_enq = num_to_process - enqueued;
1988 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1989 queue_id, &ops[enqueued],
1991 } while (unlikely(enq != num_to_enq));
1994 /* Write to thread burst_sz current number of enqueued
1995 * descriptors. It ensures that proper number of
1996 * descriptors will be dequeued in callback
1997 * function - needed for last batch in case where
1998 * the number of operations is not a multiple of
2001 rte_atomic16_set(&tp->burst_sz, num_to_enq);
2003 /* Wait until processing of previous batch is
2006 while (rte_atomic16_read(&tp->nb_dequeued) !=
2010 if (j != TEST_REPETITIONS - 1)
2011 rte_atomic16_clear(&tp->nb_dequeued);
2014 return TEST_SUCCESS;
2018 throughput_pmd_lcore_dec(void *arg)
2020 struct thread_params *tp = arg;
2022 uint64_t total_time = 0, start_time;
2023 const uint16_t queue_id = tp->queue_id;
2024 const uint16_t burst_sz = tp->op_params->burst_sz;
2025 const uint16_t num_ops = tp->op_params->num_to_process;
2026 struct rte_bbdev_dec_op *ops_enq[num_ops];
2027 struct rte_bbdev_dec_op *ops_deq[num_ops];
2028 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2029 struct test_buffers *bufs = NULL;
2031 struct rte_bbdev_info info;
2032 uint16_t num_to_enq;
2034 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2035 "BURST_SIZE should be <= %u", MAX_BURST);
2037 rte_bbdev_info_get(tp->dev_id, &info);
2039 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2040 "NUM_OPS cannot exceed %u for this device",
2041 info.drv.queue_size_lim);
2043 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2045 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2048 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2049 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2051 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2052 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2053 bufs->hard_outputs, bufs->soft_outputs, ref_op);
2055 /* Set counter to validate the ordering */
2056 for (j = 0; j < num_ops; ++j)
2057 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2059 for (i = 0; i < TEST_REPETITIONS; ++i) {
2061 for (j = 0; j < num_ops; ++j)
2062 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
2064 start_time = rte_rdtsc_precise();
2066 for (enq = 0, deq = 0; enq < num_ops;) {
2067 num_to_enq = burst_sz;
2069 if (unlikely(num_ops - enq < num_to_enq))
2070 num_to_enq = num_ops - enq;
2072 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
2073 queue_id, &ops_enq[enq], num_to_enq);
2075 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
2076 queue_id, &ops_deq[deq], enq - deq);
2079 /* dequeue the remaining */
2081 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
2082 queue_id, &ops_deq[deq], enq - deq);
2085 total_time += rte_rdtsc_precise() - start_time;
2089 /* get the max of iter_count for all dequeued ops */
2090 for (i = 0; i < num_ops; ++i) {
2091 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count,
2095 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2096 ret = validate_dec_op(ops_deq, num_ops, ref_op,
2097 tp->op_params->vector_mask);
2098 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2101 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
2103 double tb_len_bits = calc_dec_TB_size(ref_op);
2105 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2106 ((double)total_time / (double)rte_get_tsc_hz());
2107 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
2108 1000000.0) / ((double)total_time /
2109 (double)rte_get_tsc_hz());
2111 return TEST_SUCCESS;
2115 throughput_pmd_lcore_ldpc_dec(void *arg)
2117 struct thread_params *tp = arg;
2119 uint64_t total_time = 0, start_time;
2120 const uint16_t queue_id = tp->queue_id;
2121 const uint16_t burst_sz = tp->op_params->burst_sz;
2122 const uint16_t num_ops = tp->op_params->num_to_process;
2123 struct rte_bbdev_dec_op *ops_enq[num_ops];
2124 struct rte_bbdev_dec_op *ops_deq[num_ops];
2125 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
2126 struct test_buffers *bufs = NULL;
2128 struct rte_bbdev_info info;
2129 uint16_t num_to_enq;
2131 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2132 "BURST_SIZE should be <= %u", MAX_BURST);
2134 rte_bbdev_info_get(tp->dev_id, &info);
2136 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2137 "NUM_OPS cannot exceed %u for this device",
2138 info.drv.queue_size_lim);
2140 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2142 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2145 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
2146 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
2148 /* For throughput tests we need to disable early termination */
2149 if (check_bit(ref_op->ldpc_dec.op_flags,
2150 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
2151 ref_op->ldpc_dec.op_flags -=
2152 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
2153 ref_op->ldpc_dec.iter_max = 6;
2154 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max;
2156 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2157 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
2158 bufs->hard_outputs, bufs->soft_outputs,
2159 bufs->harq_inputs, bufs->harq_outputs, ref_op);
2161 /* Set counter to validate the ordering */
2162 for (j = 0; j < num_ops; ++j)
2163 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2165 for (i = 0; i < TEST_REPETITIONS; ++i) {
2166 for (j = 0; j < num_ops; ++j) {
2167 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data);
2168 if (check_bit(ref_op->ldpc_dec.op_flags,
2169 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
2171 ops_enq[j]->ldpc_dec.harq_combined_output.data);
2174 start_time = rte_rdtsc_precise();
2176 for (enq = 0, deq = 0; enq < num_ops;) {
2177 num_to_enq = burst_sz;
2179 if (unlikely(num_ops - enq < num_to_enq))
2180 num_to_enq = num_ops - enq;
2182 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id,
2183 queue_id, &ops_enq[enq], num_to_enq);
2185 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
2186 queue_id, &ops_deq[deq], enq - deq);
2189 /* dequeue the remaining */
2191 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
2192 queue_id, &ops_deq[deq], enq - deq);
2195 total_time += rte_rdtsc_precise() - start_time;
2199 /* get the max of iter_count for all dequeued ops */
2200 for (i = 0; i < num_ops; ++i) {
2201 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
2205 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2206 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op,
2207 tp->op_params->vector_mask);
2208 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2211 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
2213 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op);
2215 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2216 ((double)total_time / (double)rte_get_tsc_hz());
2217 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) /
2218 1000000.0) / ((double)total_time /
2219 (double)rte_get_tsc_hz());
2221 return TEST_SUCCESS;
2225 throughput_pmd_lcore_enc(void *arg)
2227 struct thread_params *tp = arg;
2229 uint64_t total_time = 0, start_time;
2230 const uint16_t queue_id = tp->queue_id;
2231 const uint16_t burst_sz = tp->op_params->burst_sz;
2232 const uint16_t num_ops = tp->op_params->num_to_process;
2233 struct rte_bbdev_enc_op *ops_enq[num_ops];
2234 struct rte_bbdev_enc_op *ops_deq[num_ops];
2235 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2236 struct test_buffers *bufs = NULL;
2238 struct rte_bbdev_info info;
2239 uint16_t num_to_enq;
2241 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2242 "BURST_SIZE should be <= %u", MAX_BURST);
2244 rte_bbdev_info_get(tp->dev_id, &info);
2246 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2247 "NUM_OPS cannot exceed %u for this device",
2248 info.drv.queue_size_lim);
2250 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2252 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2255 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
2257 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2259 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2260 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs,
2261 bufs->hard_outputs, ref_op);
2263 /* Set counter to validate the ordering */
2264 for (j = 0; j < num_ops; ++j)
2265 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2267 for (i = 0; i < TEST_REPETITIONS; ++i) {
2269 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2270 for (j = 0; j < num_ops; ++j)
2271 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
2273 start_time = rte_rdtsc_precise();
2275 for (enq = 0, deq = 0; enq < num_ops;) {
2276 num_to_enq = burst_sz;
2278 if (unlikely(num_ops - enq < num_to_enq))
2279 num_to_enq = num_ops - enq;
2281 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
2282 queue_id, &ops_enq[enq], num_to_enq);
2284 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
2285 queue_id, &ops_deq[deq], enq - deq);
2288 /* dequeue the remaining */
2290 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id,
2291 queue_id, &ops_deq[deq], enq - deq);
2294 total_time += rte_rdtsc_precise() - start_time;
2297 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2298 ret = validate_enc_op(ops_deq, num_ops, ref_op);
2299 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2302 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
2304 double tb_len_bits = calc_enc_TB_size(ref_op);
2306 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2307 ((double)total_time / (double)rte_get_tsc_hz());
2308 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
2309 / 1000000.0) / ((double)total_time /
2310 (double)rte_get_tsc_hz());
2312 return TEST_SUCCESS;
2316 throughput_pmd_lcore_ldpc_enc(void *arg)
2318 struct thread_params *tp = arg;
2320 uint64_t total_time = 0, start_time;
2321 const uint16_t queue_id = tp->queue_id;
2322 const uint16_t burst_sz = tp->op_params->burst_sz;
2323 const uint16_t num_ops = tp->op_params->num_to_process;
2324 struct rte_bbdev_enc_op *ops_enq[num_ops];
2325 struct rte_bbdev_enc_op *ops_deq[num_ops];
2326 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
2327 struct test_buffers *bufs = NULL;
2329 struct rte_bbdev_info info;
2330 uint16_t num_to_enq;
2332 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2333 "BURST_SIZE should be <= %u", MAX_BURST);
2335 rte_bbdev_info_get(tp->dev_id, &info);
2337 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
2338 "NUM_OPS cannot exceed %u for this device",
2339 info.drv.queue_size_lim);
2341 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2343 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
2346 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq,
2348 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
2350 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2351 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs,
2352 bufs->hard_outputs, ref_op);
2354 /* Set counter to validate the ordering */
2355 for (j = 0; j < num_ops; ++j)
2356 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2358 for (i = 0; i < TEST_REPETITIONS; ++i) {
2360 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2361 for (j = 0; j < num_ops; ++j)
2362 mbuf_reset(ops_enq[j]->turbo_enc.output.data);
2364 start_time = rte_rdtsc_precise();
2366 for (enq = 0, deq = 0; enq < num_ops;) {
2367 num_to_enq = burst_sz;
2369 if (unlikely(num_ops - enq < num_to_enq))
2370 num_to_enq = num_ops - enq;
2372 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id,
2373 queue_id, &ops_enq[enq], num_to_enq);
2375 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
2376 queue_id, &ops_deq[deq], enq - deq);
2379 /* dequeue the remaining */
2381 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id,
2382 queue_id, &ops_deq[deq], enq - deq);
2385 total_time += rte_rdtsc_precise() - start_time;
2388 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2389 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op);
2390 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2393 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
2395 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op);
2397 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) /
2398 ((double)total_time / (double)rte_get_tsc_hz());
2399 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits))
2400 / 1000000.0) / ((double)total_time /
2401 (double)rte_get_tsc_hz());
2403 return TEST_SUCCESS;
2407 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores)
2409 unsigned int iter = 0;
2410 double total_mops = 0, total_mbps = 0;
2412 for (iter = 0; iter < used_cores; iter++) {
2414 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n",
2415 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
2416 t_params[iter].mbps);
2417 total_mops += t_params[iter].ops_per_sec;
2418 total_mbps += t_params[iter].mbps;
2421 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n",
2422 used_cores, total_mops, total_mbps);
2426 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores)
2428 unsigned int iter = 0;
2429 double total_mops = 0, total_mbps = 0;
2430 uint8_t iter_count = 0;
2432 for (iter = 0; iter < used_cores; iter++) {
2434 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n",
2435 t_params[iter].lcore_id, t_params[iter].ops_per_sec,
2436 t_params[iter].mbps, t_params[iter].iter_count);
2437 total_mops += t_params[iter].ops_per_sec;
2438 total_mbps += t_params[iter].mbps;
2439 iter_count = RTE_MAX(iter_count, t_params[iter].iter_count);
2442 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n",
2443 used_cores, total_mops, total_mbps, iter_count);
2447 * Test function that determines how long an enqueue + dequeue of a burst
2448 * takes on available lcores.
2451 throughput_test(struct active_device *ad,
2452 struct test_op_params *op_params)
2455 unsigned int lcore_id, used_cores = 0;
2456 struct thread_params *t_params, *tp;
2457 struct rte_bbdev_info info;
2458 lcore_function_t *throughput_function;
2459 uint16_t num_lcores;
2460 const char *op_type_str;
2462 rte_bbdev_info_get(ad->dev_id, &info);
2464 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
2465 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
2466 test_vector.op_type);
2468 printf("+ ------------------------------------------------------- +\n");
2469 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n",
2470 info.dev_name, ad->nb_queues, op_params->burst_sz,
2471 op_params->num_to_process, op_params->num_lcores,
2473 intr_enabled ? "Interrupt mode" : "PMD mode",
2474 (double)rte_get_tsc_hz() / 1000000000.0);
2476 /* Set number of lcores */
2477 num_lcores = (ad->nb_queues < (op_params->num_lcores))
2479 : op_params->num_lcores;
2481 /* Allocate memory for thread parameters structure */
2482 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
2483 RTE_CACHE_LINE_SIZE);
2484 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
2485 RTE_ALIGN(sizeof(struct thread_params) * num_lcores,
2486 RTE_CACHE_LINE_SIZE));
2489 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2490 throughput_function = throughput_intr_lcore_dec;
2491 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2492 throughput_function = throughput_intr_lcore_dec;
2493 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2494 throughput_function = throughput_intr_lcore_enc;
2495 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2496 throughput_function = throughput_intr_lcore_enc;
2498 throughput_function = throughput_intr_lcore_enc;
2500 /* Dequeue interrupt callback registration */
2501 ret = rte_bbdev_callback_register(ad->dev_id,
2502 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
2509 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
2510 throughput_function = throughput_pmd_lcore_dec;
2511 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2512 throughput_function = throughput_pmd_lcore_ldpc_dec;
2513 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
2514 throughput_function = throughput_pmd_lcore_enc;
2515 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2516 throughput_function = throughput_pmd_lcore_ldpc_enc;
2518 throughput_function = throughput_pmd_lcore_enc;
2521 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
2523 /* Master core is set at first entry */
2524 t_params[0].dev_id = ad->dev_id;
2525 t_params[0].lcore_id = rte_lcore_id();
2526 t_params[0].op_params = op_params;
2527 t_params[0].queue_id = ad->queue_ids[used_cores++];
2528 t_params[0].iter_count = 0;
2530 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2531 if (used_cores >= num_lcores)
2534 t_params[used_cores].dev_id = ad->dev_id;
2535 t_params[used_cores].lcore_id = lcore_id;
2536 t_params[used_cores].op_params = op_params;
2537 t_params[used_cores].queue_id = ad->queue_ids[used_cores];
2538 t_params[used_cores].iter_count = 0;
2540 rte_eal_remote_launch(throughput_function,
2541 &t_params[used_cores++], lcore_id);
2544 rte_atomic16_set(&op_params->sync, SYNC_START);
2545 ret = throughput_function(&t_params[0]);
2547 /* Master core is always used */
2548 for (used_cores = 1; used_cores < num_lcores; used_cores++)
2549 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
2551 /* Return if test failed */
2557 /* Print throughput if interrupts are disabled and test passed */
2558 if (!intr_enabled) {
2559 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
2560 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2561 print_dec_throughput(t_params, num_lcores);
2563 print_enc_throughput(t_params, num_lcores);
2568 /* In interrupt TC we need to wait for the interrupt callback to deqeue
2569 * all pending operations. Skip waiting for queues which reported an
2570 * error using processing_status variable.
2571 * Wait for master lcore operations.
2574 while ((rte_atomic16_read(&tp->nb_dequeued) <
2575 op_params->num_to_process) &&
2576 (rte_atomic16_read(&tp->processing_status) !=
2580 tp->ops_per_sec /= TEST_REPETITIONS;
2581 tp->mbps /= TEST_REPETITIONS;
2582 ret |= (int)rte_atomic16_read(&tp->processing_status);
2584 /* Wait for slave lcores operations */
2585 for (used_cores = 1; used_cores < num_lcores; used_cores++) {
2586 tp = &t_params[used_cores];
2588 while ((rte_atomic16_read(&tp->nb_dequeued) <
2589 op_params->num_to_process) &&
2590 (rte_atomic16_read(&tp->processing_status) !=
2594 tp->ops_per_sec /= TEST_REPETITIONS;
2595 tp->mbps /= TEST_REPETITIONS;
2596 ret |= (int)rte_atomic16_read(&tp->processing_status);
2599 /* Print throughput if test passed */
2601 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC ||
2602 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC)
2603 print_dec_throughput(t_params, num_lcores);
2604 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC ||
2605 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC)
2606 print_enc_throughput(t_params, num_lcores);
2614 latency_test_dec(struct rte_mempool *mempool,
2615 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
2616 int vector_mask, uint16_t dev_id, uint16_t queue_id,
2617 const uint16_t num_to_process, uint16_t burst_sz,
2618 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2620 int ret = TEST_SUCCESS;
2621 uint16_t i, j, dequeued;
2622 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2623 uint64_t start_time = 0, last_time = 0;
2625 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2626 uint16_t enq = 0, deq = 0;
2627 bool first_time = true;
2630 if (unlikely(num_to_process - dequeued < burst_sz))
2631 burst_sz = num_to_process - dequeued;
2633 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
2634 TEST_ASSERT_SUCCESS(ret,
2635 "rte_bbdev_dec_op_alloc_bulk() failed");
2636 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2637 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
2643 /* Set counter to validate the ordering */
2644 for (j = 0; j < burst_sz; ++j)
2645 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2647 start_time = rte_rdtsc_precise();
2649 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
2651 TEST_ASSERT(enq == burst_sz,
2652 "Error enqueueing burst, expected %u, got %u",
2657 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2658 &ops_deq[deq], burst_sz - deq);
2659 if (likely(first_time && (deq > 0))) {
2660 last_time = rte_rdtsc_precise() - start_time;
2663 } while (unlikely(burst_sz != deq));
2665 *max_time = RTE_MAX(*max_time, last_time);
2666 *min_time = RTE_MIN(*min_time, last_time);
2667 *total_time += last_time;
2669 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2670 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
2672 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2675 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2683 latency_test_ldpc_dec(struct rte_mempool *mempool,
2684 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
2685 int vector_mask, uint16_t dev_id, uint16_t queue_id,
2686 const uint16_t num_to_process, uint16_t burst_sz,
2687 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2689 int ret = TEST_SUCCESS;
2690 uint16_t i, j, dequeued;
2691 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2692 uint64_t start_time = 0, last_time = 0;
2694 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2695 uint16_t enq = 0, deq = 0;
2696 bool first_time = true;
2699 if (unlikely(num_to_process - dequeued < burst_sz))
2700 burst_sz = num_to_process - dequeued;
2702 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
2703 TEST_ASSERT_SUCCESS(ret,
2704 "rte_bbdev_dec_op_alloc_bulk() failed");
2705 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2706 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
2714 /* Set counter to validate the ordering */
2715 for (j = 0; j < burst_sz; ++j)
2716 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2718 start_time = rte_rdtsc_precise();
2720 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
2721 &ops_enq[enq], burst_sz);
2722 TEST_ASSERT(enq == burst_sz,
2723 "Error enqueueing burst, expected %u, got %u",
2728 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
2729 &ops_deq[deq], burst_sz - deq);
2730 if (likely(first_time && (deq > 0))) {
2731 last_time = rte_rdtsc_precise() - start_time;
2734 } while (unlikely(burst_sz != deq));
2736 *max_time = RTE_MAX(*max_time, last_time);
2737 *min_time = RTE_MIN(*min_time, last_time);
2738 *total_time += last_time;
2740 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2741 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op,
2743 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2746 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2754 latency_test_enc(struct rte_mempool *mempool,
2755 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
2756 uint16_t dev_id, uint16_t queue_id,
2757 const uint16_t num_to_process, uint16_t burst_sz,
2758 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2760 int ret = TEST_SUCCESS;
2761 uint16_t i, j, dequeued;
2762 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2763 uint64_t start_time = 0, last_time = 0;
2765 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2766 uint16_t enq = 0, deq = 0;
2767 bool first_time = true;
2770 if (unlikely(num_to_process - dequeued < burst_sz))
2771 burst_sz = num_to_process - dequeued;
2773 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2774 TEST_ASSERT_SUCCESS(ret,
2775 "rte_bbdev_enc_op_alloc_bulk() failed");
2776 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2777 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
2782 /* Set counter to validate the ordering */
2783 for (j = 0; j < burst_sz; ++j)
2784 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2786 start_time = rte_rdtsc_precise();
2788 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
2790 TEST_ASSERT(enq == burst_sz,
2791 "Error enqueueing burst, expected %u, got %u",
2796 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2797 &ops_deq[deq], burst_sz - deq);
2798 if (likely(first_time && (deq > 0))) {
2799 last_time += rte_rdtsc_precise() - start_time;
2802 } while (unlikely(burst_sz != deq));
2804 *max_time = RTE_MAX(*max_time, last_time);
2805 *min_time = RTE_MIN(*min_time, last_time);
2806 *total_time += last_time;
2808 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2809 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
2810 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2813 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2821 latency_test_ldpc_enc(struct rte_mempool *mempool,
2822 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
2823 uint16_t dev_id, uint16_t queue_id,
2824 const uint16_t num_to_process, uint16_t burst_sz,
2825 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
2827 int ret = TEST_SUCCESS;
2828 uint16_t i, j, dequeued;
2829 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2830 uint64_t start_time = 0, last_time = 0;
2832 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2833 uint16_t enq = 0, deq = 0;
2834 bool first_time = true;
2837 if (unlikely(num_to_process - dequeued < burst_sz))
2838 burst_sz = num_to_process - dequeued;
2840 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2842 TEST_ASSERT_SUCCESS(ret,
2843 "rte_bbdev_enc_op_alloc_bulk() failed");
2844 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2845 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
2850 /* Set counter to validate the ordering */
2851 for (j = 0; j < burst_sz; ++j)
2852 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
2854 start_time = rte_rdtsc_precise();
2856 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
2857 &ops_enq[enq], burst_sz);
2858 TEST_ASSERT(enq == burst_sz,
2859 "Error enqueueing burst, expected %u, got %u",
2864 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
2865 &ops_deq[deq], burst_sz - deq);
2866 if (likely(first_time && (deq > 0))) {
2867 last_time += rte_rdtsc_precise() - start_time;
2870 } while (unlikely(burst_sz != deq));
2872 *max_time = RTE_MAX(*max_time, last_time);
2873 *min_time = RTE_MIN(*min_time, last_time);
2874 *total_time += last_time;
2876 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
2877 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
2878 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
2881 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2889 latency_test(struct active_device *ad,
2890 struct test_op_params *op_params)
2893 uint16_t burst_sz = op_params->burst_sz;
2894 const uint16_t num_to_process = op_params->num_to_process;
2895 const enum rte_bbdev_op_type op_type = test_vector.op_type;
2896 const uint16_t queue_id = ad->queue_ids[0];
2897 struct test_buffers *bufs = NULL;
2898 struct rte_bbdev_info info;
2899 uint64_t total_time, min_time, max_time;
2900 const char *op_type_str;
2902 total_time = max_time = 0;
2903 min_time = UINT64_MAX;
2905 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2906 "BURST_SIZE should be <= %u", MAX_BURST);
2908 rte_bbdev_info_get(ad->dev_id, &info);
2909 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2911 op_type_str = rte_bbdev_op_type_str(op_type);
2912 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2914 printf("+ ------------------------------------------------------- +\n");
2915 printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
2916 info.dev_name, burst_sz, num_to_process, op_type_str);
2918 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2919 iter = latency_test_dec(op_params->mp, bufs,
2920 op_params->ref_dec_op, op_params->vector_mask,
2921 ad->dev_id, queue_id, num_to_process,
2922 burst_sz, &total_time, &min_time, &max_time);
2923 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
2924 iter = latency_test_enc(op_params->mp, bufs,
2925 op_params->ref_enc_op, ad->dev_id, queue_id,
2926 num_to_process, burst_sz, &total_time,
2927 &min_time, &max_time);
2928 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
2929 iter = latency_test_ldpc_enc(op_params->mp, bufs,
2930 op_params->ref_enc_op, ad->dev_id, queue_id,
2931 num_to_process, burst_sz, &total_time,
2932 &min_time, &max_time);
2933 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
2934 iter = latency_test_ldpc_dec(op_params->mp, bufs,
2935 op_params->ref_dec_op, op_params->vector_mask,
2936 ad->dev_id, queue_id, num_to_process,
2937 burst_sz, &total_time, &min_time, &max_time);
2939 iter = latency_test_enc(op_params->mp, bufs,
2940 op_params->ref_enc_op,
2941 ad->dev_id, queue_id,
2942 num_to_process, burst_sz, &total_time,
2943 &min_time, &max_time);
2948 printf("Operation latency:\n"
2949 "\tavg: %lg cycles, %lg us\n"
2950 "\tmin: %lg cycles, %lg us\n"
2951 "\tmax: %lg cycles, %lg us\n",
2952 (double)total_time / (double)iter,
2953 (double)(total_time * 1000000) / (double)iter /
2954 (double)rte_get_tsc_hz(), (double)min_time,
2955 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
2956 (double)max_time, (double)(max_time * 1000000) /
2957 (double)rte_get_tsc_hz());
2959 return TEST_SUCCESS;
2962 #ifdef RTE_BBDEV_OFFLOAD_COST
2964 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
2965 struct rte_bbdev_stats *stats)
2967 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
2968 struct rte_bbdev_stats *q_stats;
2970 if (queue_id >= dev->data->num_queues)
2973 q_stats = &dev->data->queues[queue_id].queue_stats;
2975 stats->enqueued_count = q_stats->enqueued_count;
2976 stats->dequeued_count = q_stats->dequeued_count;
2977 stats->enqueue_err_count = q_stats->enqueue_err_count;
2978 stats->dequeue_err_count = q_stats->dequeue_err_count;
2979 stats->acc_offload_cycles = q_stats->acc_offload_cycles;
2985 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
2986 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
2987 uint16_t queue_id, const uint16_t num_to_process,
2988 uint16_t burst_sz, struct test_time_stats *time_st)
2990 int i, dequeued, ret;
2991 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2992 uint64_t enq_start_time, deq_start_time;
2993 uint64_t enq_sw_last_time, deq_last_time;
2994 struct rte_bbdev_stats stats;
2996 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2997 uint16_t enq = 0, deq = 0;
2999 if (unlikely(num_to_process - dequeued < burst_sz))
3000 burst_sz = num_to_process - dequeued;
3002 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3003 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3004 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
3010 /* Start time meas for enqueue function offload latency */
3011 enq_start_time = rte_rdtsc_precise();
3013 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
3014 &ops_enq[enq], burst_sz - enq);
3015 } while (unlikely(burst_sz != enq));
3017 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3018 TEST_ASSERT_SUCCESS(ret,
3019 "Failed to get stats for queue (%u) of device (%u)",
3022 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3023 stats.acc_offload_cycles;
3024 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3026 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3028 time_st->enq_sw_total_time += enq_sw_last_time;
3030 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3031 stats.acc_offload_cycles);
3032 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3033 stats.acc_offload_cycles);
3034 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3036 /* give time for device to process ops */
3039 /* Start time meas for dequeue function offload latency */
3040 deq_start_time = rte_rdtsc_precise();
3041 /* Dequeue one operation */
3043 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3045 } while (unlikely(deq != 1));
3047 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3048 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3050 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3052 time_st->deq_total_time += deq_last_time;
3054 /* Dequeue remaining operations if needed*/
3055 while (burst_sz != deq)
3056 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3057 &ops_deq[deq], burst_sz - deq);
3059 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3067 offload_latency_test_ldpc_dec(struct rte_mempool *mempool,
3068 struct test_buffers *bufs,
3069 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
3070 uint16_t queue_id, const uint16_t num_to_process,
3071 uint16_t burst_sz, struct test_time_stats *time_st)
3073 int i, dequeued, ret;
3074 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3075 uint64_t enq_start_time, deq_start_time;
3076 uint64_t enq_sw_last_time, deq_last_time;
3077 struct rte_bbdev_stats stats;
3079 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3080 uint16_t enq = 0, deq = 0;
3082 if (unlikely(num_to_process - dequeued < burst_sz))
3083 burst_sz = num_to_process - dequeued;
3085 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
3086 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3087 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued,
3095 /* Start time meas for enqueue function offload latency */
3096 enq_start_time = rte_rdtsc_precise();
3098 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id,
3099 &ops_enq[enq], burst_sz - enq);
3100 } while (unlikely(burst_sz != enq));
3102 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3103 TEST_ASSERT_SUCCESS(ret,
3104 "Failed to get stats for queue (%u) of device (%u)",
3107 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3108 stats.acc_offload_cycles;
3109 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3111 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3113 time_st->enq_sw_total_time += enq_sw_last_time;
3115 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3116 stats.acc_offload_cycles);
3117 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3118 stats.acc_offload_cycles);
3119 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3121 /* give time for device to process ops */
3124 /* Start time meas for dequeue function offload latency */
3125 deq_start_time = rte_rdtsc_precise();
3126 /* Dequeue one operation */
3128 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id,
3130 } while (unlikely(deq != 1));
3132 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3133 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3135 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3137 time_st->deq_total_time += deq_last_time;
3139 /* Dequeue remaining operations if needed*/
3140 while (burst_sz != deq)
3141 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
3142 &ops_deq[deq], burst_sz - deq);
3144 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
3152 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
3153 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
3154 uint16_t queue_id, const uint16_t num_to_process,
3155 uint16_t burst_sz, struct test_time_stats *time_st)
3157 int i, dequeued, ret;
3158 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3159 uint64_t enq_start_time, deq_start_time;
3160 uint64_t enq_sw_last_time, deq_last_time;
3161 struct rte_bbdev_stats stats;
3163 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3164 uint16_t enq = 0, deq = 0;
3166 if (unlikely(num_to_process - dequeued < burst_sz))
3167 burst_sz = num_to_process - dequeued;
3169 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3170 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
3171 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3172 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
3177 /* Start time meas for enqueue function offload latency */
3178 enq_start_time = rte_rdtsc_precise();
3180 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
3181 &ops_enq[enq], burst_sz - enq);
3182 } while (unlikely(burst_sz != enq));
3184 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3185 TEST_ASSERT_SUCCESS(ret,
3186 "Failed to get stats for queue (%u) of device (%u)",
3189 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3190 stats.acc_offload_cycles;
3191 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3193 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3195 time_st->enq_sw_total_time += enq_sw_last_time;
3197 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3198 stats.acc_offload_cycles);
3199 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3200 stats.acc_offload_cycles);
3201 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3203 /* give time for device to process ops */
3206 /* Start time meas for dequeue function offload latency */
3207 deq_start_time = rte_rdtsc_precise();
3208 /* Dequeue one operation */
3210 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
3212 } while (unlikely(deq != 1));
3214 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3215 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3217 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3219 time_st->deq_total_time += deq_last_time;
3221 while (burst_sz != deq)
3222 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
3223 &ops_deq[deq], burst_sz - deq);
3225 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
3233 offload_latency_test_ldpc_enc(struct rte_mempool *mempool,
3234 struct test_buffers *bufs,
3235 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
3236 uint16_t queue_id, const uint16_t num_to_process,
3237 uint16_t burst_sz, struct test_time_stats *time_st)
3239 int i, dequeued, ret;
3240 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
3241 uint64_t enq_start_time, deq_start_time;
3242 uint64_t enq_sw_last_time, deq_last_time;
3243 struct rte_bbdev_stats stats;
3245 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
3246 uint16_t enq = 0, deq = 0;
3248 if (unlikely(num_to_process - dequeued < burst_sz))
3249 burst_sz = num_to_process - dequeued;
3251 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
3252 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
3253 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
3254 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued,
3259 /* Start time meas for enqueue function offload latency */
3260 enq_start_time = rte_rdtsc_precise();
3262 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id,
3263 &ops_enq[enq], burst_sz - enq);
3264 } while (unlikely(burst_sz != enq));
3266 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
3267 TEST_ASSERT_SUCCESS(ret,
3268 "Failed to get stats for queue (%u) of device (%u)",
3271 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
3272 stats.acc_offload_cycles;
3273 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
3275 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
3277 time_st->enq_sw_total_time += enq_sw_last_time;
3279 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time,
3280 stats.acc_offload_cycles);
3281 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time,
3282 stats.acc_offload_cycles);
3283 time_st->enq_acc_total_time += stats.acc_offload_cycles;
3285 /* give time for device to process ops */
3288 /* Start time meas for dequeue function offload latency */
3289 deq_start_time = rte_rdtsc_precise();
3290 /* Dequeue one operation */
3292 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
3294 } while (unlikely(deq != 1));
3296 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3297 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
3299 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
3301 time_st->deq_total_time += deq_last_time;
3303 while (burst_sz != deq)
3304 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id,
3305 &ops_deq[deq], burst_sz - deq);
3307 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
3316 offload_cost_test(struct active_device *ad,
3317 struct test_op_params *op_params)
3319 #ifndef RTE_BBDEV_OFFLOAD_COST
3321 RTE_SET_USED(op_params);
3322 printf("Offload latency test is disabled.\n");
3323 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
3324 return TEST_SKIPPED;
3327 uint16_t burst_sz = op_params->burst_sz;
3328 const uint16_t num_to_process = op_params->num_to_process;
3329 const enum rte_bbdev_op_type op_type = test_vector.op_type;
3330 const uint16_t queue_id = ad->queue_ids[0];
3331 struct test_buffers *bufs = NULL;
3332 struct rte_bbdev_info info;
3333 const char *op_type_str;
3334 struct test_time_stats time_st;
3336 memset(&time_st, 0, sizeof(struct test_time_stats));
3337 time_st.enq_sw_min_time = UINT64_MAX;
3338 time_st.enq_acc_min_time = UINT64_MAX;
3339 time_st.deq_min_time = UINT64_MAX;
3341 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3342 "BURST_SIZE should be <= %u", MAX_BURST);
3344 rte_bbdev_info_get(ad->dev_id, &info);
3345 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
3347 op_type_str = rte_bbdev_op_type_str(op_type);
3348 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
3350 printf("+ ------------------------------------------------------- +\n");
3351 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
3352 info.dev_name, burst_sz, num_to_process, op_type_str);
3354 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
3355 iter = offload_latency_test_dec(op_params->mp, bufs,
3356 op_params->ref_dec_op, ad->dev_id, queue_id,
3357 num_to_process, burst_sz, &time_st);
3358 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
3359 iter = offload_latency_test_enc(op_params->mp, bufs,
3360 op_params->ref_enc_op, ad->dev_id, queue_id,
3361 num_to_process, burst_sz, &time_st);
3362 else if (op_type == RTE_BBDEV_OP_LDPC_ENC)
3363 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs,
3364 op_params->ref_enc_op, ad->dev_id, queue_id,
3365 num_to_process, burst_sz, &time_st);
3366 else if (op_type == RTE_BBDEV_OP_LDPC_DEC)
3367 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs,
3368 op_params->ref_dec_op, ad->dev_id, queue_id,
3369 num_to_process, burst_sz, &time_st);
3371 iter = offload_latency_test_enc(op_params->mp, bufs,
3372 op_params->ref_enc_op, ad->dev_id, queue_id,
3373 num_to_process, burst_sz, &time_st);
3378 printf("Enqueue driver offload cost latency:\n"
3379 "\tavg: %lg cycles, %lg us\n"
3380 "\tmin: %lg cycles, %lg us\n"
3381 "\tmax: %lg cycles, %lg us\n"
3382 "Enqueue accelerator offload cost latency:\n"
3383 "\tavg: %lg cycles, %lg us\n"
3384 "\tmin: %lg cycles, %lg us\n"
3385 "\tmax: %lg cycles, %lg us\n",
3386 (double)time_st.enq_sw_total_time / (double)iter,
3387 (double)(time_st.enq_sw_total_time * 1000000) /
3388 (double)iter / (double)rte_get_tsc_hz(),
3389 (double)time_st.enq_sw_min_time,
3390 (double)(time_st.enq_sw_min_time * 1000000) /
3391 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
3392 (double)(time_st.enq_sw_max_time * 1000000) /
3393 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time /
3395 (double)(time_st.enq_acc_total_time * 1000000) /
3396 (double)iter / (double)rte_get_tsc_hz(),
3397 (double)time_st.enq_acc_min_time,
3398 (double)(time_st.enq_acc_min_time * 1000000) /
3399 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time,
3400 (double)(time_st.enq_acc_max_time * 1000000) /
3403 printf("Dequeue offload cost latency - one op:\n"
3404 "\tavg: %lg cycles, %lg us\n"
3405 "\tmin: %lg cycles, %lg us\n"
3406 "\tmax: %lg cycles, %lg us\n",
3407 (double)time_st.deq_total_time / (double)iter,
3408 (double)(time_st.deq_total_time * 1000000) /
3409 (double)iter / (double)rte_get_tsc_hz(),
3410 (double)time_st.deq_min_time,
3411 (double)(time_st.deq_min_time * 1000000) /
3412 rte_get_tsc_hz(), (double)time_st.deq_max_time,
3413 (double)(time_st.deq_max_time * 1000000) /
3416 return TEST_SUCCESS;
3420 #ifdef RTE_BBDEV_OFFLOAD_COST
3422 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
3423 const uint16_t num_to_process, uint16_t burst_sz,
3424 uint64_t *deq_total_time, uint64_t *deq_min_time,
3425 uint64_t *deq_max_time)
3428 struct rte_bbdev_dec_op *ops[MAX_BURST];
3429 uint64_t deq_start_time, deq_last_time;
3431 /* Test deq offload latency from an empty queue */
3433 for (i = 0, deq_total = 0; deq_total < num_to_process;
3434 ++i, deq_total += burst_sz) {
3435 deq_start_time = rte_rdtsc_precise();
3437 if (unlikely(num_to_process - deq_total < burst_sz))
3438 burst_sz = num_to_process - deq_total;
3439 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
3441 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3442 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
3443 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
3444 *deq_total_time += deq_last_time;
3451 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
3452 const uint16_t num_to_process, uint16_t burst_sz,
3453 uint64_t *deq_total_time, uint64_t *deq_min_time,
3454 uint64_t *deq_max_time)
3457 struct rte_bbdev_enc_op *ops[MAX_BURST];
3458 uint64_t deq_start_time, deq_last_time;
3460 /* Test deq offload latency from an empty queue */
3461 for (i = 0, deq_total = 0; deq_total < num_to_process;
3462 ++i, deq_total += burst_sz) {
3463 deq_start_time = rte_rdtsc_precise();
3465 if (unlikely(num_to_process - deq_total < burst_sz))
3466 burst_sz = num_to_process - deq_total;
3467 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
3469 deq_last_time = rte_rdtsc_precise() - deq_start_time;
3470 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
3471 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
3472 *deq_total_time += deq_last_time;
3480 offload_latency_empty_q_test(struct active_device *ad,
3481 struct test_op_params *op_params)
3483 #ifndef RTE_BBDEV_OFFLOAD_COST
3485 RTE_SET_USED(op_params);
3486 printf("Offload latency empty dequeue test is disabled.\n");
3487 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
3488 return TEST_SKIPPED;
3491 uint64_t deq_total_time, deq_min_time, deq_max_time;
3492 uint16_t burst_sz = op_params->burst_sz;
3493 const uint16_t num_to_process = op_params->num_to_process;
3494 const enum rte_bbdev_op_type op_type = test_vector.op_type;
3495 const uint16_t queue_id = ad->queue_ids[0];
3496 struct rte_bbdev_info info;
3497 const char *op_type_str;
3499 deq_total_time = deq_max_time = 0;
3500 deq_min_time = UINT64_MAX;
3502 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
3503 "BURST_SIZE should be <= %u", MAX_BURST);
3505 rte_bbdev_info_get(ad->dev_id, &info);
3507 op_type_str = rte_bbdev_op_type_str(op_type);
3508 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
3510 printf("+ ------------------------------------------------------- +\n");
3511 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n",
3512 info.dev_name, burst_sz, num_to_process, op_type_str);
3514 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
3515 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
3516 num_to_process, burst_sz, &deq_total_time,
3517 &deq_min_time, &deq_max_time);
3519 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
3520 num_to_process, burst_sz, &deq_total_time,
3521 &deq_min_time, &deq_max_time);
3526 printf("Empty dequeue offload:\n"
3527 "\tavg: %lg cycles, %lg us\n"
3528 "\tmin: %lg cycles, %lg us\n"
3529 "\tmax: %lg cycles, %lg us\n",
3530 (double)deq_total_time / (double)iter,
3531 (double)(deq_total_time * 1000000) / (double)iter /
3532 (double)rte_get_tsc_hz(), (double)deq_min_time,
3533 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
3534 (double)deq_max_time, (double)(deq_max_time * 1000000) /
3537 return TEST_SUCCESS;
3544 return run_test_case(throughput_test);
3548 offload_cost_tc(void)
3550 return run_test_case(offload_cost_test);
3554 offload_latency_empty_q_tc(void)
3556 return run_test_case(offload_latency_empty_q_test);
3562 return run_test_case(latency_test);
3568 return run_test_case(throughput_test);
3571 static struct unit_test_suite bbdev_throughput_testsuite = {
3572 .suite_name = "BBdev Throughput Tests",
3573 .setup = testsuite_setup,
3574 .teardown = testsuite_teardown,
3575 .unit_test_cases = {
3576 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
3577 TEST_CASES_END() /**< NULL terminate unit test array */
3581 static struct unit_test_suite bbdev_validation_testsuite = {
3582 .suite_name = "BBdev Validation Tests",
3583 .setup = testsuite_setup,
3584 .teardown = testsuite_teardown,
3585 .unit_test_cases = {
3586 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
3587 TEST_CASES_END() /**< NULL terminate unit test array */
3591 static struct unit_test_suite bbdev_latency_testsuite = {
3592 .suite_name = "BBdev Latency Tests",
3593 .setup = testsuite_setup,
3594 .teardown = testsuite_teardown,
3595 .unit_test_cases = {
3596 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
3597 TEST_CASES_END() /**< NULL terminate unit test array */
3601 static struct unit_test_suite bbdev_offload_cost_testsuite = {
3602 .suite_name = "BBdev Offload Cost Tests",
3603 .setup = testsuite_setup,
3604 .teardown = testsuite_teardown,
3605 .unit_test_cases = {
3606 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
3607 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
3608 TEST_CASES_END() /**< NULL terminate unit test array */
3612 static struct unit_test_suite bbdev_interrupt_testsuite = {
3613 .suite_name = "BBdev Interrupt Tests",
3614 .setup = interrupt_testsuite_setup,
3615 .teardown = testsuite_teardown,
3616 .unit_test_cases = {
3617 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
3618 TEST_CASES_END() /**< NULL terminate unit test array */
3622 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
3623 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
3624 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
3625 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
3626 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);