1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_launch.h>
13 #include <rte_bbdev.h>
14 #include <rte_cycles.h>
15 #include <rte_lcore.h>
16 #include <rte_malloc.h>
17 #include <rte_random.h>
18 #include <rte_hexdump.h>
21 #include "test_bbdev_vector.h"
23 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
25 #define MAX_QUEUES RTE_MAX_LCORE
27 #define OPS_CACHE_SIZE 256U
28 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
33 #define INVALID_QUEUE_ID -1
35 static struct test_bbdev_vector test_vector;
37 /* Switch between PMD and Interrupt for throughput TC */
38 static bool intr_enabled;
40 /* Represents tested active devices */
41 static struct active_device {
42 const char *driver_name;
44 uint16_t supported_ops;
45 uint16_t queue_ids[MAX_QUEUES];
47 struct rte_mempool *ops_mempool;
48 struct rte_mempool *in_mbuf_pool;
49 struct rte_mempool *hard_out_mbuf_pool;
50 struct rte_mempool *soft_out_mbuf_pool;
51 } active_devs[RTE_BBDEV_MAX_DEVS];
53 static uint8_t nb_active_devs;
55 /* Data buffers used by BBDEV ops */
57 struct rte_bbdev_op_data *inputs;
58 struct rte_bbdev_op_data *hard_outputs;
59 struct rte_bbdev_op_data *soft_outputs;
62 /* Operation parameters specific for given test case */
63 struct test_op_params {
64 struct rte_mempool *mp;
65 struct rte_bbdev_dec_op *ref_dec_op;
66 struct rte_bbdev_enc_op *ref_enc_op;
68 uint16_t num_to_process;
72 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
75 /* Contains per lcore params */
76 struct thread_params {
82 rte_atomic16_t nb_dequeued;
83 rte_atomic16_t processing_status;
84 struct test_op_params *op_params;
87 #ifdef RTE_BBDEV_OFFLOAD_COST
88 /* Stores time statistics */
89 struct test_time_stats {
90 /* Stores software enqueue total working time */
91 uint64_t enq_sw_tot_time;
92 /* Stores minimum value of software enqueue working time */
93 uint64_t enq_sw_min_time;
94 /* Stores maximum value of software enqueue working time */
95 uint64_t enq_sw_max_time;
96 /* Stores turbo enqueue total working time */
97 uint64_t enq_tur_tot_time;
98 /* Stores minimum value of turbo enqueue working time */
99 uint64_t enq_tur_min_time;
100 /* Stores maximum value of turbo enqueue working time */
101 uint64_t enq_tur_max_time;
102 /* Stores dequeue total working time */
103 uint64_t deq_tot_time;
104 /* Stores minimum value of dequeue working time */
105 uint64_t deq_min_time;
106 /* Stores maximum value of dequeue working time */
107 uint64_t deq_max_time;
111 typedef int (test_case_function)(struct active_device *ad,
112 struct test_op_params *op_params);
115 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
117 ad->supported_ops |= (1 << op_type);
121 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
123 return ad->supported_ops & (1 << op_type);
127 flags_match(uint32_t flags_req, uint32_t flags_present)
129 return (flags_req & flags_present) == flags_req;
133 clear_soft_out_cap(uint32_t *op_flags)
135 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
136 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
137 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
141 check_dev_cap(const struct rte_bbdev_info *dev_info)
144 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs;
145 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
147 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
148 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
149 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
151 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
152 if (op_cap->type != test_vector.op_type)
155 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
156 const struct rte_bbdev_op_cap_turbo_dec *cap =
157 &op_cap->cap.turbo_dec;
158 /* Ignore lack of soft output capability, just skip
159 * checking if soft output is valid.
161 if ((test_vector.turbo_dec.op_flags &
162 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
163 !(cap->capability_flags &
164 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
166 "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
169 &test_vector.turbo_dec.op_flags);
172 if (!flags_match(test_vector.turbo_dec.op_flags,
173 cap->capability_flags))
175 if (nb_inputs > cap->num_buffers_src) {
176 printf("Too many inputs defined: %u, max: %u\n",
177 nb_inputs, cap->num_buffers_src);
180 if (nb_soft_outputs > cap->num_buffers_soft_out &&
181 (test_vector.turbo_dec.op_flags &
182 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
184 "Too many soft outputs defined: %u, max: %u\n",
186 cap->num_buffers_soft_out);
189 if (nb_hard_outputs > cap->num_buffers_hard_out) {
191 "Too many hard outputs defined: %u, max: %u\n",
193 cap->num_buffers_hard_out);
196 if (intr_enabled && !(cap->capability_flags &
197 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
199 "Dequeue interrupts are not supported!\n");
204 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
205 const struct rte_bbdev_op_cap_turbo_enc *cap =
206 &op_cap->cap.turbo_enc;
208 if (!flags_match(test_vector.turbo_enc.op_flags,
209 cap->capability_flags))
211 if (nb_inputs > cap->num_buffers_src) {
212 printf("Too many inputs defined: %u, max: %u\n",
213 nb_inputs, cap->num_buffers_src);
216 if (nb_hard_outputs > cap->num_buffers_dst) {
218 "Too many hard outputs defined: %u, max: %u\n",
219 nb_hard_outputs, cap->num_buffers_src);
222 if (intr_enabled && !(cap->capability_flags &
223 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
225 "Dequeue interrupts are not supported!\n");
233 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
234 return TEST_SUCCESS; /* Special case for NULL device */
239 /* calculates optimal mempool size not smaller than the val */
241 optimal_mempool_size(unsigned int val)
243 return rte_align32pow2(val + 1) - 1;
246 /* allocates mbuf mempool for inputs and outputs */
247 static struct rte_mempool *
248 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
249 int socket_id, unsigned int mbuf_pool_size,
250 const char *op_type_str)
253 uint32_t max_seg_sz = 0;
254 char pool_name[RTE_MEMPOOL_NAMESIZE];
256 /* find max input segment size */
257 for (i = 0; i < entries->nb_segments; ++i)
258 if (entries->segments[i].length > max_seg_sz)
259 max_seg_sz = entries->segments[i].length;
261 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
263 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
264 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
265 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
269 create_mempools(struct active_device *ad, int socket_id,
270 enum rte_bbdev_op_type op_type, uint16_t num_ops)
272 struct rte_mempool *mp;
273 unsigned int ops_pool_size, mbuf_pool_size = 0;
274 char pool_name[RTE_MEMPOOL_NAMESIZE];
275 const char *op_type_str;
277 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
278 struct op_data_entries *hard_out =
279 &test_vector.entries[DATA_HARD_OUTPUT];
280 struct op_data_entries *soft_out =
281 &test_vector.entries[DATA_SOFT_OUTPUT];
283 /* allocate ops mempool */
284 ops_pool_size = optimal_mempool_size(RTE_MAX(
285 /* Ops used plus 1 reference op */
286 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
287 /* Minimal cache size plus 1 reference op */
288 (unsigned int)(1.5 * rte_lcore_count() *
289 OPS_CACHE_SIZE + 1)),
292 op_type_str = rte_bbdev_op_type_str(op_type);
293 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
295 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
297 mp = rte_bbdev_op_pool_create(pool_name, op_type,
298 ops_pool_size, OPS_CACHE_SIZE, socket_id);
299 TEST_ASSERT_NOT_NULL(mp,
300 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
304 ad->ops_mempool = mp;
307 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
308 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
309 TEST_ASSERT_NOT_NULL(mp,
310 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
314 ad->in_mbuf_pool = mp;
317 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
318 hard_out->nb_segments);
319 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
321 TEST_ASSERT_NOT_NULL(mp,
322 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
326 ad->hard_out_mbuf_pool = mp;
328 if (soft_out->nb_segments == 0)
332 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
333 soft_out->nb_segments);
334 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size,
336 TEST_ASSERT_NOT_NULL(mp,
337 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
341 ad->soft_out_mbuf_pool = mp;
347 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
348 struct test_bbdev_vector *vector)
351 unsigned int queue_id;
352 struct rte_bbdev_queue_conf qconf;
353 struct active_device *ad = &active_devs[nb_active_devs];
354 unsigned int nb_queues;
355 enum rte_bbdev_op_type op_type = vector->op_type;
357 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
359 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
361 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
362 dev_id, nb_queues, info->socket_id, ret);
366 /* configure interrupts if needed */
368 ret = rte_bbdev_intr_enable(dev_id);
370 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
376 /* setup device queues */
377 qconf.socket = info->socket_id;
378 qconf.queue_size = info->drv.default_queue_conf.queue_size;
380 qconf.deferred_start = 0;
381 qconf.op_type = op_type;
383 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
384 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
387 "Allocated all queues (id=%u) at prio%u on dev%u\n",
388 queue_id, qconf.priority, dev_id);
390 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
394 printf("All queues on dev %u allocated: %u\n",
398 ad->queue_ids[queue_id] = queue_id;
400 TEST_ASSERT(queue_id != 0,
401 "ERROR Failed to configure any queues on dev %u",
403 ad->nb_queues = queue_id;
405 set_avail_op(ad, op_type);
411 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
412 struct test_bbdev_vector *vector)
416 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
417 active_devs[nb_active_devs].dev_id = dev_id;
419 ret = add_bbdev_dev(dev_id, info, vector);
420 if (ret == TEST_SUCCESS)
426 populate_active_devices(void)
430 uint8_t nb_devs_added = 0;
431 struct rte_bbdev_info info;
433 RTE_BBDEV_FOREACH(dev_id) {
434 rte_bbdev_info_get(dev_id, &info);
436 if (check_dev_cap(&info)) {
438 "Device %d (%s) does not support specified capabilities\n",
439 dev_id, info.dev_name);
443 ret = add_active_device(dev_id, &info, &test_vector);
445 printf("Adding active bbdev %s skipped\n",
452 return nb_devs_added;
456 read_test_vector(void)
460 memset(&test_vector, 0, sizeof(test_vector));
461 printf("Test vector file = %s\n", get_vector_filename());
462 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
463 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
464 get_vector_filename());
470 testsuite_setup(void)
472 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
474 if (populate_active_devices() == 0) {
475 printf("No suitable devices found!\n");
483 interrupt_testsuite_setup(void)
485 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
487 /* Enable interrupts */
490 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
491 if (populate_active_devices() == 0 ||
492 test_vector.op_type == RTE_BBDEV_OP_NONE) {
493 intr_enabled = false;
494 printf("No suitable devices found!\n");
502 testsuite_teardown(void)
506 /* Unconfigure devices */
507 RTE_BBDEV_FOREACH(dev_id)
508 rte_bbdev_close(dev_id);
510 /* Clear active devices structs. */
511 memset(active_devs, 0, sizeof(active_devs));
520 for (i = 0; i < nb_active_devs; i++) {
521 dev_id = active_devs[i].dev_id;
522 /* reset bbdev stats */
523 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
524 "Failed to reset stats of bbdev %u", dev_id);
525 /* start the device */
526 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
527 "Failed to start bbdev %u", dev_id);
537 struct rte_bbdev_stats stats;
539 for (i = 0; i < nb_active_devs; i++) {
540 dev_id = active_devs[i].dev_id;
541 /* read stats and print */
542 rte_bbdev_stats_get(dev_id, &stats);
543 /* Stop the device */
544 rte_bbdev_stop(dev_id);
549 init_op_data_objs(struct rte_bbdev_op_data *bufs,
550 struct op_data_entries *ref_entries,
551 struct rte_mempool *mbuf_pool, const uint16_t n,
552 enum op_data_type op_type, uint16_t min_alignment)
557 for (i = 0; i < n; ++i) {
559 struct op_data_buf *seg = &ref_entries->segments[0];
560 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
561 TEST_ASSERT_NOT_NULL(m_head,
562 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
563 op_type, n * ref_entries->nb_segments,
566 bufs[i].data = m_head;
570 if (op_type == DATA_INPUT) {
571 data = rte_pktmbuf_append(m_head, seg->length);
572 TEST_ASSERT_NOT_NULL(data,
573 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
574 seg->length, op_type);
576 TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
577 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
578 data, min_alignment);
579 rte_memcpy(data, seg->addr, seg->length);
580 bufs[i].length += seg->length;
583 for (j = 1; j < ref_entries->nb_segments; ++j) {
584 struct rte_mbuf *m_tail =
585 rte_pktmbuf_alloc(mbuf_pool);
586 TEST_ASSERT_NOT_NULL(m_tail,
587 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
589 n * ref_entries->nb_segments,
593 data = rte_pktmbuf_append(m_tail, seg->length);
594 TEST_ASSERT_NOT_NULL(data,
595 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
596 seg->length, op_type);
598 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
600 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
601 data, min_alignment);
602 rte_memcpy(data, seg->addr, seg->length);
603 bufs[i].length += seg->length;
605 ret = rte_pktmbuf_chain(m_head, m_tail);
606 TEST_ASSERT_SUCCESS(ret,
607 "Couldn't chain mbufs from %d data type mbuf pool",
617 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
622 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
623 if (*buffers == NULL) {
624 printf("WARNING: Failed to allocate op_data on socket %d\n",
626 /* try to allocate memory on other detected sockets */
627 for (i = 0; i < socket; i++) {
628 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
629 if (*buffers != NULL)
634 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
638 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,
639 uint16_t n, int8_t max_llr_modulus)
641 uint16_t i, byte_idx;
643 for (i = 0; i < n; ++i) {
644 struct rte_mbuf *m = input_ops[i].data;
646 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,
647 input_ops[i].offset);
648 for (byte_idx = 0; byte_idx < input_ops[i].length;
650 llr[byte_idx] = round((double)max_llr_modulus *
651 llr[byte_idx] / INT8_MAX);
659 fill_queue_buffers(struct test_op_params *op_params,
660 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
661 struct rte_mempool *soft_out_mp, uint16_t queue_id,
662 const struct rte_bbdev_op_cap *capabilities,
663 uint16_t min_alignment, const int socket_id)
666 enum op_data_type type;
667 const uint16_t n = op_params->num_to_process;
669 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
675 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
676 &op_params->q_bufs[socket_id][queue_id].inputs,
677 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
678 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
681 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
682 struct op_data_entries *ref_entries =
683 &test_vector.entries[type];
684 if (ref_entries->nb_segments == 0)
687 ret = allocate_buffers_on_socket(queue_ops[type],
688 n * sizeof(struct rte_bbdev_op_data),
690 TEST_ASSERT_SUCCESS(ret,
691 "Couldn't allocate memory for rte_bbdev_op_data structs");
693 ret = init_op_data_objs(*queue_ops[type], ref_entries,
694 mbuf_pools[type], n, type, min_alignment);
695 TEST_ASSERT_SUCCESS(ret,
696 "Couldn't init rte_bbdev_op_data structs");
699 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
700 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,
701 capabilities->cap.turbo_dec.max_llr_modulus);
707 free_buffers(struct active_device *ad, struct test_op_params *op_params)
711 rte_mempool_free(ad->ops_mempool);
712 rte_mempool_free(ad->in_mbuf_pool);
713 rte_mempool_free(ad->hard_out_mbuf_pool);
714 rte_mempool_free(ad->soft_out_mbuf_pool);
716 for (i = 0; i < rte_lcore_count(); ++i) {
717 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
718 rte_free(op_params->q_bufs[j][i].inputs);
719 rte_free(op_params->q_bufs[j][i].hard_outputs);
720 rte_free(op_params->q_bufs[j][i].soft_outputs);
726 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
727 unsigned int start_idx,
728 struct rte_bbdev_op_data *inputs,
729 struct rte_bbdev_op_data *hard_outputs,
730 struct rte_bbdev_op_data *soft_outputs,
731 struct rte_bbdev_dec_op *ref_op)
734 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
736 for (i = 0; i < n; ++i) {
737 if (turbo_dec->code_block_mode == 0) {
738 ops[i]->turbo_dec.tb_params.ea =
739 turbo_dec->tb_params.ea;
740 ops[i]->turbo_dec.tb_params.eb =
741 turbo_dec->tb_params.eb;
742 ops[i]->turbo_dec.tb_params.k_pos =
743 turbo_dec->tb_params.k_pos;
744 ops[i]->turbo_dec.tb_params.k_neg =
745 turbo_dec->tb_params.k_neg;
746 ops[i]->turbo_dec.tb_params.c =
747 turbo_dec->tb_params.c;
748 ops[i]->turbo_dec.tb_params.c_neg =
749 turbo_dec->tb_params.c_neg;
750 ops[i]->turbo_dec.tb_params.cab =
751 turbo_dec->tb_params.cab;
753 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
754 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
757 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
758 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
759 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
760 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
761 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
762 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
763 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
765 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
766 ops[i]->turbo_dec.input = inputs[start_idx + i];
767 if (soft_outputs != NULL)
768 ops[i]->turbo_dec.soft_output =
769 soft_outputs[start_idx + i];
774 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
775 unsigned int start_idx,
776 struct rte_bbdev_op_data *inputs,
777 struct rte_bbdev_op_data *outputs,
778 struct rte_bbdev_enc_op *ref_op)
781 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
782 for (i = 0; i < n; ++i) {
783 if (turbo_enc->code_block_mode == 0) {
784 ops[i]->turbo_enc.tb_params.ea =
785 turbo_enc->tb_params.ea;
786 ops[i]->turbo_enc.tb_params.eb =
787 turbo_enc->tb_params.eb;
788 ops[i]->turbo_enc.tb_params.k_pos =
789 turbo_enc->tb_params.k_pos;
790 ops[i]->turbo_enc.tb_params.k_neg =
791 turbo_enc->tb_params.k_neg;
792 ops[i]->turbo_enc.tb_params.c =
793 turbo_enc->tb_params.c;
794 ops[i]->turbo_enc.tb_params.c_neg =
795 turbo_enc->tb_params.c_neg;
796 ops[i]->turbo_enc.tb_params.cab =
797 turbo_enc->tb_params.cab;
798 ops[i]->turbo_enc.tb_params.ncb_pos =
799 turbo_enc->tb_params.ncb_pos;
800 ops[i]->turbo_enc.tb_params.ncb_neg =
801 turbo_enc->tb_params.ncb_neg;
802 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
804 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
805 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
806 ops[i]->turbo_enc.cb_params.ncb =
807 turbo_enc->cb_params.ncb;
809 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
810 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
811 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
813 ops[i]->turbo_enc.output = outputs[start_idx + i];
814 ops[i]->turbo_enc.input = inputs[start_idx + i];
819 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
820 unsigned int order_idx, const int expected_status)
822 TEST_ASSERT(op->status == expected_status,
823 "op_status (%d) != expected_status (%d)",
824 op->status, expected_status);
826 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
827 "Ordering error, expected %p, got %p",
828 (void *)(uintptr_t)order_idx, op->opaque_data);
834 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
835 unsigned int order_idx, const int expected_status)
837 TEST_ASSERT(op->status == expected_status,
838 "op_status (%d) != expected_status (%d)",
839 op->status, expected_status);
841 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
842 "Ordering error, expected %p, got %p",
843 (void *)(uintptr_t)order_idx, op->opaque_data);
849 validate_op_chain(struct rte_bbdev_op_data *op,
850 struct op_data_entries *orig_op)
853 struct rte_mbuf *m = op->data;
854 uint8_t nb_dst_segments = orig_op->nb_segments;
856 TEST_ASSERT(nb_dst_segments == m->nb_segs,
857 "Number of segments differ in original (%u) and filled (%u) op",
858 nb_dst_segments, m->nb_segs);
860 for (i = 0; i < nb_dst_segments; ++i) {
861 /* Apply offset to the first mbuf segment */
862 uint16_t offset = (i == 0) ? op->offset : 0;
863 uint16_t data_len = m->data_len - offset;
865 TEST_ASSERT(orig_op->segments[i].length == data_len,
866 "Length of segment differ in original (%u) and filled (%u) op",
867 orig_op->segments[i].length, data_len);
868 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
869 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
871 "Output buffers (CB=%u) are not equal", i);
879 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs,
880 const uint16_t num_to_process)
884 struct op_data_entries *hard_data_orig =
885 &test_vector.entries[DATA_HARD_OUTPUT];
886 struct op_data_entries *soft_data_orig =
887 &test_vector.entries[DATA_SOFT_OUTPUT];
889 for (i = 0; i < num_to_process; i++) {
890 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
892 "Hard output buffers are not equal");
893 if (ref_op->turbo_dec.op_flags &
894 RTE_BBDEV_TURBO_SOFT_OUTPUT)
895 TEST_ASSERT_SUCCESS(validate_op_chain(
896 &bufs->soft_outputs[i],
898 "Soft output buffers are not equal");
905 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process)
909 struct op_data_entries *hard_data_orig =
910 &test_vector.entries[DATA_HARD_OUTPUT];
912 for (i = 0; i < num_to_process; i++)
913 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
914 hard_data_orig), "");
920 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
921 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
925 struct op_data_entries *hard_data_orig =
926 &test_vector.entries[DATA_HARD_OUTPUT];
927 struct op_data_entries *soft_data_orig =
928 &test_vector.entries[DATA_SOFT_OUTPUT];
929 struct rte_bbdev_op_turbo_dec *ops_td;
930 struct rte_bbdev_op_data *hard_output;
931 struct rte_bbdev_op_data *soft_output;
932 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
934 for (i = 0; i < n; ++i) {
935 ops_td = &ops[i]->turbo_dec;
936 hard_output = &ops_td->hard_output;
937 soft_output = &ops_td->soft_output;
939 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
940 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
941 "Returned iter_count (%d) > expected iter_count (%d)",
942 ops_td->iter_count, ref_td->iter_count);
943 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
944 TEST_ASSERT_SUCCESS(ret,
945 "Checking status and ordering for decoder failed");
947 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
949 "Hard output buffers (CB=%u) are not equal",
952 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
953 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
955 "Soft output buffers (CB=%u) are not equal",
963 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
964 struct rte_bbdev_enc_op *ref_op)
968 struct op_data_entries *hard_data_orig =
969 &test_vector.entries[DATA_HARD_OUTPUT];
971 for (i = 0; i < n; ++i) {
972 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
973 TEST_ASSERT_SUCCESS(ret,
974 "Checking status and ordering for encoder failed");
975 TEST_ASSERT_SUCCESS(validate_op_chain(
976 &ops[i]->turbo_enc.output,
978 "Output buffers (CB=%u) are not equal",
986 create_reference_dec_op(struct rte_bbdev_dec_op *op)
989 struct op_data_entries *entry;
991 op->turbo_dec = test_vector.turbo_dec;
992 entry = &test_vector.entries[DATA_INPUT];
993 for (i = 0; i < entry->nb_segments; ++i)
994 op->turbo_dec.input.length +=
995 entry->segments[i].length;
999 create_reference_enc_op(struct rte_bbdev_enc_op *op)
1002 struct op_data_entries *entry;
1004 op->turbo_enc = test_vector.turbo_enc;
1005 entry = &test_vector.entries[DATA_INPUT];
1006 for (i = 0; i < entry->nb_segments; ++i)
1007 op->turbo_enc.input.length +=
1008 entry->segments[i].length;
1012 init_test_op_params(struct test_op_params *op_params,
1013 enum rte_bbdev_op_type op_type, const int expected_status,
1014 const int vector_mask, struct rte_mempool *ops_mp,
1015 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
1018 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1019 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
1020 &op_params->ref_dec_op, 1);
1022 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
1023 &op_params->ref_enc_op, 1);
1025 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
1027 op_params->mp = ops_mp;
1028 op_params->burst_sz = burst_sz;
1029 op_params->num_to_process = num_to_process;
1030 op_params->num_lcores = num_lcores;
1031 op_params->vector_mask = vector_mask;
1032 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1033 op_params->ref_dec_op->status = expected_status;
1034 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
1035 op_params->ref_enc_op->status = expected_status;
1041 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
1042 struct test_op_params *op_params)
1044 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
1046 struct active_device *ad;
1047 unsigned int burst_sz = get_burst_sz();
1048 enum rte_bbdev_op_type op_type = test_vector.op_type;
1049 const struct rte_bbdev_op_cap *capabilities = NULL;
1051 ad = &active_devs[dev_id];
1053 /* Check if device supports op_type */
1054 if (!is_avail_op(ad, test_vector.op_type))
1055 return TEST_SUCCESS;
1057 struct rte_bbdev_info info;
1058 rte_bbdev_info_get(ad->dev_id, &info);
1059 socket_id = GET_SOCKET(info.socket_id);
1061 if (op_type == RTE_BBDEV_OP_NONE)
1062 op_type = RTE_BBDEV_OP_TURBO_ENC;
1063 f_ret = create_mempools(ad, socket_id, op_type,
1065 if (f_ret != TEST_SUCCESS) {
1066 printf("Couldn't create mempools");
1070 f_ret = init_test_op_params(op_params, test_vector.op_type,
1071 test_vector.expected_status,
1077 if (f_ret != TEST_SUCCESS) {
1078 printf("Couldn't init test op params");
1082 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) {
1083 /* Find Decoder capabilities */
1084 const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
1085 while (cap->type != RTE_BBDEV_OP_NONE) {
1086 if (cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1091 TEST_ASSERT_NOT_NULL(capabilities,
1092 "Couldn't find Decoder capabilities");
1094 create_reference_dec_op(op_params->ref_dec_op);
1095 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1096 create_reference_enc_op(op_params->ref_enc_op);
1098 for (i = 0; i < ad->nb_queues; ++i) {
1099 f_ret = fill_queue_buffers(op_params,
1101 ad->hard_out_mbuf_pool,
1102 ad->soft_out_mbuf_pool,
1105 info.drv.min_alignment,
1107 if (f_ret != TEST_SUCCESS) {
1108 printf("Couldn't init queue buffers");
1113 /* Run test case function */
1114 t_ret = test_case_func(ad, op_params);
1116 /* Free active device resources and return */
1117 free_buffers(ad, op_params);
1121 free_buffers(ad, op_params);
1125 /* Run given test function per active device per supported op type
1129 run_test_case(test_case_function *test_case_func)
1134 /* Alloc op_params */
1135 struct test_op_params *op_params = rte_zmalloc(NULL,
1136 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1137 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1138 RTE_ALIGN(sizeof(struct test_op_params),
1139 RTE_CACHE_LINE_SIZE));
1141 /* For each device run test case function */
1142 for (dev = 0; dev < nb_active_devs; ++dev)
1143 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1145 rte_free(op_params);
1151 dequeue_event_callback(uint16_t dev_id,
1152 enum rte_bbdev_event_type event, void *cb_arg,
1157 uint64_t total_time;
1158 uint16_t deq, burst_sz, num_to_process;
1159 uint16_t queue_id = INVALID_QUEUE_ID;
1160 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
1161 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
1162 struct test_buffers *bufs;
1163 struct rte_bbdev_info info;
1165 /* Input length in bytes, million operations per second,
1166 * million bits per second.
1170 struct thread_params *tp = cb_arg;
1171 RTE_SET_USED(ret_param);
1172 queue_id = tp->queue_id;
1174 /* Find matching thread params using queue_id */
1175 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1176 if (tp->queue_id == queue_id)
1179 if (i == MAX_QUEUES) {
1180 printf("%s: Queue_id from interrupt details was not found!\n",
1185 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1186 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1188 "Dequeue interrupt handler called for incorrect event!\n");
1192 burst_sz = tp->op_params->burst_sz;
1193 num_to_process = tp->op_params->num_to_process;
1195 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1196 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops,
1199 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops,
1202 if (deq < burst_sz) {
1204 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1206 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1210 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) {
1211 rte_atomic16_add(&tp->nb_dequeued, deq);
1215 total_time = rte_rdtsc_precise() - tp->start_time;
1217 rte_bbdev_info_get(dev_id, &info);
1219 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1222 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1223 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs,
1225 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1226 ret = validate_enc_buffers(bufs, num_to_process);
1229 printf("Buffers validation failed\n");
1230 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1233 switch (test_vector.op_type) {
1234 case RTE_BBDEV_OP_TURBO_DEC:
1235 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length;
1237 case RTE_BBDEV_OP_TURBO_ENC:
1238 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length;
1240 case RTE_BBDEV_OP_NONE:
1244 printf("Unknown op type: %d\n", test_vector.op_type);
1245 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1249 tp->mops = ((double)num_to_process / 1000000.0) /
1250 ((double)total_time / (double)rte_get_tsc_hz());
1251 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1252 ((double)total_time / (double)rte_get_tsc_hz());
1254 rte_atomic16_add(&tp->nb_dequeued, deq);
1258 throughput_intr_lcore_dec(void *arg)
1260 struct thread_params *tp = arg;
1261 unsigned int enqueued;
1262 struct rte_bbdev_dec_op *ops[MAX_BURST];
1263 const uint16_t queue_id = tp->queue_id;
1264 const uint16_t burst_sz = tp->op_params->burst_sz;
1265 const uint16_t num_to_process = tp->op_params->num_to_process;
1266 struct test_buffers *bufs = NULL;
1267 unsigned int allocs_failed = 0;
1268 struct rte_bbdev_info info;
1271 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1272 "BURST_SIZE should be <= %u", MAX_BURST);
1274 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1275 "Failed to enable interrupts for dev: %u, queue_id: %u",
1276 tp->dev_id, queue_id);
1278 rte_bbdev_info_get(tp->dev_id, &info);
1279 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1281 rte_atomic16_clear(&tp->processing_status);
1282 rte_atomic16_clear(&tp->nb_dequeued);
1284 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1287 tp->start_time = rte_rdtsc_precise();
1288 for (enqueued = 0; enqueued < num_to_process;) {
1290 uint16_t num_to_enq = burst_sz;
1292 if (unlikely(num_to_process - enqueued < num_to_enq))
1293 num_to_enq = num_to_process - enqueued;
1295 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1302 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1303 copy_reference_dec_op(ops, num_to_enq, enqueued,
1307 tp->op_params->ref_dec_op);
1309 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops,
1312 rte_bbdev_dec_op_free_bulk(ops, num_to_enq);
1315 if (allocs_failed > 0)
1316 printf("WARNING: op allocations failed: %u times\n",
1319 return TEST_SUCCESS;
1323 throughput_intr_lcore_enc(void *arg)
1325 struct thread_params *tp = arg;
1326 unsigned int enqueued;
1327 struct rte_bbdev_enc_op *ops[MAX_BURST];
1328 const uint16_t queue_id = tp->queue_id;
1329 const uint16_t burst_sz = tp->op_params->burst_sz;
1330 const uint16_t num_to_process = tp->op_params->num_to_process;
1331 struct test_buffers *bufs = NULL;
1332 unsigned int allocs_failed = 0;
1333 struct rte_bbdev_info info;
1336 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1337 "BURST_SIZE should be <= %u", MAX_BURST);
1339 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1340 "Failed to enable interrupts for dev: %u, queue_id: %u",
1341 tp->dev_id, queue_id);
1343 rte_bbdev_info_get(tp->dev_id, &info);
1344 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1346 rte_atomic16_clear(&tp->processing_status);
1347 rte_atomic16_clear(&tp->nb_dequeued);
1349 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1352 tp->start_time = rte_rdtsc_precise();
1353 for (enqueued = 0; enqueued < num_to_process;) {
1355 uint16_t num_to_enq = burst_sz;
1357 if (unlikely(num_to_process - enqueued < num_to_enq))
1358 num_to_enq = num_to_process - enqueued;
1360 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1367 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1368 copy_reference_enc_op(ops, num_to_enq, enqueued,
1371 tp->op_params->ref_enc_op);
1373 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops,
1376 rte_bbdev_enc_op_free_bulk(ops, num_to_enq);
1379 if (allocs_failed > 0)
1380 printf("WARNING: op allocations failed: %u times\n",
1383 return TEST_SUCCESS;
1387 throughput_pmd_lcore_dec(void *arg)
1389 struct thread_params *tp = arg;
1390 unsigned int enqueued, dequeued;
1391 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1392 uint64_t total_time, start_time;
1393 const uint16_t queue_id = tp->queue_id;
1394 const uint16_t burst_sz = tp->op_params->burst_sz;
1395 const uint16_t num_to_process = tp->op_params->num_to_process;
1396 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1397 struct test_buffers *bufs = NULL;
1398 unsigned int allocs_failed = 0;
1400 struct rte_bbdev_info info;
1402 /* Input length in bytes, million operations per second, million bits
1407 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1408 "BURST_SIZE should be <= %u", MAX_BURST);
1410 rte_bbdev_info_get(tp->dev_id, &info);
1411 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1413 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1416 start_time = rte_rdtsc_precise();
1417 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1420 if (likely(enqueued < num_to_process)) {
1422 uint16_t num_to_enq = burst_sz;
1424 if (unlikely(num_to_process - enqueued < num_to_enq))
1425 num_to_enq = num_to_process - enqueued;
1427 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp,
1428 ops_enq, num_to_enq);
1434 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1435 copy_reference_dec_op(ops_enq, num_to_enq,
1442 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1443 queue_id, ops_enq, num_to_enq);
1446 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq,
1449 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1451 total_time = rte_rdtsc_precise() - start_time;
1453 if (allocs_failed > 0)
1454 printf("WARNING: op allocations failed: %u times\n",
1457 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1458 enqueued, dequeued);
1460 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1461 ret = validate_dec_buffers(ref_op, bufs, num_to_process);
1462 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1465 in_len = ref_op->turbo_dec.input.length;
1466 tp->mops = ((double)num_to_process / 1000000.0) /
1467 ((double)total_time / (double)rte_get_tsc_hz());
1468 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1469 ((double)total_time / (double)rte_get_tsc_hz());
1471 return TEST_SUCCESS;
1475 throughput_pmd_lcore_enc(void *arg)
1477 struct thread_params *tp = arg;
1478 unsigned int enqueued, dequeued;
1479 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1480 uint64_t total_time, start_time;
1481 const uint16_t queue_id = tp->queue_id;
1482 const uint16_t burst_sz = tp->op_params->burst_sz;
1483 const uint16_t num_to_process = tp->op_params->num_to_process;
1484 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1485 struct test_buffers *bufs = NULL;
1486 unsigned int allocs_failed = 0;
1488 struct rte_bbdev_info info;
1490 /* Input length in bytes, million operations per second, million bits
1495 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1496 "BURST_SIZE should be <= %u", MAX_BURST);
1498 rte_bbdev_info_get(tp->dev_id, &info);
1499 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1501 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1504 start_time = rte_rdtsc_precise();
1505 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1508 if (likely(enqueued < num_to_process)) {
1510 uint16_t num_to_enq = burst_sz;
1512 if (unlikely(num_to_process - enqueued < num_to_enq))
1513 num_to_enq = num_to_process - enqueued;
1515 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp,
1516 ops_enq, num_to_enq);
1522 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1523 copy_reference_enc_op(ops_enq, num_to_enq,
1529 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1530 queue_id, ops_enq, num_to_enq);
1533 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq,
1536 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1538 total_time = rte_rdtsc_precise() - start_time;
1540 if (allocs_failed > 0)
1541 printf("WARNING: op allocations failed: %u times\n",
1544 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1545 enqueued, dequeued);
1547 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1548 ret = validate_enc_buffers(bufs, num_to_process);
1549 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1552 in_len = ref_op->turbo_enc.input.length;
1554 tp->mops = ((double)num_to_process / 1000000.0) /
1555 ((double)total_time / (double)rte_get_tsc_hz());
1556 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1557 ((double)total_time / (double)rte_get_tsc_hz());
1559 return TEST_SUCCESS;
1562 print_throughput(struct thread_params *t_params, unsigned int used_cores)
1564 unsigned int lcore_id, iter = 0;
1565 double total_mops = 0, total_mbps = 0;
1567 RTE_LCORE_FOREACH(lcore_id) {
1568 if (iter++ >= used_cores)
1570 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n",
1571 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps);
1572 total_mops += t_params[lcore_id].mops;
1573 total_mbps += t_params[lcore_id].mbps;
1576 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n",
1577 used_cores, total_mops, total_mbps);
1581 * Test function that determines how long an enqueue + dequeue of a burst
1582 * takes on available lcores.
1585 throughput_test(struct active_device *ad,
1586 struct test_op_params *op_params)
1589 unsigned int lcore_id, used_cores = 0;
1590 struct thread_params t_params[MAX_QUEUES];
1591 struct rte_bbdev_info info;
1592 lcore_function_t *throughput_function;
1593 struct thread_params *tp;
1594 uint16_t num_lcores;
1595 const char *op_type_str;
1597 rte_bbdev_info_get(ad->dev_id, &info);
1599 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
1600 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
1601 test_vector.op_type);
1604 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n",
1605 info.dev_name, ad->nb_queues, op_params->burst_sz,
1606 op_params->num_to_process, op_params->num_lcores,
1608 intr_enabled ? "Interrupt mode" : "PMD mode",
1609 (double)rte_get_tsc_hz() / 1000000000.0);
1611 /* Set number of lcores */
1612 num_lcores = (ad->nb_queues < (op_params->num_lcores))
1614 : op_params->num_lcores;
1617 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1618 throughput_function = throughput_intr_lcore_dec;
1620 throughput_function = throughput_intr_lcore_enc;
1622 /* Dequeue interrupt callback registration */
1623 ret = rte_bbdev_callback_register(ad->dev_id,
1624 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
1629 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1630 throughput_function = throughput_pmd_lcore_dec;
1632 throughput_function = throughput_pmd_lcore_enc;
1635 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
1637 t_params[rte_lcore_id()].dev_id = ad->dev_id;
1638 t_params[rte_lcore_id()].op_params = op_params;
1639 t_params[rte_lcore_id()].queue_id =
1640 ad->queue_ids[used_cores++];
1642 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1643 if (used_cores >= num_lcores)
1646 t_params[lcore_id].dev_id = ad->dev_id;
1647 t_params[lcore_id].op_params = op_params;
1648 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++];
1650 rte_eal_remote_launch(throughput_function, &t_params[lcore_id],
1654 rte_atomic16_set(&op_params->sync, SYNC_START);
1655 ret = throughput_function(&t_params[rte_lcore_id()]);
1657 /* Master core is always used */
1659 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1660 if (used_cores++ >= num_lcores)
1663 ret |= rte_eal_wait_lcore(lcore_id);
1666 /* Return if test failed */
1670 /* Print throughput if interrupts are disabled and test passed */
1671 if (!intr_enabled) {
1672 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1673 print_throughput(t_params, num_lcores);
1677 /* In interrupt TC we need to wait for the interrupt callback to deqeue
1678 * all pending operations. Skip waiting for queues which reported an
1679 * error using processing_status variable.
1680 * Wait for master lcore operations.
1682 tp = &t_params[rte_lcore_id()];
1683 while ((rte_atomic16_read(&tp->nb_dequeued) <
1684 op_params->num_to_process) &&
1685 (rte_atomic16_read(&tp->processing_status) !=
1689 ret |= rte_atomic16_read(&tp->processing_status);
1691 /* Wait for slave lcores operations */
1693 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1694 tp = &t_params[lcore_id];
1695 if (used_cores++ >= num_lcores)
1698 while ((rte_atomic16_read(&tp->nb_dequeued) <
1699 op_params->num_to_process) &&
1700 (rte_atomic16_read(&tp->processing_status) !=
1704 ret |= rte_atomic16_read(&tp->processing_status);
1707 /* Print throughput if test passed */
1708 if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE)
1709 print_throughput(t_params, num_lcores);
1715 latency_test_dec(struct rte_mempool *mempool,
1716 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
1717 int vector_mask, uint16_t dev_id, uint16_t queue_id,
1718 const uint16_t num_to_process, uint16_t burst_sz,
1719 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
1721 int ret = TEST_SUCCESS;
1722 uint16_t i, j, dequeued;
1723 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1724 uint64_t start_time = 0, last_time = 0;
1726 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1727 uint16_t enq = 0, deq = 0;
1728 bool first_time = true;
1731 if (unlikely(num_to_process - dequeued < burst_sz))
1732 burst_sz = num_to_process - dequeued;
1734 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1735 TEST_ASSERT_SUCCESS(ret,
1736 "rte_bbdev_dec_op_alloc_bulk() failed");
1737 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1738 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1744 /* Set counter to validate the ordering */
1745 for (j = 0; j < burst_sz; ++j)
1746 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1748 start_time = rte_rdtsc_precise();
1750 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
1752 TEST_ASSERT(enq == burst_sz,
1753 "Error enqueueing burst, expected %u, got %u",
1758 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1759 &ops_deq[deq], burst_sz - deq);
1760 if (likely(first_time && (deq > 0))) {
1761 last_time = rte_rdtsc_precise() - start_time;
1764 } while (unlikely(burst_sz != deq));
1766 *max_time = RTE_MAX(*max_time, last_time);
1767 *min_time = RTE_MIN(*min_time, last_time);
1768 *total_time += last_time;
1770 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1771 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
1773 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1776 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1784 latency_test_enc(struct rte_mempool *mempool,
1785 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
1786 uint16_t dev_id, uint16_t queue_id,
1787 const uint16_t num_to_process, uint16_t burst_sz,
1788 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time)
1790 int ret = TEST_SUCCESS;
1791 uint16_t i, j, dequeued;
1792 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1793 uint64_t start_time = 0, last_time = 0;
1795 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1796 uint16_t enq = 0, deq = 0;
1797 bool first_time = true;
1800 if (unlikely(num_to_process - dequeued < burst_sz))
1801 burst_sz = num_to_process - dequeued;
1803 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1804 TEST_ASSERT_SUCCESS(ret,
1805 "rte_bbdev_enc_op_alloc_bulk() failed");
1806 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1807 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1812 /* Set counter to validate the ordering */
1813 for (j = 0; j < burst_sz; ++j)
1814 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1816 start_time = rte_rdtsc_precise();
1818 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
1820 TEST_ASSERT(enq == burst_sz,
1821 "Error enqueueing burst, expected %u, got %u",
1826 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1827 &ops_deq[deq], burst_sz - deq);
1828 if (likely(first_time && (deq > 0))) {
1829 last_time += rte_rdtsc_precise() - start_time;
1832 } while (unlikely(burst_sz != deq));
1834 *max_time = RTE_MAX(*max_time, last_time);
1835 *min_time = RTE_MIN(*min_time, last_time);
1836 *total_time += last_time;
1838 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1839 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
1840 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1843 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1851 latency_test(struct active_device *ad,
1852 struct test_op_params *op_params)
1855 uint16_t burst_sz = op_params->burst_sz;
1856 const uint16_t num_to_process = op_params->num_to_process;
1857 const enum rte_bbdev_op_type op_type = test_vector.op_type;
1858 const uint16_t queue_id = ad->queue_ids[0];
1859 struct test_buffers *bufs = NULL;
1860 struct rte_bbdev_info info;
1861 uint64_t total_time, min_time, max_time;
1862 const char *op_type_str;
1864 total_time = max_time = 0;
1865 min_time = UINT64_MAX;
1867 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1868 "BURST_SIZE should be <= %u", MAX_BURST);
1870 rte_bbdev_info_get(ad->dev_id, &info);
1871 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1873 op_type_str = rte_bbdev_op_type_str(op_type);
1874 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1877 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1878 info.dev_name, burst_sz, num_to_process, op_type_str);
1880 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1881 iter = latency_test_dec(op_params->mp, bufs,
1882 op_params->ref_dec_op, op_params->vector_mask,
1883 ad->dev_id, queue_id, num_to_process,
1884 burst_sz, &total_time, &min_time, &max_time);
1886 iter = latency_test_enc(op_params->mp, bufs,
1887 op_params->ref_enc_op, ad->dev_id, queue_id,
1888 num_to_process, burst_sz, &total_time,
1889 &min_time, &max_time);
1894 printf("\toperation latency:\n"
1895 "\t\tavg latency: %lg cycles, %lg us\n"
1896 "\t\tmin latency: %lg cycles, %lg us\n"
1897 "\t\tmax latency: %lg cycles, %lg us\n",
1898 (double)total_time / (double)iter,
1899 (double)(total_time * 1000000) / (double)iter /
1900 (double)rte_get_tsc_hz(), (double)min_time,
1901 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(),
1902 (double)max_time, (double)(max_time * 1000000) /
1903 (double)rte_get_tsc_hz());
1905 return TEST_SUCCESS;
1908 #ifdef RTE_BBDEV_OFFLOAD_COST
1910 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id,
1911 struct rte_bbdev_stats *stats)
1913 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id];
1914 struct rte_bbdev_stats *q_stats;
1916 if (queue_id >= dev->data->num_queues)
1919 q_stats = &dev->data->queues[queue_id].queue_stats;
1921 stats->enqueued_count = q_stats->enqueued_count;
1922 stats->dequeued_count = q_stats->dequeued_count;
1923 stats->enqueue_err_count = q_stats->enqueue_err_count;
1924 stats->dequeue_err_count = q_stats->dequeue_err_count;
1925 stats->offload_time = q_stats->offload_time;
1931 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
1932 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
1933 uint16_t queue_id, const uint16_t num_to_process,
1934 uint16_t burst_sz, struct test_time_stats *time_st)
1936 int i, dequeued, ret;
1937 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1938 uint64_t enq_start_time, deq_start_time;
1939 uint64_t enq_sw_last_time, deq_last_time;
1940 struct rte_bbdev_stats stats;
1942 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1943 uint16_t enq = 0, deq = 0;
1945 if (unlikely(num_to_process - dequeued < burst_sz))
1946 burst_sz = num_to_process - dequeued;
1948 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1949 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1950 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1956 /* Start time meas for enqueue function offload latency */
1957 enq_start_time = rte_rdtsc_precise();
1959 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
1960 &ops_enq[enq], burst_sz - enq);
1961 } while (unlikely(burst_sz != enq));
1963 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
1964 TEST_ASSERT_SUCCESS(ret,
1965 "Failed to get stats for queue (%u) of device (%u)",
1968 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
1970 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
1972 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
1974 time_st->enq_sw_tot_time += enq_sw_last_time;
1976 time_st->enq_tur_max_time = RTE_MAX(time_st->enq_tur_max_time,
1977 stats.offload_time);
1978 time_st->enq_tur_min_time = RTE_MIN(time_st->enq_tur_min_time,
1979 stats.offload_time);
1980 time_st->enq_tur_tot_time += stats.offload_time;
1982 /* ensure enqueue has been completed */
1985 /* Start time meas for dequeue function offload latency */
1986 deq_start_time = rte_rdtsc_precise();
1987 /* Dequeue one operation */
1989 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1991 } while (unlikely(deq != 1));
1993 deq_last_time = rte_rdtsc_precise() - deq_start_time;
1994 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
1996 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
1998 time_st->deq_tot_time += deq_last_time;
2000 /* Dequeue remaining operations if needed*/
2001 while (burst_sz != deq)
2002 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
2003 &ops_deq[deq], burst_sz - deq);
2005 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
2013 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
2014 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
2015 uint16_t queue_id, const uint16_t num_to_process,
2016 uint16_t burst_sz, struct test_time_stats *time_st)
2018 int i, dequeued, ret;
2019 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
2020 uint64_t enq_start_time, deq_start_time;
2021 uint64_t enq_sw_last_time, deq_last_time;
2022 struct rte_bbdev_stats stats;
2024 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
2025 uint16_t enq = 0, deq = 0;
2027 if (unlikely(num_to_process - dequeued < burst_sz))
2028 burst_sz = num_to_process - dequeued;
2030 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
2031 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
2032 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
2037 /* Start time meas for enqueue function offload latency */
2038 enq_start_time = rte_rdtsc_precise();
2040 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
2041 &ops_enq[enq], burst_sz - enq);
2042 } while (unlikely(burst_sz != enq));
2044 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats);
2045 TEST_ASSERT_SUCCESS(ret,
2046 "Failed to get stats for queue (%u) of device (%u)",
2049 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time -
2051 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time,
2053 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time,
2055 time_st->enq_sw_tot_time += enq_sw_last_time;
2057 time_st->enq_tur_max_time = RTE_MAX(time_st->enq_tur_max_time,
2058 stats.offload_time);
2059 time_st->enq_tur_min_time = RTE_MIN(time_st->enq_tur_min_time,
2060 stats.offload_time);
2061 time_st->enq_tur_tot_time += stats.offload_time;
2063 /* ensure enqueue has been completed */
2066 /* Start time meas for dequeue function offload latency */
2067 deq_start_time = rte_rdtsc_precise();
2068 /* Dequeue one operation */
2070 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2072 } while (unlikely(deq != 1));
2074 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2075 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time,
2077 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time,
2079 time_st->deq_tot_time += deq_last_time;
2081 while (burst_sz != deq)
2082 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
2083 &ops_deq[deq], burst_sz - deq);
2085 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
2094 offload_cost_test(struct active_device *ad,
2095 struct test_op_params *op_params)
2097 #ifndef RTE_BBDEV_OFFLOAD_COST
2099 RTE_SET_USED(op_params);
2100 printf("Offload latency test is disabled.\n");
2101 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
2102 return TEST_SKIPPED;
2105 uint16_t burst_sz = op_params->burst_sz;
2106 const uint16_t num_to_process = op_params->num_to_process;
2107 const enum rte_bbdev_op_type op_type = test_vector.op_type;
2108 const uint16_t queue_id = ad->queue_ids[0];
2109 struct test_buffers *bufs = NULL;
2110 struct rte_bbdev_info info;
2111 const char *op_type_str;
2112 struct test_time_stats time_st;
2114 memset(&time_st, 0, sizeof(struct test_time_stats));
2115 time_st.enq_sw_min_time = UINT64_MAX;
2116 time_st.enq_tur_min_time = UINT64_MAX;
2117 time_st.deq_min_time = UINT64_MAX;
2119 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2120 "BURST_SIZE should be <= %u", MAX_BURST);
2122 rte_bbdev_info_get(ad->dev_id, &info);
2123 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
2125 op_type_str = rte_bbdev_op_type_str(op_type);
2126 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2129 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2130 info.dev_name, burst_sz, num_to_process, op_type_str);
2132 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2133 iter = offload_latency_test_dec(op_params->mp, bufs,
2134 op_params->ref_dec_op, ad->dev_id, queue_id,
2135 num_to_process, burst_sz, &time_st);
2137 iter = offload_latency_test_enc(op_params->mp, bufs,
2138 op_params->ref_enc_op, ad->dev_id, queue_id,
2139 num_to_process, burst_sz, &time_st);
2144 printf("\tenq offload cost latency:\n"
2145 "\t\tsoftware avg %lg cycles, %lg us\n"
2146 "\t\tsoftware min %lg cycles, %lg us\n"
2147 "\t\tsoftware max %lg cycles, %lg us\n"
2148 "\t\tturbo avg %lg cycles, %lg us\n"
2149 "\t\tturbo min %lg cycles, %lg us\n"
2150 "\t\tturbo max %lg cycles, %lg us\n",
2151 (double)time_st.enq_sw_tot_time / (double)iter,
2152 (double)(time_st.enq_sw_tot_time * 1000000) /
2153 (double)iter / (double)rte_get_tsc_hz(),
2154 (double)time_st.enq_sw_min_time,
2155 (double)(time_st.enq_sw_min_time * 1000000) /
2156 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time,
2157 (double)(time_st.enq_sw_max_time * 1000000) /
2158 rte_get_tsc_hz(), (double)time_st.enq_tur_tot_time /
2160 (double)(time_st.enq_tur_tot_time * 1000000) /
2161 (double)iter / (double)rte_get_tsc_hz(),
2162 (double)time_st.enq_tur_min_time,
2163 (double)(time_st.enq_tur_min_time * 1000000) /
2164 rte_get_tsc_hz(), (double)time_st.enq_tur_max_time,
2165 (double)(time_st.enq_tur_max_time * 1000000) /
2168 printf("\tdeq offload cost latency - one op:\n"
2169 "\t\tavg %lg cycles, %lg us\n"
2170 "\t\tmin %lg cycles, %lg us\n"
2171 "\t\tmax %lg cycles, %lg us\n",
2172 (double)time_st.deq_tot_time / (double)iter,
2173 (double)(time_st.deq_tot_time * 1000000) /
2174 (double)iter / (double)rte_get_tsc_hz(),
2175 (double)time_st.deq_min_time,
2176 (double)(time_st.deq_min_time * 1000000) /
2177 rte_get_tsc_hz(), (double)time_st.deq_max_time,
2178 (double)(time_st.deq_max_time * 1000000) /
2181 return TEST_SUCCESS;
2185 #ifdef RTE_BBDEV_OFFLOAD_COST
2187 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
2188 const uint16_t num_to_process, uint16_t burst_sz,
2189 uint64_t *deq_tot_time, uint64_t *deq_min_time,
2190 uint64_t *deq_max_time)
2193 struct rte_bbdev_dec_op *ops[MAX_BURST];
2194 uint64_t deq_start_time, deq_last_time;
2196 /* Test deq offload latency from an empty queue */
2198 for (i = 0, deq_total = 0; deq_total < num_to_process;
2199 ++i, deq_total += burst_sz) {
2200 deq_start_time = rte_rdtsc_precise();
2202 if (unlikely(num_to_process - deq_total < burst_sz))
2203 burst_sz = num_to_process - deq_total;
2204 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
2206 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2207 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
2208 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
2209 *deq_tot_time += deq_last_time;
2216 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
2217 const uint16_t num_to_process, uint16_t burst_sz,
2218 uint64_t *deq_tot_time, uint64_t *deq_min_time,
2219 uint64_t *deq_max_time)
2222 struct rte_bbdev_enc_op *ops[MAX_BURST];
2223 uint64_t deq_start_time, deq_last_time;
2225 /* Test deq offload latency from an empty queue */
2226 for (i = 0, deq_total = 0; deq_total < num_to_process;
2227 ++i, deq_total += burst_sz) {
2228 deq_start_time = rte_rdtsc_precise();
2230 if (unlikely(num_to_process - deq_total < burst_sz))
2231 burst_sz = num_to_process - deq_total;
2232 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
2234 deq_last_time = rte_rdtsc_precise() - deq_start_time;
2235 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time);
2236 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time);
2237 *deq_tot_time += deq_last_time;
2245 offload_latency_empty_q_test(struct active_device *ad,
2246 struct test_op_params *op_params)
2248 #ifndef RTE_BBDEV_OFFLOAD_COST
2250 RTE_SET_USED(op_params);
2251 printf("Offload latency empty dequeue test is disabled.\n");
2252 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n");
2253 return TEST_SKIPPED;
2256 uint64_t deq_tot_time, deq_min_time, deq_max_time;
2257 uint16_t burst_sz = op_params->burst_sz;
2258 const uint16_t num_to_process = op_params->num_to_process;
2259 const enum rte_bbdev_op_type op_type = test_vector.op_type;
2260 const uint16_t queue_id = ad->queue_ids[0];
2261 struct rte_bbdev_info info;
2262 const char *op_type_str;
2264 deq_tot_time = deq_max_time = 0;
2265 deq_min_time = UINT64_MAX;
2267 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2268 "BURST_SIZE should be <= %u", MAX_BURST);
2270 rte_bbdev_info_get(ad->dev_id, &info);
2272 op_type_str = rte_bbdev_op_type_str(op_type);
2273 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2276 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2277 info.dev_name, burst_sz, num_to_process, op_type_str);
2279 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2280 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
2281 num_to_process, burst_sz, &deq_tot_time,
2282 &deq_min_time, &deq_max_time);
2284 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
2285 num_to_process, burst_sz, &deq_tot_time,
2286 &deq_min_time, &deq_max_time);
2291 printf("\tempty deq offload\n"
2292 "\t\tavg. latency: %lg cycles, %lg us\n"
2293 "\t\tmin. latency: %lg cycles, %lg us\n"
2294 "\t\tmax. latency: %lg cycles, %lg us\n",
2295 (double)deq_tot_time / (double)iter,
2296 (double)(deq_tot_time * 1000000) / (double)iter /
2297 (double)rte_get_tsc_hz(), (double)deq_min_time,
2298 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(),
2299 (double)deq_max_time, (double)(deq_max_time * 1000000) /
2302 return TEST_SUCCESS;
2309 return run_test_case(throughput_test);
2313 offload_cost_tc(void)
2315 return run_test_case(offload_cost_test);
2319 offload_latency_empty_q_tc(void)
2321 return run_test_case(offload_latency_empty_q_test);
2327 return run_test_case(latency_test);
2333 return run_test_case(throughput_test);
2336 static struct unit_test_suite bbdev_throughput_testsuite = {
2337 .suite_name = "BBdev Throughput Tests",
2338 .setup = testsuite_setup,
2339 .teardown = testsuite_teardown,
2340 .unit_test_cases = {
2341 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
2342 TEST_CASES_END() /**< NULL terminate unit test array */
2346 static struct unit_test_suite bbdev_validation_testsuite = {
2347 .suite_name = "BBdev Validation Tests",
2348 .setup = testsuite_setup,
2349 .teardown = testsuite_teardown,
2350 .unit_test_cases = {
2351 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
2352 TEST_CASES_END() /**< NULL terminate unit test array */
2356 static struct unit_test_suite bbdev_latency_testsuite = {
2357 .suite_name = "BBdev Latency Tests",
2358 .setup = testsuite_setup,
2359 .teardown = testsuite_teardown,
2360 .unit_test_cases = {
2361 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc),
2362 TEST_CASES_END() /**< NULL terminate unit test array */
2366 static struct unit_test_suite bbdev_offload_cost_testsuite = {
2367 .suite_name = "BBdev Offload Cost Tests",
2368 .setup = testsuite_setup,
2369 .teardown = testsuite_teardown,
2370 .unit_test_cases = {
2371 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc),
2372 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
2373 TEST_CASES_END() /**< NULL terminate unit test array */
2377 static struct unit_test_suite bbdev_interrupt_testsuite = {
2378 .suite_name = "BBdev Interrupt Tests",
2379 .setup = interrupt_testsuite_setup,
2380 .teardown = testsuite_teardown,
2381 .unit_test_cases = {
2382 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
2383 TEST_CASES_END() /**< NULL terminate unit test array */
2387 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
2388 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
2389 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
2390 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite);
2391 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);