1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
9 #include <rte_common.h>
11 #include <rte_launch.h>
12 #include <rte_bbdev.h>
13 #include <rte_cycles.h>
14 #include <rte_lcore.h>
15 #include <rte_malloc.h>
16 #include <rte_random.h>
17 #include <rte_hexdump.h>
20 #include "test_bbdev_vector.h"
22 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
24 #define MAX_QUEUES RTE_MAX_LCORE
26 #define OPS_CACHE_SIZE 256U
27 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
32 #define INVALID_QUEUE_ID -1
34 static struct test_bbdev_vector test_vector;
36 /* Switch between PMD and Interrupt for throughput TC */
37 static bool intr_enabled;
39 /* Represents tested active devices */
40 static struct active_device {
41 const char *driver_name;
43 uint16_t supported_ops;
44 uint16_t queue_ids[MAX_QUEUES];
46 struct rte_mempool *ops_mempool;
47 struct rte_mempool *in_mbuf_pool;
48 struct rte_mempool *hard_out_mbuf_pool;
49 struct rte_mempool *soft_out_mbuf_pool;
50 } active_devs[RTE_BBDEV_MAX_DEVS];
52 static uint8_t nb_active_devs;
54 /* Data buffers used by BBDEV ops */
56 struct rte_bbdev_op_data *inputs;
57 struct rte_bbdev_op_data *hard_outputs;
58 struct rte_bbdev_op_data *soft_outputs;
61 /* Operation parameters specific for given test case */
62 struct test_op_params {
63 struct rte_mempool *mp;
64 struct rte_bbdev_dec_op *ref_dec_op;
65 struct rte_bbdev_enc_op *ref_enc_op;
67 uint16_t num_to_process;
71 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
74 /* Contains per lcore params */
75 struct thread_params {
81 rte_atomic16_t nb_dequeued;
82 rte_atomic16_t processing_status;
83 struct test_op_params *op_params;
86 typedef int (test_case_function)(struct active_device *ad,
87 struct test_op_params *op_params);
90 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
92 ad->supported_ops |= (1 << op_type);
96 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
98 return ad->supported_ops & (1 << op_type);
102 flags_match(uint32_t flags_req, uint32_t flags_present)
104 return (flags_req & flags_present) == flags_req;
108 clear_soft_out_cap(uint32_t *op_flags)
110 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
111 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
112 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
116 check_dev_cap(const struct rte_bbdev_info *dev_info)
119 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs;
120 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
122 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
123 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
124 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
126 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
127 if (op_cap->type != test_vector.op_type)
130 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
131 const struct rte_bbdev_op_cap_turbo_dec *cap =
132 &op_cap->cap.turbo_dec;
133 /* Ignore lack of soft output capability, just skip
134 * checking if soft output is valid.
136 if ((test_vector.turbo_dec.op_flags &
137 RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
138 !(cap->capability_flags &
139 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
141 "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
144 &test_vector.turbo_dec.op_flags);
147 if (!flags_match(test_vector.turbo_dec.op_flags,
148 cap->capability_flags))
150 if (nb_inputs > cap->num_buffers_src) {
151 printf("Too many inputs defined: %u, max: %u\n",
152 nb_inputs, cap->num_buffers_src);
155 if (nb_soft_outputs > cap->num_buffers_soft_out &&
156 (test_vector.turbo_dec.op_flags &
157 RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
159 "Too many soft outputs defined: %u, max: %u\n",
161 cap->num_buffers_soft_out);
164 if (nb_hard_outputs > cap->num_buffers_hard_out) {
166 "Too many hard outputs defined: %u, max: %u\n",
168 cap->num_buffers_hard_out);
171 if (intr_enabled && !(cap->capability_flags &
172 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
174 "Dequeue interrupts are not supported!\n");
179 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
180 const struct rte_bbdev_op_cap_turbo_enc *cap =
181 &op_cap->cap.turbo_enc;
183 if (!flags_match(test_vector.turbo_enc.op_flags,
184 cap->capability_flags))
186 if (nb_inputs > cap->num_buffers_src) {
187 printf("Too many inputs defined: %u, max: %u\n",
188 nb_inputs, cap->num_buffers_src);
191 if (nb_hard_outputs > cap->num_buffers_dst) {
193 "Too many hard outputs defined: %u, max: %u\n",
194 nb_hard_outputs, cap->num_buffers_src);
197 if (intr_enabled && !(cap->capability_flags &
198 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
200 "Dequeue interrupts are not supported!\n");
208 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
209 return TEST_SUCCESS; /* Special case for NULL device */
214 /* calculates optimal mempool size not smaller than the val */
216 optimal_mempool_size(unsigned int val)
218 return rte_align32pow2(val + 1) - 1;
221 /* allocates mbuf mempool for inputs and outputs */
222 static struct rte_mempool *
223 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
224 int socket_id, unsigned int mbuf_pool_size,
225 const char *op_type_str)
228 uint32_t max_seg_sz = 0;
229 char pool_name[RTE_MEMPOOL_NAMESIZE];
231 /* find max input segment size */
232 for (i = 0; i < entries->nb_segments; ++i)
233 if (entries->segments[i].length > max_seg_sz)
234 max_seg_sz = entries->segments[i].length;
236 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
238 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
239 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
240 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
244 create_mempools(struct active_device *ad, int socket_id,
245 enum rte_bbdev_op_type op_type, uint16_t num_ops)
247 struct rte_mempool *mp;
248 unsigned int ops_pool_size, mbuf_pool_size = 0;
249 char pool_name[RTE_MEMPOOL_NAMESIZE];
250 const char *op_type_str;
252 struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
253 struct op_data_entries *hard_out =
254 &test_vector.entries[DATA_HARD_OUTPUT];
255 struct op_data_entries *soft_out =
256 &test_vector.entries[DATA_SOFT_OUTPUT];
258 /* allocate ops mempool */
259 ops_pool_size = optimal_mempool_size(RTE_MAX(
260 /* Ops used plus 1 reference op */
261 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
262 /* Minimal cache size plus 1 reference op */
263 (unsigned int)(1.5 * rte_lcore_count() *
264 OPS_CACHE_SIZE + 1)),
267 op_type_str = rte_bbdev_op_type_str(op_type);
268 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
270 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
272 mp = rte_bbdev_op_pool_create(pool_name, op_type,
273 ops_pool_size, OPS_CACHE_SIZE, socket_id);
274 TEST_ASSERT_NOT_NULL(mp,
275 "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
279 ad->ops_mempool = mp;
282 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
283 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
284 TEST_ASSERT_NOT_NULL(mp,
285 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
289 ad->in_mbuf_pool = mp;
292 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
293 hard_out->nb_segments);
294 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
296 TEST_ASSERT_NOT_NULL(mp,
297 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
301 ad->hard_out_mbuf_pool = mp;
303 if (soft_out->nb_segments == 0)
307 mbuf_pool_size = optimal_mempool_size(ops_pool_size *
308 soft_out->nb_segments);
309 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size,
311 TEST_ASSERT_NOT_NULL(mp,
312 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
316 ad->soft_out_mbuf_pool = mp;
322 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
323 struct test_bbdev_vector *vector)
326 unsigned int queue_id;
327 struct rte_bbdev_queue_conf qconf;
328 struct active_device *ad = &active_devs[nb_active_devs];
329 unsigned int nb_queues;
330 enum rte_bbdev_op_type op_type = vector->op_type;
332 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
334 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
336 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
337 dev_id, nb_queues, info->socket_id, ret);
341 /* configure interrupts if needed */
343 ret = rte_bbdev_intr_enable(dev_id);
345 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
351 /* setup device queues */
352 qconf.socket = info->socket_id;
353 qconf.queue_size = info->drv.default_queue_conf.queue_size;
355 qconf.deferred_start = 0;
356 qconf.op_type = op_type;
358 for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
359 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
362 "Allocated all queues (id=%u) at prio%u on dev%u\n",
363 queue_id, qconf.priority, dev_id);
365 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
369 printf("All queues on dev %u allocated: %u\n",
373 ad->queue_ids[queue_id] = queue_id;
375 TEST_ASSERT(queue_id != 0,
376 "ERROR Failed to configure any queues on dev %u",
378 ad->nb_queues = queue_id;
380 set_avail_op(ad, op_type);
386 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
387 struct test_bbdev_vector *vector)
391 active_devs[nb_active_devs].driver_name = info->drv.driver_name;
392 active_devs[nb_active_devs].dev_id = dev_id;
394 ret = add_bbdev_dev(dev_id, info, vector);
395 if (ret == TEST_SUCCESS)
401 populate_active_devices(void)
405 uint8_t nb_devs_added = 0;
406 struct rte_bbdev_info info;
408 RTE_BBDEV_FOREACH(dev_id) {
409 rte_bbdev_info_get(dev_id, &info);
411 if (check_dev_cap(&info)) {
413 "Device %d (%s) does not support specified capabilities\n",
414 dev_id, info.dev_name);
418 ret = add_active_device(dev_id, &info, &test_vector);
420 printf("Adding active bbdev %s skipped\n",
427 return nb_devs_added;
431 read_test_vector(void)
435 memset(&test_vector, 0, sizeof(test_vector));
436 printf("Test vector file = %s\n", get_vector_filename());
437 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
438 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
439 get_vector_filename());
445 testsuite_setup(void)
447 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
449 if (populate_active_devices() == 0) {
450 printf("No suitable devices found!\n");
458 interrupt_testsuite_setup(void)
460 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
462 /* Enable interrupts */
465 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
466 if (populate_active_devices() == 0 ||
467 test_vector.op_type == RTE_BBDEV_OP_NONE) {
468 intr_enabled = false;
469 printf("No suitable devices found!\n");
477 testsuite_teardown(void)
481 /* Unconfigure devices */
482 RTE_BBDEV_FOREACH(dev_id)
483 rte_bbdev_close(dev_id);
485 /* Clear active devices structs. */
486 memset(active_devs, 0, sizeof(active_devs));
495 for (i = 0; i < nb_active_devs; i++) {
496 dev_id = active_devs[i].dev_id;
497 /* reset bbdev stats */
498 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
499 "Failed to reset stats of bbdev %u", dev_id);
500 /* start the device */
501 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
502 "Failed to start bbdev %u", dev_id);
512 struct rte_bbdev_stats stats;
514 for (i = 0; i < nb_active_devs; i++) {
515 dev_id = active_devs[i].dev_id;
516 /* read stats and print */
517 rte_bbdev_stats_get(dev_id, &stats);
518 /* Stop the device */
519 rte_bbdev_stop(dev_id);
524 init_op_data_objs(struct rte_bbdev_op_data *bufs,
525 struct op_data_entries *ref_entries,
526 struct rte_mempool *mbuf_pool, const uint16_t n,
527 enum op_data_type op_type, uint16_t min_alignment)
532 for (i = 0; i < n; ++i) {
534 struct op_data_buf *seg = &ref_entries->segments[0];
535 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
536 TEST_ASSERT_NOT_NULL(m_head,
537 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
538 op_type, n * ref_entries->nb_segments,
541 bufs[i].data = m_head;
545 if (op_type == DATA_INPUT) {
546 data = rte_pktmbuf_append(m_head, seg->length);
547 TEST_ASSERT_NOT_NULL(data,
548 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
549 seg->length, op_type);
551 TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
552 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
553 data, min_alignment);
554 rte_memcpy(data, seg->addr, seg->length);
555 bufs[i].length += seg->length;
558 for (j = 1; j < ref_entries->nb_segments; ++j) {
559 struct rte_mbuf *m_tail =
560 rte_pktmbuf_alloc(mbuf_pool);
561 TEST_ASSERT_NOT_NULL(m_tail,
562 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
564 n * ref_entries->nb_segments,
568 data = rte_pktmbuf_append(m_tail, seg->length);
569 TEST_ASSERT_NOT_NULL(data,
570 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
571 seg->length, op_type);
573 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
575 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
576 data, min_alignment);
577 rte_memcpy(data, seg->addr, seg->length);
578 bufs[i].length += seg->length;
580 ret = rte_pktmbuf_chain(m_head, m_tail);
581 TEST_ASSERT_SUCCESS(ret,
582 "Couldn't chain mbufs from %d data type mbuf pool",
592 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
597 *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
598 if (*buffers == NULL) {
599 printf("WARNING: Failed to allocate op_data on socket %d\n",
601 /* try to allocate memory on other detected sockets */
602 for (i = 0; i < socket; i++) {
603 *buffers = rte_zmalloc_socket(NULL, len, 0, i);
604 if (*buffers != NULL)
609 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
613 fill_queue_buffers(struct test_op_params *op_params,
614 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
615 struct rte_mempool *soft_out_mp, uint16_t queue_id,
616 uint16_t min_alignment, const int socket_id)
619 enum op_data_type type;
620 const uint16_t n = op_params->num_to_process;
622 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
628 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
629 &op_params->q_bufs[socket_id][queue_id].inputs,
630 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
631 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
634 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
635 struct op_data_entries *ref_entries =
636 &test_vector.entries[type];
637 if (ref_entries->nb_segments == 0)
640 ret = allocate_buffers_on_socket(queue_ops[type],
641 n * sizeof(struct rte_bbdev_op_data),
643 TEST_ASSERT_SUCCESS(ret,
644 "Couldn't allocate memory for rte_bbdev_op_data structs");
646 ret = init_op_data_objs(*queue_ops[type], ref_entries,
647 mbuf_pools[type], n, type, min_alignment);
648 TEST_ASSERT_SUCCESS(ret,
649 "Couldn't init rte_bbdev_op_data structs");
656 free_buffers(struct active_device *ad, struct test_op_params *op_params)
660 rte_mempool_free(ad->ops_mempool);
661 rte_mempool_free(ad->in_mbuf_pool);
662 rte_mempool_free(ad->hard_out_mbuf_pool);
663 rte_mempool_free(ad->soft_out_mbuf_pool);
665 for (i = 0; i < rte_lcore_count(); ++i) {
666 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
667 rte_free(op_params->q_bufs[j][i].inputs);
668 rte_free(op_params->q_bufs[j][i].hard_outputs);
669 rte_free(op_params->q_bufs[j][i].soft_outputs);
675 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
676 unsigned int start_idx,
677 struct rte_bbdev_op_data *inputs,
678 struct rte_bbdev_op_data *hard_outputs,
679 struct rte_bbdev_op_data *soft_outputs,
680 struct rte_bbdev_dec_op *ref_op)
683 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
685 for (i = 0; i < n; ++i) {
686 if (turbo_dec->code_block_mode == 0) {
687 ops[i]->turbo_dec.tb_params.ea =
688 turbo_dec->tb_params.ea;
689 ops[i]->turbo_dec.tb_params.eb =
690 turbo_dec->tb_params.eb;
691 ops[i]->turbo_dec.tb_params.k_pos =
692 turbo_dec->tb_params.k_pos;
693 ops[i]->turbo_dec.tb_params.k_neg =
694 turbo_dec->tb_params.k_neg;
695 ops[i]->turbo_dec.tb_params.c =
696 turbo_dec->tb_params.c;
697 ops[i]->turbo_dec.tb_params.c_neg =
698 turbo_dec->tb_params.c_neg;
699 ops[i]->turbo_dec.tb_params.cab =
700 turbo_dec->tb_params.cab;
702 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
703 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
706 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
707 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
708 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
709 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
710 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
711 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
712 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
714 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
715 ops[i]->turbo_dec.input = inputs[start_idx + i];
716 if (soft_outputs != NULL)
717 ops[i]->turbo_dec.soft_output =
718 soft_outputs[start_idx + i];
723 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
724 unsigned int start_idx,
725 struct rte_bbdev_op_data *inputs,
726 struct rte_bbdev_op_data *outputs,
727 struct rte_bbdev_enc_op *ref_op)
730 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
731 for (i = 0; i < n; ++i) {
732 if (turbo_enc->code_block_mode == 0) {
733 ops[i]->turbo_enc.tb_params.ea =
734 turbo_enc->tb_params.ea;
735 ops[i]->turbo_enc.tb_params.eb =
736 turbo_enc->tb_params.eb;
737 ops[i]->turbo_enc.tb_params.k_pos =
738 turbo_enc->tb_params.k_pos;
739 ops[i]->turbo_enc.tb_params.k_neg =
740 turbo_enc->tb_params.k_neg;
741 ops[i]->turbo_enc.tb_params.c =
742 turbo_enc->tb_params.c;
743 ops[i]->turbo_enc.tb_params.c_neg =
744 turbo_enc->tb_params.c_neg;
745 ops[i]->turbo_enc.tb_params.cab =
746 turbo_enc->tb_params.cab;
747 ops[i]->turbo_enc.tb_params.ncb_pos =
748 turbo_enc->tb_params.ncb_pos;
749 ops[i]->turbo_enc.tb_params.ncb_neg =
750 turbo_enc->tb_params.ncb_neg;
751 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
753 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
754 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
755 ops[i]->turbo_enc.cb_params.ncb =
756 turbo_enc->cb_params.ncb;
758 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
759 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
760 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
762 ops[i]->turbo_enc.output = outputs[start_idx + i];
763 ops[i]->turbo_enc.input = inputs[start_idx + i];
768 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
769 unsigned int order_idx, const int expected_status)
771 TEST_ASSERT(op->status == expected_status,
772 "op_status (%d) != expected_status (%d)",
773 op->status, expected_status);
775 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
776 "Ordering error, expected %p, got %p",
777 (void *)(uintptr_t)order_idx, op->opaque_data);
783 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
784 unsigned int order_idx, const int expected_status)
786 TEST_ASSERT(op->status == expected_status,
787 "op_status (%d) != expected_status (%d)",
788 op->status, expected_status);
790 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
791 "Ordering error, expected %p, got %p",
792 (void *)(uintptr_t)order_idx, op->opaque_data);
798 validate_op_chain(struct rte_bbdev_op_data *op,
799 struct op_data_entries *orig_op)
802 struct rte_mbuf *m = op->data;
803 uint8_t nb_dst_segments = orig_op->nb_segments;
805 TEST_ASSERT(nb_dst_segments == m->nb_segs,
806 "Number of segments differ in original (%u) and filled (%u) op",
807 nb_dst_segments, m->nb_segs);
809 for (i = 0; i < nb_dst_segments; ++i) {
810 /* Apply offset to the first mbuf segment */
811 uint16_t offset = (i == 0) ? op->offset : 0;
812 uint16_t data_len = m->data_len - offset;
814 TEST_ASSERT(orig_op->segments[i].length == data_len,
815 "Length of segment differ in original (%u) and filled (%u) op",
816 orig_op->segments[i].length, data_len);
817 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
818 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
820 "Output buffers (CB=%u) are not equal", i);
828 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs,
829 const uint16_t num_to_process)
833 struct op_data_entries *hard_data_orig =
834 &test_vector.entries[DATA_HARD_OUTPUT];
835 struct op_data_entries *soft_data_orig =
836 &test_vector.entries[DATA_SOFT_OUTPUT];
838 for (i = 0; i < num_to_process; i++) {
839 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
841 "Hard output buffers are not equal");
842 if (ref_op->turbo_dec.op_flags &
843 RTE_BBDEV_TURBO_SOFT_OUTPUT)
844 TEST_ASSERT_SUCCESS(validate_op_chain(
845 &bufs->soft_outputs[i],
847 "Soft output buffers are not equal");
854 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process)
858 struct op_data_entries *hard_data_orig =
859 &test_vector.entries[DATA_HARD_OUTPUT];
861 for (i = 0; i < num_to_process; i++)
862 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
863 hard_data_orig), "");
869 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
870 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
874 struct op_data_entries *hard_data_orig =
875 &test_vector.entries[DATA_HARD_OUTPUT];
876 struct op_data_entries *soft_data_orig =
877 &test_vector.entries[DATA_SOFT_OUTPUT];
878 struct rte_bbdev_op_turbo_dec *ops_td;
879 struct rte_bbdev_op_data *hard_output;
880 struct rte_bbdev_op_data *soft_output;
881 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
883 for (i = 0; i < n; ++i) {
884 ops_td = &ops[i]->turbo_dec;
885 hard_output = &ops_td->hard_output;
886 soft_output = &ops_td->soft_output;
888 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
889 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
890 "Returned iter_count (%d) > expected iter_count (%d)",
891 ops_td->iter_count, ref_td->iter_count);
892 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
893 TEST_ASSERT_SUCCESS(ret,
894 "Checking status and ordering for decoder failed");
896 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
898 "Hard output buffers (CB=%u) are not equal",
901 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
902 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
904 "Soft output buffers (CB=%u) are not equal",
912 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
913 struct rte_bbdev_enc_op *ref_op)
917 struct op_data_entries *hard_data_orig =
918 &test_vector.entries[DATA_HARD_OUTPUT];
920 for (i = 0; i < n; ++i) {
921 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
922 TEST_ASSERT_SUCCESS(ret,
923 "Checking status and ordering for encoder failed");
924 TEST_ASSERT_SUCCESS(validate_op_chain(
925 &ops[i]->turbo_enc.output,
927 "Output buffers (CB=%u) are not equal",
935 create_reference_dec_op(struct rte_bbdev_dec_op *op)
938 struct op_data_entries *entry;
940 op->turbo_dec = test_vector.turbo_dec;
941 entry = &test_vector.entries[DATA_INPUT];
942 for (i = 0; i < entry->nb_segments; ++i)
943 op->turbo_dec.input.length +=
944 entry->segments[i].length;
948 create_reference_enc_op(struct rte_bbdev_enc_op *op)
951 struct op_data_entries *entry;
953 op->turbo_enc = test_vector.turbo_enc;
954 entry = &test_vector.entries[DATA_INPUT];
955 for (i = 0; i < entry->nb_segments; ++i)
956 op->turbo_enc.input.length +=
957 entry->segments[i].length;
961 init_test_op_params(struct test_op_params *op_params,
962 enum rte_bbdev_op_type op_type, const int expected_status,
963 const int vector_mask, struct rte_mempool *ops_mp,
964 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
967 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
968 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
969 &op_params->ref_dec_op, 1);
971 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
972 &op_params->ref_enc_op, 1);
974 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
976 op_params->mp = ops_mp;
977 op_params->burst_sz = burst_sz;
978 op_params->num_to_process = num_to_process;
979 op_params->num_lcores = num_lcores;
980 op_params->vector_mask = vector_mask;
981 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
982 op_params->ref_dec_op->status = expected_status;
983 else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
984 op_params->ref_enc_op->status = expected_status;
990 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
991 struct test_op_params *op_params)
993 int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
995 struct active_device *ad;
996 unsigned int burst_sz = get_burst_sz();
997 enum rte_bbdev_op_type op_type = test_vector.op_type;
999 ad = &active_devs[dev_id];
1001 /* Check if device supports op_type */
1002 if (!is_avail_op(ad, test_vector.op_type))
1003 return TEST_SUCCESS;
1005 struct rte_bbdev_info info;
1006 rte_bbdev_info_get(ad->dev_id, &info);
1007 socket_id = GET_SOCKET(info.socket_id);
1009 if (op_type == RTE_BBDEV_OP_NONE)
1010 op_type = RTE_BBDEV_OP_TURBO_ENC;
1011 f_ret = create_mempools(ad, socket_id, op_type,
1013 if (f_ret != TEST_SUCCESS) {
1014 printf("Couldn't create mempools");
1018 f_ret = init_test_op_params(op_params, test_vector.op_type,
1019 test_vector.expected_status,
1025 if (f_ret != TEST_SUCCESS) {
1026 printf("Couldn't init test op params");
1030 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1031 create_reference_dec_op(op_params->ref_dec_op);
1032 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1033 create_reference_enc_op(op_params->ref_enc_op);
1035 for (i = 0; i < ad->nb_queues; ++i) {
1036 f_ret = fill_queue_buffers(op_params,
1038 ad->hard_out_mbuf_pool,
1039 ad->soft_out_mbuf_pool,
1041 info.drv.min_alignment,
1043 if (f_ret != TEST_SUCCESS) {
1044 printf("Couldn't init queue buffers");
1049 /* Run test case function */
1050 t_ret = test_case_func(ad, op_params);
1052 /* Free active device resources and return */
1053 free_buffers(ad, op_params);
1057 free_buffers(ad, op_params);
1061 /* Run given test function per active device per supported op type
1065 run_test_case(test_case_function *test_case_func)
1070 /* Alloc op_params */
1071 struct test_op_params *op_params = rte_zmalloc(NULL,
1072 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1073 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1074 RTE_ALIGN(sizeof(struct test_op_params),
1075 RTE_CACHE_LINE_SIZE));
1077 /* For each device run test case function */
1078 for (dev = 0; dev < nb_active_devs; ++dev)
1079 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1081 rte_free(op_params);
1087 dequeue_event_callback(uint16_t dev_id,
1088 enum rte_bbdev_event_type event, void *cb_arg,
1093 uint64_t total_time;
1094 uint16_t deq, burst_sz, num_to_process;
1095 uint16_t queue_id = INVALID_QUEUE_ID;
1096 struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
1097 struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
1098 struct test_buffers *bufs;
1099 struct rte_bbdev_info info;
1101 /* Input length in bytes, million operations per second,
1102 * million bits per second.
1106 struct thread_params *tp = cb_arg;
1108 RTE_SET_USED(ret_param);
1109 queue_id = tp->queue_id;
1111 /* Find matching thread params using queue_id */
1112 for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1113 if (tp->queue_id == queue_id)
1116 if (i == MAX_QUEUES) {
1117 printf("%s: Queue_id from interrupt details was not found!\n",
1122 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1123 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1125 "Dequeue interrupt handler called for incorrect event!\n");
1129 burst_sz = tp->op_params->burst_sz;
1130 num_to_process = tp->op_params->num_to_process;
1132 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1133 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops,
1136 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops,
1139 if (deq < burst_sz) {
1141 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1143 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1147 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) {
1148 rte_atomic16_add(&tp->nb_dequeued, deq);
1152 total_time = rte_rdtsc_precise() - tp->start_time;
1154 rte_bbdev_info_get(dev_id, &info);
1156 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1159 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1160 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs,
1162 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1163 ret = validate_enc_buffers(bufs, num_to_process);
1166 printf("Buffers validation failed\n");
1167 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1170 switch (test_vector.op_type) {
1171 case RTE_BBDEV_OP_TURBO_DEC:
1172 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length;
1174 case RTE_BBDEV_OP_TURBO_ENC:
1175 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length;
1177 case RTE_BBDEV_OP_NONE:
1181 printf("Unknown op type: %d\n", test_vector.op_type);
1182 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1186 tp->mops = ((double)num_to_process / 1000000.0) /
1187 ((double)total_time / (double)rte_get_tsc_hz());
1188 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1189 ((double)total_time / (double)rte_get_tsc_hz());
1191 rte_atomic16_add(&tp->nb_dequeued, deq);
1195 throughput_intr_lcore_dec(void *arg)
1197 struct thread_params *tp = arg;
1198 unsigned int enqueued;
1199 struct rte_bbdev_dec_op *ops[MAX_BURST];
1200 const uint16_t queue_id = tp->queue_id;
1201 const uint16_t burst_sz = tp->op_params->burst_sz;
1202 const uint16_t num_to_process = tp->op_params->num_to_process;
1203 struct test_buffers *bufs = NULL;
1204 unsigned int allocs_failed = 0;
1205 struct rte_bbdev_info info;
1208 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1209 "BURST_SIZE should be <= %u", MAX_BURST);
1211 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1212 "Failed to enable interrupts for dev: %u, queue_id: %u",
1213 tp->dev_id, queue_id);
1215 rte_bbdev_info_get(tp->dev_id, &info);
1216 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1218 rte_atomic16_clear(&tp->processing_status);
1219 rte_atomic16_clear(&tp->nb_dequeued);
1221 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1224 tp->start_time = rte_rdtsc_precise();
1225 for (enqueued = 0; enqueued < num_to_process;) {
1227 uint16_t num_to_enq = burst_sz;
1229 if (unlikely(num_to_process - enqueued < num_to_enq))
1230 num_to_enq = num_to_process - enqueued;
1232 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1239 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1240 copy_reference_dec_op(ops, num_to_enq, enqueued,
1244 tp->op_params->ref_dec_op);
1246 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops,
1249 rte_bbdev_dec_op_free_bulk(ops, num_to_enq);
1252 if (allocs_failed > 0)
1253 printf("WARNING: op allocations failed: %u times\n",
1256 return TEST_SUCCESS;
1260 throughput_intr_lcore_enc(void *arg)
1262 struct thread_params *tp = arg;
1263 unsigned int enqueued;
1264 struct rte_bbdev_enc_op *ops[MAX_BURST];
1265 const uint16_t queue_id = tp->queue_id;
1266 const uint16_t burst_sz = tp->op_params->burst_sz;
1267 const uint16_t num_to_process = tp->op_params->num_to_process;
1268 struct test_buffers *bufs = NULL;
1269 unsigned int allocs_failed = 0;
1270 struct rte_bbdev_info info;
1273 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1274 "BURST_SIZE should be <= %u", MAX_BURST);
1276 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1277 "Failed to enable interrupts for dev: %u, queue_id: %u",
1278 tp->dev_id, queue_id);
1280 rte_bbdev_info_get(tp->dev_id, &info);
1281 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1283 rte_atomic16_clear(&tp->processing_status);
1284 rte_atomic16_clear(&tp->nb_dequeued);
1286 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1289 tp->start_time = rte_rdtsc_precise();
1290 for (enqueued = 0; enqueued < num_to_process;) {
1292 uint16_t num_to_enq = burst_sz;
1294 if (unlikely(num_to_process - enqueued < num_to_enq))
1295 num_to_enq = num_to_process - enqueued;
1297 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1304 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1305 copy_reference_enc_op(ops, num_to_enq, enqueued,
1308 tp->op_params->ref_enc_op);
1310 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops,
1313 rte_bbdev_enc_op_free_bulk(ops, num_to_enq);
1316 if (allocs_failed > 0)
1317 printf("WARNING: op allocations failed: %u times\n",
1320 return TEST_SUCCESS;
1324 throughput_pmd_lcore_dec(void *arg)
1326 struct thread_params *tp = arg;
1327 unsigned int enqueued, dequeued;
1328 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1329 uint64_t total_time, start_time;
1330 const uint16_t queue_id = tp->queue_id;
1331 const uint16_t burst_sz = tp->op_params->burst_sz;
1332 const uint16_t num_to_process = tp->op_params->num_to_process;
1333 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1334 struct test_buffers *bufs = NULL;
1335 unsigned int allocs_failed = 0;
1337 struct rte_bbdev_info info;
1339 /* Input length in bytes, million operations per second, million bits
1344 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1345 "BURST_SIZE should be <= %u", MAX_BURST);
1347 rte_bbdev_info_get(tp->dev_id, &info);
1348 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1350 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1353 start_time = rte_rdtsc_precise();
1354 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1357 if (likely(enqueued < num_to_process)) {
1359 uint16_t num_to_enq = burst_sz;
1361 if (unlikely(num_to_process - enqueued < num_to_enq))
1362 num_to_enq = num_to_process - enqueued;
1364 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp,
1365 ops_enq, num_to_enq);
1371 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1372 copy_reference_dec_op(ops_enq, num_to_enq,
1379 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1380 queue_id, ops_enq, num_to_enq);
1383 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq,
1386 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1388 total_time = rte_rdtsc_precise() - start_time;
1390 if (allocs_failed > 0)
1391 printf("WARNING: op allocations failed: %u times\n",
1394 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1395 enqueued, dequeued);
1397 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1398 ret = validate_dec_buffers(ref_op, bufs, num_to_process);
1399 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1402 in_len = ref_op->turbo_dec.input.length;
1403 tp->mops = ((double)num_to_process / 1000000.0) /
1404 ((double)total_time / (double)rte_get_tsc_hz());
1405 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1406 ((double)total_time / (double)rte_get_tsc_hz());
1408 return TEST_SUCCESS;
1412 throughput_pmd_lcore_enc(void *arg)
1414 struct thread_params *tp = arg;
1415 unsigned int enqueued, dequeued;
1416 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1417 uint64_t total_time, start_time;
1418 const uint16_t queue_id = tp->queue_id;
1419 const uint16_t burst_sz = tp->op_params->burst_sz;
1420 const uint16_t num_to_process = tp->op_params->num_to_process;
1421 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1422 struct test_buffers *bufs = NULL;
1423 unsigned int allocs_failed = 0;
1425 struct rte_bbdev_info info;
1427 /* Input length in bytes, million operations per second, million bits
1432 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1433 "BURST_SIZE should be <= %u", MAX_BURST);
1435 rte_bbdev_info_get(tp->dev_id, &info);
1436 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1438 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1441 start_time = rte_rdtsc_precise();
1442 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1445 if (likely(enqueued < num_to_process)) {
1447 uint16_t num_to_enq = burst_sz;
1449 if (unlikely(num_to_process - enqueued < num_to_enq))
1450 num_to_enq = num_to_process - enqueued;
1452 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp,
1453 ops_enq, num_to_enq);
1459 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1460 copy_reference_enc_op(ops_enq, num_to_enq,
1466 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1467 queue_id, ops_enq, num_to_enq);
1470 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq,
1473 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1475 total_time = rte_rdtsc_precise() - start_time;
1477 if (allocs_failed > 0)
1478 printf("WARNING: op allocations failed: %u times\n",
1481 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1482 enqueued, dequeued);
1484 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1485 ret = validate_enc_buffers(bufs, num_to_process);
1486 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1489 in_len = ref_op->turbo_enc.input.length;
1491 tp->mops = ((double)num_to_process / 1000000.0) /
1492 ((double)total_time / (double)rte_get_tsc_hz());
1493 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1494 ((double)total_time / (double)rte_get_tsc_hz());
1496 return TEST_SUCCESS;
1499 print_throughput(struct thread_params *t_params, unsigned int used_cores)
1501 unsigned int lcore_id, iter = 0;
1502 double total_mops = 0, total_mbps = 0;
1504 RTE_LCORE_FOREACH(lcore_id) {
1505 if (iter++ >= used_cores)
1507 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n",
1508 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps);
1509 total_mops += t_params[lcore_id].mops;
1510 total_mbps += t_params[lcore_id].mbps;
1513 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n",
1514 used_cores, total_mops, total_mbps);
1518 * Test function that determines how long an enqueue + dequeue of a burst
1519 * takes on available lcores.
1522 throughput_test(struct active_device *ad,
1523 struct test_op_params *op_params)
1526 unsigned int lcore_id, used_cores = 0;
1527 struct thread_params t_params[MAX_QUEUES];
1528 struct rte_bbdev_info info;
1529 lcore_function_t *throughput_function;
1530 struct thread_params *tp;
1531 uint16_t num_lcores;
1532 const char *op_type_str;
1534 rte_bbdev_info_get(ad->dev_id, &info);
1536 op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
1537 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
1538 test_vector.op_type);
1541 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n",
1542 info.dev_name, ad->nb_queues, op_params->burst_sz,
1543 op_params->num_to_process, op_params->num_lcores,
1545 intr_enabled ? "Interrupt mode" : "PMD mode",
1546 (double)rte_get_tsc_hz() / 1000000000.0);
1548 /* Set number of lcores */
1549 num_lcores = (ad->nb_queues < (op_params->num_lcores))
1551 : op_params->num_lcores;
1554 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1555 throughput_function = throughput_intr_lcore_dec;
1557 throughput_function = throughput_intr_lcore_enc;
1559 /* Dequeue interrupt callback registration */
1560 ret = rte_bbdev_callback_register(ad->dev_id,
1561 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
1566 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1567 throughput_function = throughput_pmd_lcore_dec;
1569 throughput_function = throughput_pmd_lcore_enc;
1572 rte_atomic16_set(&op_params->sync, SYNC_WAIT);
1574 t_params[rte_lcore_id()].dev_id = ad->dev_id;
1575 t_params[rte_lcore_id()].op_params = op_params;
1576 t_params[rte_lcore_id()].queue_id =
1577 ad->queue_ids[used_cores++];
1579 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1580 if (used_cores >= num_lcores)
1583 t_params[lcore_id].dev_id = ad->dev_id;
1584 t_params[lcore_id].op_params = op_params;
1585 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++];
1587 rte_eal_remote_launch(throughput_function, &t_params[lcore_id],
1591 rte_atomic16_set(&op_params->sync, SYNC_START);
1592 ret = throughput_function(&t_params[rte_lcore_id()]);
1594 /* Master core is always used */
1596 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1597 if (used_cores++ >= num_lcores)
1600 ret |= rte_eal_wait_lcore(lcore_id);
1603 /* Return if test failed */
1607 /* Print throughput if interrupts are disabled and test passed */
1608 if (!intr_enabled) {
1609 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1610 print_throughput(t_params, num_lcores);
1614 /* In interrupt TC we need to wait for the interrupt callback to deqeue
1615 * all pending operations. Skip waiting for queues which reported an
1616 * error using processing_status variable.
1617 * Wait for master lcore operations.
1619 tp = &t_params[rte_lcore_id()];
1620 while ((rte_atomic16_read(&tp->nb_dequeued) <
1621 op_params->num_to_process) &&
1622 (rte_atomic16_read(&tp->processing_status) !=
1626 ret |= rte_atomic16_read(&tp->processing_status);
1628 /* Wait for slave lcores operations */
1630 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1631 tp = &t_params[lcore_id];
1632 if (used_cores++ >= num_lcores)
1635 while ((rte_atomic16_read(&tp->nb_dequeued) <
1636 op_params->num_to_process) &&
1637 (rte_atomic16_read(&tp->processing_status) !=
1641 ret |= rte_atomic16_read(&tp->processing_status);
1644 /* Print throughput if test passed */
1645 if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE)
1646 print_throughput(t_params, num_lcores);
1652 operation_latency_test_dec(struct rte_mempool *mempool,
1653 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
1654 int vector_mask, uint16_t dev_id, uint16_t queue_id,
1655 const uint16_t num_to_process, uint16_t burst_sz,
1656 uint64_t *total_time)
1658 int ret = TEST_SUCCESS;
1659 uint16_t i, j, dequeued;
1660 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1661 uint64_t start_time = 0;
1663 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1664 uint16_t enq = 0, deq = 0;
1665 bool first_time = true;
1667 if (unlikely(num_to_process - dequeued < burst_sz))
1668 burst_sz = num_to_process - dequeued;
1670 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1671 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1672 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1678 /* Set counter to validate the ordering */
1679 for (j = 0; j < burst_sz; ++j)
1680 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1682 start_time = rte_rdtsc_precise();
1684 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
1686 TEST_ASSERT(enq == burst_sz,
1687 "Error enqueueing burst, expected %u, got %u",
1692 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1693 &ops_deq[deq], burst_sz - deq);
1694 if (likely(first_time && (deq > 0))) {
1695 *total_time += rte_rdtsc_precise() - start_time;
1698 } while (unlikely(burst_sz != deq));
1700 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1701 ret = validate_dec_op(ops_deq, burst_sz, ref_op,
1703 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1706 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1714 operation_latency_test_enc(struct rte_mempool *mempool,
1715 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
1716 uint16_t dev_id, uint16_t queue_id,
1717 const uint16_t num_to_process, uint16_t burst_sz,
1718 uint64_t *total_time)
1720 int ret = TEST_SUCCESS;
1721 uint16_t i, j, dequeued;
1722 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1723 uint64_t start_time = 0;
1725 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1726 uint16_t enq = 0, deq = 0;
1727 bool first_time = true;
1729 if (unlikely(num_to_process - dequeued < burst_sz))
1730 burst_sz = num_to_process - dequeued;
1732 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1733 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1734 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1739 /* Set counter to validate the ordering */
1740 for (j = 0; j < burst_sz; ++j)
1741 ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1743 start_time = rte_rdtsc_precise();
1745 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
1747 TEST_ASSERT(enq == burst_sz,
1748 "Error enqueueing burst, expected %u, got %u",
1753 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1754 &ops_deq[deq], burst_sz - deq);
1755 if (likely(first_time && (deq > 0))) {
1756 *total_time += rte_rdtsc_precise() - start_time;
1759 } while (unlikely(burst_sz != deq));
1761 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1762 ret = validate_enc_op(ops_deq, burst_sz, ref_op);
1763 TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1766 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1774 operation_latency_test(struct active_device *ad,
1775 struct test_op_params *op_params)
1778 uint16_t burst_sz = op_params->burst_sz;
1779 const uint16_t num_to_process = op_params->num_to_process;
1780 const enum rte_bbdev_op_type op_type = test_vector.op_type;
1781 const uint16_t queue_id = ad->queue_ids[0];
1782 struct test_buffers *bufs = NULL;
1783 struct rte_bbdev_info info;
1784 uint64_t total_time = 0;
1785 const char *op_type_str;
1787 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1788 "BURST_SIZE should be <= %u", MAX_BURST);
1790 rte_bbdev_info_get(ad->dev_id, &info);
1791 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1793 op_type_str = rte_bbdev_op_type_str(op_type);
1794 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1797 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1798 info.dev_name, burst_sz, num_to_process, op_type_str);
1800 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1801 iter = operation_latency_test_dec(op_params->mp, bufs,
1802 op_params->ref_dec_op, op_params->vector_mask,
1803 ad->dev_id, queue_id, num_to_process,
1804 burst_sz, &total_time);
1806 iter = operation_latency_test_enc(op_params->mp, bufs,
1807 op_params->ref_enc_op, ad->dev_id, queue_id,
1808 num_to_process, burst_sz, &total_time);
1813 printf("\toperation avg. latency: %lg cycles, %lg us\n",
1814 (double)total_time / (double)iter,
1815 (double)(total_time * 1000000) / (double)iter /
1816 (double)rte_get_tsc_hz());
1818 return TEST_SUCCESS;
1822 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
1823 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
1824 uint16_t queue_id, const uint16_t num_to_process,
1825 uint16_t burst_sz, uint64_t *enq_total_time,
1826 uint64_t *deq_total_time)
1829 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1830 uint64_t enq_start_time, deq_start_time;
1832 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1833 uint16_t enq = 0, deq = 0;
1835 if (unlikely(num_to_process - dequeued < burst_sz))
1836 burst_sz = num_to_process - dequeued;
1838 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1839 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1840 copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1846 /* Start time measurment for enqueue function offload latency */
1847 enq_start_time = rte_rdtsc();
1849 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
1850 &ops_enq[enq], burst_sz - enq);
1851 } while (unlikely(burst_sz != enq));
1852 *enq_total_time += rte_rdtsc() - enq_start_time;
1854 /* ensure enqueue has been completed */
1857 /* Start time measurment for dequeue function offload latency */
1858 deq_start_time = rte_rdtsc();
1860 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1861 &ops_deq[deq], burst_sz - deq);
1862 } while (unlikely(burst_sz != deq));
1863 *deq_total_time += rte_rdtsc() - deq_start_time;
1865 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1873 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
1874 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
1875 uint16_t queue_id, const uint16_t num_to_process,
1876 uint16_t burst_sz, uint64_t *enq_total_time,
1877 uint64_t *deq_total_time)
1880 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1881 uint64_t enq_start_time, deq_start_time;
1883 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1884 uint16_t enq = 0, deq = 0;
1886 if (unlikely(num_to_process - dequeued < burst_sz))
1887 burst_sz = num_to_process - dequeued;
1889 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1890 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1891 copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1896 /* Start time measurment for enqueue function offload latency */
1897 enq_start_time = rte_rdtsc();
1899 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
1900 &ops_enq[enq], burst_sz - enq);
1901 } while (unlikely(burst_sz != enq));
1902 *enq_total_time += rte_rdtsc() - enq_start_time;
1904 /* ensure enqueue has been completed */
1907 /* Start time measurment for dequeue function offload latency */
1908 deq_start_time = rte_rdtsc();
1910 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1911 &ops_deq[deq], burst_sz - deq);
1912 } while (unlikely(burst_sz != deq));
1913 *deq_total_time += rte_rdtsc() - deq_start_time;
1915 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1923 offload_latency_test(struct active_device *ad,
1924 struct test_op_params *op_params)
1927 uint64_t enq_total_time = 0, deq_total_time = 0;
1928 uint16_t burst_sz = op_params->burst_sz;
1929 const uint16_t num_to_process = op_params->num_to_process;
1930 const enum rte_bbdev_op_type op_type = test_vector.op_type;
1931 const uint16_t queue_id = ad->queue_ids[0];
1932 struct test_buffers *bufs = NULL;
1933 struct rte_bbdev_info info;
1934 const char *op_type_str;
1936 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1937 "BURST_SIZE should be <= %u", MAX_BURST);
1939 rte_bbdev_info_get(ad->dev_id, &info);
1940 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1942 op_type_str = rte_bbdev_op_type_str(op_type);
1943 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1946 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1947 info.dev_name, burst_sz, num_to_process, op_type_str);
1949 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1950 iter = offload_latency_test_dec(op_params->mp, bufs,
1951 op_params->ref_dec_op, ad->dev_id, queue_id,
1952 num_to_process, burst_sz, &enq_total_time,
1955 iter = offload_latency_test_enc(op_params->mp, bufs,
1956 op_params->ref_enc_op, ad->dev_id, queue_id,
1957 num_to_process, burst_sz, &enq_total_time,
1963 printf("\tenq offload avg. latency: %lg cycles, %lg us\n",
1964 (double)enq_total_time / (double)iter,
1965 (double)(enq_total_time * 1000000) / (double)iter /
1966 (double)rte_get_tsc_hz());
1968 printf("\tdeq offload avg. latency: %lg cycles, %lg us\n",
1969 (double)deq_total_time / (double)iter,
1970 (double)(deq_total_time * 1000000) / (double)iter /
1971 (double)rte_get_tsc_hz());
1973 return TEST_SUCCESS;
1977 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
1978 const uint16_t num_to_process, uint16_t burst_sz,
1979 uint64_t *deq_total_time)
1982 struct rte_bbdev_dec_op *ops[MAX_BURST];
1983 uint64_t deq_start_time;
1985 /* Test deq offload latency from an empty queue */
1986 deq_start_time = rte_rdtsc_precise();
1987 for (i = 0, deq_total = 0; deq_total < num_to_process;
1988 ++i, deq_total += burst_sz) {
1989 if (unlikely(num_to_process - deq_total < burst_sz))
1990 burst_sz = num_to_process - deq_total;
1991 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
1993 *deq_total_time = rte_rdtsc_precise() - deq_start_time;
1999 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
2000 const uint16_t num_to_process, uint16_t burst_sz,
2001 uint64_t *deq_total_time)
2004 struct rte_bbdev_enc_op *ops[MAX_BURST];
2005 uint64_t deq_start_time;
2007 /* Test deq offload latency from an empty queue */
2008 deq_start_time = rte_rdtsc_precise();
2009 for (i = 0, deq_total = 0; deq_total < num_to_process;
2010 ++i, deq_total += burst_sz) {
2011 if (unlikely(num_to_process - deq_total < burst_sz))
2012 burst_sz = num_to_process - deq_total;
2013 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
2015 *deq_total_time = rte_rdtsc_precise() - deq_start_time;
2021 offload_latency_empty_q_test(struct active_device *ad,
2022 struct test_op_params *op_params)
2025 uint64_t deq_total_time = 0;
2026 uint16_t burst_sz = op_params->burst_sz;
2027 const uint16_t num_to_process = op_params->num_to_process;
2028 const enum rte_bbdev_op_type op_type = test_vector.op_type;
2029 const uint16_t queue_id = ad->queue_ids[0];
2030 struct rte_bbdev_info info;
2031 const char *op_type_str;
2033 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2034 "BURST_SIZE should be <= %u", MAX_BURST);
2036 rte_bbdev_info_get(ad->dev_id, &info);
2038 op_type_str = rte_bbdev_op_type_str(op_type);
2039 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2042 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2043 info.dev_name, burst_sz, num_to_process, op_type_str);
2045 if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2046 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
2047 num_to_process, burst_sz, &deq_total_time);
2049 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
2050 num_to_process, burst_sz, &deq_total_time);
2055 printf("\tempty deq offload avg. latency: %lg cycles, %lg us\n",
2056 (double)deq_total_time / (double)iter,
2057 (double)(deq_total_time * 1000000) / (double)iter /
2058 (double)rte_get_tsc_hz());
2060 return TEST_SUCCESS;
2066 return run_test_case(throughput_test);
2070 offload_latency_tc(void)
2072 return run_test_case(offload_latency_test);
2076 offload_latency_empty_q_tc(void)
2078 return run_test_case(offload_latency_empty_q_test);
2082 operation_latency_tc(void)
2084 return run_test_case(operation_latency_test);
2090 return run_test_case(throughput_test);
2093 static struct unit_test_suite bbdev_throughput_testsuite = {
2094 .suite_name = "BBdev Throughput Tests",
2095 .setup = testsuite_setup,
2096 .teardown = testsuite_teardown,
2097 .unit_test_cases = {
2098 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
2099 TEST_CASES_END() /**< NULL terminate unit test array */
2103 static struct unit_test_suite bbdev_validation_testsuite = {
2104 .suite_name = "BBdev Validation Tests",
2105 .setup = testsuite_setup,
2106 .teardown = testsuite_teardown,
2107 .unit_test_cases = {
2108 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc),
2109 TEST_CASES_END() /**< NULL terminate unit test array */
2113 static struct unit_test_suite bbdev_latency_testsuite = {
2114 .suite_name = "BBdev Latency Tests",
2115 .setup = testsuite_setup,
2116 .teardown = testsuite_teardown,
2117 .unit_test_cases = {
2118 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_tc),
2119 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
2120 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc),
2121 TEST_CASES_END() /**< NULL terminate unit test array */
2125 static struct unit_test_suite bbdev_interrupt_testsuite = {
2126 .suite_name = "BBdev Interrupt Tests",
2127 .setup = interrupt_testsuite_setup,
2128 .teardown = testsuite_teardown,
2129 .unit_test_cases = {
2130 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
2131 TEST_CASES_END() /**< NULL terminate unit test array */
2135 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
2136 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
2137 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
2138 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);