+ } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) {
+ struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
+ ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op);
+ rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq);
+ } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {
+ struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
+ ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op,
+ tp->op_params->vector_mask);
+ rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq);
+ }
+
+ if (ret) {
+ printf("Buffers validation failed\n");
+ __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
+ }
+
+ switch (test_vector.op_type) {
+ case RTE_BBDEV_OP_TURBO_DEC:
+ tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op);
+ break;
+ case RTE_BBDEV_OP_TURBO_ENC:
+ tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op);
+ break;
+ case RTE_BBDEV_OP_LDPC_DEC:
+ tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op);
+ break;
+ case RTE_BBDEV_OP_LDPC_ENC:
+ tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op);
+ break;
+ case RTE_BBDEV_OP_NONE:
+ tb_len_bits = 0.0;
+ break;
+ default:
+ printf("Unknown op type: %d\n", test_vector.op_type);
+ __atomic_store_n(&tp->processing_status, TEST_FAILED, __ATOMIC_RELAXED);
+ return;
+ }
+
+ tp->ops_per_sec += ((double)num_ops) /
+ ((double)total_time / (double)rte_get_tsc_hz());
+ tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) /
+ ((double)total_time / (double)rte_get_tsc_hz());
+
+ __atomic_fetch_add(&tp->nb_dequeued, deq, __ATOMIC_RELAXED);
+}
+
+static int
+throughput_intr_lcore_ldpc_dec(void *arg)
+{
+ struct thread_params *tp = arg;
+ unsigned int enqueued;
+ const uint16_t queue_id = tp->queue_id;
+ const uint16_t burst_sz = tp->op_params->burst_sz;
+ const uint16_t num_to_process = tp->op_params->num_to_process;
+ struct rte_bbdev_dec_op *ops[num_to_process];
+ struct test_buffers *bufs = NULL;
+ struct rte_bbdev_info info;
+ int ret, i, j;
+ struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
+ uint16_t num_to_enq, enq;
+
+ bool loopback = check_bit(ref_op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK);
+ bool hc_out = check_bit(ref_op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);
+
+ TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
+ "BURST_SIZE should be <= %u", MAX_BURST);
+
+ TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
+ "Failed to enable interrupts for dev: %u, queue_id: %u",
+ tp->dev_id, queue_id);
+
+ rte_bbdev_info_get(tp->dev_id, &info);
+
+ TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
+ "NUM_OPS cannot exceed %u for this device",
+ info.drv.queue_size_lim);
+
+ bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
+
+ __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
+
+ rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
+
+ ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
+ num_to_process);
+ TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
+ num_to_process);
+ if (test_vector.op_type != RTE_BBDEV_OP_NONE)
+ copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs,
+ bufs->hard_outputs, bufs->soft_outputs,
+ bufs->harq_inputs, bufs->harq_outputs, ref_op);
+
+ /* Set counter to validate the ordering */
+ for (j = 0; j < num_to_process; ++j)
+ ops[j]->opaque_data = (void *)(uintptr_t)j;
+
+ for (j = 0; j < TEST_REPETITIONS; ++j) {
+ for (i = 0; i < num_to_process; ++i) {
+ if (!loopback)
+ rte_pktmbuf_reset(
+ ops[i]->ldpc_dec.hard_output.data);
+ if (hc_out || loopback)
+ mbuf_reset(
+ ops[i]->ldpc_dec.harq_combined_output.data);
+ }
+
+ tp->start_time = rte_rdtsc_precise();
+ for (enqueued = 0; enqueued < num_to_process;) {
+ num_to_enq = burst_sz;
+
+ if (unlikely(num_to_process - enqueued < num_to_enq))
+ num_to_enq = num_to_process - enqueued;
+
+ enq = 0;
+ do {
+ enq += rte_bbdev_enqueue_ldpc_dec_ops(
+ tp->dev_id,
+ queue_id, &ops[enqueued],
+ num_to_enq);
+ } while (unlikely(num_to_enq != enq));
+ enqueued += enq;
+
+ /* Write to thread burst_sz current number of enqueued
+ * descriptors. It ensures that proper number of
+ * descriptors will be dequeued in callback
+ * function - needed for last batch in case where
+ * the number of operations is not a multiple of
+ * burst size.
+ */
+ __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
+
+ /* Wait until processing of previous batch is
+ * completed
+ */
+ rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
+ }
+ if (j != TEST_REPETITIONS - 1)
+ __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+throughput_intr_lcore_dec(void *arg)
+{
+ struct thread_params *tp = arg;
+ unsigned int enqueued;
+ const uint16_t queue_id = tp->queue_id;
+ const uint16_t burst_sz = tp->op_params->burst_sz;
+ const uint16_t num_to_process = tp->op_params->num_to_process;
+ struct rte_bbdev_dec_op *ops[num_to_process];
+ struct test_buffers *bufs = NULL;
+ struct rte_bbdev_info info;
+ int ret, i, j;
+ uint16_t num_to_enq, enq;
+
+ TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
+ "BURST_SIZE should be <= %u", MAX_BURST);
+
+ TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
+ "Failed to enable interrupts for dev: %u, queue_id: %u",
+ tp->dev_id, queue_id);
+
+ rte_bbdev_info_get(tp->dev_id, &info);
+
+ TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
+ "NUM_OPS cannot exceed %u for this device",
+ info.drv.queue_size_lim);
+
+ bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
+
+ __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
+
+ rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
+
+ ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
+ num_to_process);
+ TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
+ num_to_process);
+ if (test_vector.op_type != RTE_BBDEV_OP_NONE)
+ copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs,
+ bufs->hard_outputs, bufs->soft_outputs,
+ tp->op_params->ref_dec_op);
+
+ /* Set counter to validate the ordering */
+ for (j = 0; j < num_to_process; ++j)
+ ops[j]->opaque_data = (void *)(uintptr_t)j;
+
+ for (j = 0; j < TEST_REPETITIONS; ++j) {
+ for (i = 0; i < num_to_process; ++i)
+ rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data);
+
+ tp->start_time = rte_rdtsc_precise();
+ for (enqueued = 0; enqueued < num_to_process;) {
+ num_to_enq = burst_sz;
+
+ if (unlikely(num_to_process - enqueued < num_to_enq))
+ num_to_enq = num_to_process - enqueued;
+
+ enq = 0;
+ do {
+ enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
+ queue_id, &ops[enqueued],
+ num_to_enq);
+ } while (unlikely(num_to_enq != enq));
+ enqueued += enq;
+
+ /* Write to thread burst_sz current number of enqueued
+ * descriptors. It ensures that proper number of
+ * descriptors will be dequeued in callback
+ * function - needed for last batch in case where
+ * the number of operations is not a multiple of
+ * burst size.
+ */
+ __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
+
+ /* Wait until processing of previous batch is
+ * completed
+ */
+ rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
+ }
+ if (j != TEST_REPETITIONS - 1)
+ __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+throughput_intr_lcore_enc(void *arg)
+{
+ struct thread_params *tp = arg;
+ unsigned int enqueued;
+ const uint16_t queue_id = tp->queue_id;
+ const uint16_t burst_sz = tp->op_params->burst_sz;
+ const uint16_t num_to_process = tp->op_params->num_to_process;
+ struct rte_bbdev_enc_op *ops[num_to_process];
+ struct test_buffers *bufs = NULL;
+ struct rte_bbdev_info info;
+ int ret, i, j;
+ uint16_t num_to_enq, enq;
+
+ TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
+ "BURST_SIZE should be <= %u", MAX_BURST);
+
+ TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
+ "Failed to enable interrupts for dev: %u, queue_id: %u",
+ tp->dev_id, queue_id);
+
+ rte_bbdev_info_get(tp->dev_id, &info);
+
+ TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
+ "NUM_OPS cannot exceed %u for this device",
+ info.drv.queue_size_lim);
+
+ bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
+
+ __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
+
+ rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
+
+ ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
+ num_to_process);
+ TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
+ num_to_process);
+ if (test_vector.op_type != RTE_BBDEV_OP_NONE)
+ copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs,
+ bufs->hard_outputs, tp->op_params->ref_enc_op);
+
+ /* Set counter to validate the ordering */
+ for (j = 0; j < num_to_process; ++j)
+ ops[j]->opaque_data = (void *)(uintptr_t)j;
+
+ for (j = 0; j < TEST_REPETITIONS; ++j) {
+ for (i = 0; i < num_to_process; ++i)
+ rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
+
+ tp->start_time = rte_rdtsc_precise();
+ for (enqueued = 0; enqueued < num_to_process;) {
+ num_to_enq = burst_sz;
+
+ if (unlikely(num_to_process - enqueued < num_to_enq))
+ num_to_enq = num_to_process - enqueued;
+
+ enq = 0;
+ do {
+ enq += rte_bbdev_enqueue_enc_ops(tp->dev_id,
+ queue_id, &ops[enqueued],
+ num_to_enq);
+ } while (unlikely(enq != num_to_enq));
+ enqueued += enq;
+
+ /* Write to thread burst_sz current number of enqueued
+ * descriptors. It ensures that proper number of
+ * descriptors will be dequeued in callback
+ * function - needed for last batch in case where
+ * the number of operations is not a multiple of
+ * burst size.
+ */
+ __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
+
+ /* Wait until processing of previous batch is
+ * completed
+ */
+ rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
+ }
+ if (j != TEST_REPETITIONS - 1)
+ __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
+ }
+
+ return TEST_SUCCESS;
+}
+
+
+static int
+throughput_intr_lcore_ldpc_enc(void *arg)
+{
+ struct thread_params *tp = arg;
+ unsigned int enqueued;
+ const uint16_t queue_id = tp->queue_id;
+ const uint16_t burst_sz = tp->op_params->burst_sz;
+ const uint16_t num_to_process = tp->op_params->num_to_process;
+ struct rte_bbdev_enc_op *ops[num_to_process];
+ struct test_buffers *bufs = NULL;
+ struct rte_bbdev_info info;
+ int ret, i, j;
+ uint16_t num_to_enq, enq;
+
+ TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
+ "BURST_SIZE should be <= %u", MAX_BURST);
+
+ TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
+ "Failed to enable interrupts for dev: %u, queue_id: %u",
+ tp->dev_id, queue_id);
+
+ rte_bbdev_info_get(tp->dev_id, &info);
+
+ TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim),
+ "NUM_OPS cannot exceed %u for this device",
+ info.drv.queue_size_lim);
+
+ bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
+
+ __atomic_store_n(&tp->processing_status, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
+
+ rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
+
+ ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
+ num_to_process);
+ TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops",
+ num_to_process);
+ if (test_vector.op_type != RTE_BBDEV_OP_NONE)
+ copy_reference_ldpc_enc_op(ops, num_to_process, 0,
+ bufs->inputs, bufs->hard_outputs,
+ tp->op_params->ref_enc_op);
+
+ /* Set counter to validate the ordering */
+ for (j = 0; j < num_to_process; ++j)
+ ops[j]->opaque_data = (void *)(uintptr_t)j;
+
+ for (j = 0; j < TEST_REPETITIONS; ++j) {
+ for (i = 0; i < num_to_process; ++i)
+ rte_pktmbuf_reset(ops[i]->turbo_enc.output.data);
+
+ tp->start_time = rte_rdtsc_precise();
+ for (enqueued = 0; enqueued < num_to_process;) {
+ num_to_enq = burst_sz;
+
+ if (unlikely(num_to_process - enqueued < num_to_enq))
+ num_to_enq = num_to_process - enqueued;
+
+ enq = 0;
+ do {
+ enq += rte_bbdev_enqueue_ldpc_enc_ops(
+ tp->dev_id,
+ queue_id, &ops[enqueued],
+ num_to_enq);
+ } while (unlikely(enq != num_to_enq));
+ enqueued += enq;
+
+ /* Write to thread burst_sz current number of enqueued
+ * descriptors. It ensures that proper number of
+ * descriptors will be dequeued in callback
+ * function - needed for last batch in case where
+ * the number of operations is not a multiple of
+ * burst size.
+ */
+ __atomic_store_n(&tp->burst_sz, num_to_enq, __ATOMIC_RELAXED);
+
+ /* Wait until processing of previous batch is
+ * completed
+ */
+ rte_wait_until_equal_16(&tp->nb_dequeued, enqueued, __ATOMIC_RELAXED);
+ }
+ if (j != TEST_REPETITIONS - 1)
+ __atomic_store_n(&tp->nb_dequeued, 0, __ATOMIC_RELAXED);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+throughput_pmd_lcore_dec(void *arg)
+{
+ struct thread_params *tp = arg;
+ uint16_t enq, deq;
+ uint64_t total_time = 0, start_time;
+ const uint16_t queue_id = tp->queue_id;
+ const uint16_t burst_sz = tp->op_params->burst_sz;
+ const uint16_t num_ops = tp->op_params->num_to_process;
+ struct rte_bbdev_dec_op *ops_enq[num_ops];
+ struct rte_bbdev_dec_op *ops_deq[num_ops];
+ struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
+ struct test_buffers *bufs = NULL;
+ int i, j, ret;
+ struct rte_bbdev_info info;
+ uint16_t num_to_enq;
+
+ TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
+ "BURST_SIZE should be <= %u", MAX_BURST);
+
+ rte_bbdev_info_get(tp->dev_id, &info);
+
+ TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim),
+ "NUM_OPS cannot exceed %u for this device",
+ info.drv.queue_size_lim);
+
+ bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
+
+ rte_wait_until_equal_16(&tp->op_params->sync, SYNC_START, __ATOMIC_RELAXED);
+
+ ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
+ TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
+
+ if (test_vector.op_type != RTE_BBDEV_OP_NONE)
+ copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs,
+ bufs->hard_outputs, bufs->soft_outputs, ref_op);
+
+ /* Set counter to validate the ordering */
+ for (j = 0; j < num_ops; ++j)
+ ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
+
+ for (i = 0; i < TEST_REPETITIONS; ++i) {
+
+ for (j = 0; j < num_ops; ++j)
+ mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data);
+
+ start_time = rte_rdtsc_precise();
+
+ for (enq = 0, deq = 0; enq < num_ops;) {
+ num_to_enq = burst_sz;
+
+ if (unlikely(num_ops - enq < num_to_enq))
+ num_to_enq = num_ops - enq;
+
+ enq += rte_bbdev_enqueue_dec_ops(tp->dev_id,
+ queue_id, &ops_enq[enq], num_to_enq);
+
+ deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
+ queue_id, &ops_deq[deq], enq - deq);
+ }
+
+ /* dequeue the remaining */
+ while (deq < enq) {
+ deq += rte_bbdev_dequeue_dec_ops(tp->dev_id,
+ queue_id, &ops_deq[deq], enq - deq);
+ }
+
+ total_time += rte_rdtsc_precise() - start_time;