1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
5 #include <rte_malloc.h>
8 #include <rte_cycles.h>
9 #include "rte_spinlock.h"
10 #include <rte_compressdev.h>
12 #include "comp_perf_test_cyclecount.h"
14 struct cperf_cyclecount_ctx {
15 struct cperf_verify_ctx ver;
17 uint32_t ops_enq_retries;
18 uint32_t ops_deq_retries;
21 uint64_t duration_enq;
22 uint64_t duration_deq;
26 cperf_cyclecount_test_destructor(void *arg)
28 struct cperf_cyclecount_ctx *ctx = arg;
31 comp_perf_free_memory(ctx->ver.options, &ctx->ver.mem);
37 cperf_cyclecount_test_constructor(uint8_t dev_id, uint16_t qp_id,
38 struct comp_test_data *options)
40 struct cperf_cyclecount_ctx *ctx = NULL;
42 ctx = rte_malloc(NULL, sizeof(struct cperf_cyclecount_ctx), 0);
47 ctx->ver.mem.dev_id = dev_id;
48 ctx->ver.mem.qp_id = qp_id;
49 ctx->ver.options = options;
50 ctx->ver.silent = 1; /* ver. part will be silent */
52 if (!comp_perf_allocate_memory(ctx->ver.options, &ctx->ver.mem)
53 && !prepare_bufs(ctx->ver.options, &ctx->ver.mem))
56 cperf_cyclecount_test_destructor(ctx);
61 cperf_cyclecount_op_setup(struct rte_comp_op **ops,
62 struct cperf_cyclecount_ctx *ctx,
63 struct rte_mbuf **input_bufs,
64 struct rte_mbuf **output_bufs,
68 struct comp_test_data *test_data = ctx->ver.options;
69 struct cperf_mem_resources *mem = &ctx->ver.mem;
71 uint32_t i, iter, num_iter;
75 num_iter = test_data->num_iter;
77 for (iter = 0; iter < num_iter; iter++) {
78 uint32_t remaining_ops = mem->total_bufs;
79 uint32_t total_deq_ops = 0;
80 uint32_t total_enq_ops = 0;
84 while (remaining_ops > 0) {
85 uint16_t num_ops = RTE_MIN(remaining_ops,
89 /* Allocate compression operations */
90 if (ops_needed && rte_mempool_get_bulk(
95 "Cyclecount: could not allocate enough operations\n");
100 for (i = 0; i < ops_needed; i++) {
102 /* Calculate next buffer to attach */
104 uint32_t buf_id = total_enq_ops + i;
107 /* Reset all data in output buffers */
108 struct rte_mbuf *m = output_bufs[buf_id];
110 m->pkt_len = out_seg_sz * m->nb_segs;
112 m->data_len = m->buf_len - m->data_off;
115 ops[op_id]->m_src = input_bufs[buf_id];
116 ops[op_id]->m_dst = output_bufs[buf_id];
117 ops[op_id]->src.offset = 0;
118 ops[op_id]->src.length =
119 rte_pktmbuf_pkt_len(input_bufs[buf_id]);
120 ops[op_id]->dst.offset = 0;
121 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
122 ops[op_id]->input_chksum = buf_id;
123 ops[op_id]->private_xform = priv_xform;
126 /* E N Q U E U I N G */
127 /* assuming that all ops are enqueued */
128 /* instead of the real enqueue operation */
131 remaining_ops -= num_enq;
132 total_enq_ops += num_enq;
134 /* D E Q U E U I N G */
135 /* assuming that all ops dequeued */
136 /* instead of the real dequeue operation */
139 total_deq_ops += num_deq;
140 rte_mempool_put_bulk(mem->op_pool,
141 (void **)ops, num_deq);
146 rte_mempool_put_bulk(mem->op_pool, (void **)ops, ops_needed);
153 main_loop(struct cperf_cyclecount_ctx *ctx, enum rte_comp_xform_type type)
155 struct comp_test_data *test_data = ctx->ver.options;
156 struct cperf_mem_resources *mem = &ctx->ver.mem;
157 uint8_t dev_id = mem->dev_id;
158 uint32_t i, iter, num_iter;
159 struct rte_comp_op **ops, **deq_ops;
160 void *priv_xform = NULL;
161 struct rte_comp_xform xform;
162 struct rte_mbuf **input_bufs, **output_bufs;
167 uint64_t tsc_start, tsc_end, tsc_duration;
169 if (test_data == NULL || !test_data->burst_sz) {
170 RTE_LOG(ERR, USER1, "Unknown burst size\n");
173 ctx->duration_enq = 0;
174 ctx->duration_deq = 0;
175 ctx->ops_enq_retries = 0;
176 ctx->ops_deq_retries = 0;
178 /* one array for both enqueue and dequeue */
179 ops = rte_zmalloc_socket(NULL,
180 2 * mem->total_bufs * sizeof(struct rte_comp_op *),
185 "Can't allocate memory for ops strucures\n");
189 deq_ops = &ops[mem->total_bufs];
191 if (type == RTE_COMP_COMPRESS) {
192 xform = (struct rte_comp_xform) {
193 .type = RTE_COMP_COMPRESS,
195 .algo = RTE_COMP_ALGO_DEFLATE,
196 .deflate.huffman = test_data->huffman_enc,
197 .level = test_data->level,
198 .window_size = test_data->window_sz,
199 .chksum = RTE_COMP_CHECKSUM_NONE,
200 .hash_algo = RTE_COMP_HASH_ALGO_NONE
203 input_bufs = mem->decomp_bufs;
204 output_bufs = mem->comp_bufs;
205 out_seg_sz = test_data->out_seg_sz;
207 xform = (struct rte_comp_xform) {
208 .type = RTE_COMP_DECOMPRESS,
210 .algo = RTE_COMP_ALGO_DEFLATE,
211 .chksum = RTE_COMP_CHECKSUM_NONE,
212 .window_size = test_data->window_sz,
213 .hash_algo = RTE_COMP_HASH_ALGO_NONE
216 input_bufs = mem->comp_bufs;
217 output_bufs = mem->decomp_bufs;
218 out_seg_sz = test_data->seg_sz;
221 /* Create private xform */
222 if (rte_compressdev_private_xform_create(dev_id, &xform,
224 RTE_LOG(ERR, USER1, "Private xform could not be created\n");
229 tsc_start = rte_rdtsc_precise();
230 ret = cperf_cyclecount_op_setup(ops,
237 tsc_end = rte_rdtsc_precise();
239 /* ret value check postponed a bit to cancel extra 'if' bias */
241 RTE_LOG(ERR, USER1, "Setup function failed\n");
246 tsc_duration = tsc_end - tsc_start;
247 ctx->duration_op = tsc_duration;
249 num_iter = test_data->num_iter;
250 for (iter = 0; iter < num_iter; iter++) {
251 uint32_t total_ops = mem->total_bufs;
252 uint32_t remaining_ops = mem->total_bufs;
253 uint32_t total_deq_ops = 0;
254 uint32_t total_enq_ops = 0;
255 uint16_t ops_unused = 0;
256 uint16_t num_enq = 0;
257 uint16_t num_deq = 0;
259 while (remaining_ops > 0) {
260 uint16_t num_ops = RTE_MIN(remaining_ops,
261 test_data->burst_sz);
262 uint16_t ops_needed = num_ops - ops_unused;
265 * Move the unused operations from the previous
266 * enqueue_burst call to the front, to maintain order
268 if ((ops_unused > 0) && (num_enq > 0)) {
270 ops_unused * sizeof(struct rte_comp_op *);
272 memmove(ops, &ops[num_enq], nb_b_to_mov);
275 /* Allocate compression operations */
276 if (ops_needed && rte_mempool_get_bulk(
281 "Could not allocate enough operations\n");
285 allocated += ops_needed;
287 for (i = 0; i < ops_needed; i++) {
289 * Calculate next buffer to attach to operation
291 uint32_t buf_id = total_enq_ops + i +
293 uint16_t op_id = ops_unused + i;
294 /* Reset all data in output buffers */
295 struct rte_mbuf *m = output_bufs[buf_id];
297 m->pkt_len = out_seg_sz * m->nb_segs;
299 m->data_len = m->buf_len - m->data_off;
302 ops[op_id]->m_src = input_bufs[buf_id];
303 ops[op_id]->m_dst = output_bufs[buf_id];
304 ops[op_id]->src.offset = 0;
305 ops[op_id]->src.length =
306 rte_pktmbuf_pkt_len(input_bufs[buf_id]);
307 ops[op_id]->dst.offset = 0;
308 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
309 ops[op_id]->input_chksum = buf_id;
310 ops[op_id]->private_xform = priv_xform;
313 if (unlikely(test_data->perf_comp_force_stop))
316 tsc_start = rte_rdtsc_precise();
317 num_enq = rte_compressdev_enqueue_burst(dev_id,
320 tsc_end = rte_rdtsc_precise();
321 tsc_duration = tsc_end - tsc_start;
322 ctx->duration_enq += tsc_duration;
324 if (num_enq < num_ops)
325 ctx->ops_enq_retries++;
327 if (test_data->cyclecount_delay)
328 rte_delay_us_block(test_data->cyclecount_delay);
331 struct rte_compressdev_stats stats;
333 rte_compressdev_stats_get(dev_id, &stats);
334 if (stats.enqueue_err_count) {
340 ops_unused = num_ops - num_enq;
341 remaining_ops -= num_enq;
342 total_enq_ops += num_enq;
344 tsc_start = rte_rdtsc_precise();
345 num_deq = rte_compressdev_dequeue_burst(dev_id,
349 tsc_end = rte_rdtsc_precise();
350 tsc_duration = tsc_end - tsc_start;
351 ctx->duration_deq += tsc_duration;
353 if (num_deq < allocated)
354 ctx->ops_deq_retries++;
356 total_deq_ops += num_deq;
358 if (iter == num_iter - 1) {
359 for (i = 0; i < num_deq; i++) {
360 struct rte_comp_op *op = deq_ops[i];
363 RTE_COMP_OP_STATUS_SUCCESS) {
364 RTE_LOG(ERR, USER1, "Some operations were not successful\n");
368 struct rte_mbuf *m = op->m_dst;
370 m->pkt_len = op->produced;
371 uint32_t remaining_data = op->produced;
372 uint16_t data_to_append;
374 while (remaining_data > 0) {
376 RTE_MIN(remaining_data,
378 m->data_len = data_to_append;
385 rte_mempool_put_bulk(mem->op_pool,
386 (void **)deq_ops, num_deq);
387 allocated -= num_deq;
390 /* Dequeue the last operations */
391 while (total_deq_ops < total_ops) {
392 if (unlikely(test_data->perf_comp_force_stop))
395 tsc_start = rte_rdtsc_precise();
396 num_deq = rte_compressdev_dequeue_burst(dev_id,
399 test_data->burst_sz);
400 tsc_end = rte_rdtsc_precise();
401 tsc_duration = tsc_end - tsc_start;
402 ctx->duration_deq += tsc_duration;
403 ctx->ops_deq_retries++;
406 struct rte_compressdev_stats stats;
408 rte_compressdev_stats_get(dev_id, &stats);
409 if (stats.dequeue_err_count) {
414 total_deq_ops += num_deq;
416 if (iter == num_iter - 1) {
417 for (i = 0; i < num_deq; i++) {
418 struct rte_comp_op *op = deq_ops[i];
421 RTE_COMP_OP_STATUS_SUCCESS) {
422 RTE_LOG(ERR, USER1, "Some operations were not successful\n");
426 struct rte_mbuf *m = op->m_dst;
428 m->pkt_len = op->produced;
429 uint32_t remaining_data = op->produced;
430 uint16_t data_to_append;
432 while (remaining_data > 0) {
434 RTE_MIN(remaining_data,
436 m->data_len = data_to_append;
443 rte_mempool_put_bulk(mem->op_pool,
444 (void **)deq_ops, num_deq);
445 allocated -= num_deq;
452 rte_mempool_put_bulk(mem->op_pool, (void **)ops, allocated);
453 rte_compressdev_private_xform_free(dev_id, priv_xform);
456 if (test_data->perf_comp_force_stop) {
458 "lcore: %d Perf. test has been aborted by user\n",
466 cperf_cyclecount_test_runner(void *test_ctx)
468 struct cperf_cyclecount_ctx *ctx = test_ctx;
469 struct comp_test_data *test_data = ctx->ver.options;
470 uint32_t lcore = rte_lcore_id();
471 static rte_atomic16_t display_once = RTE_ATOMIC16_INIT(0);
472 static rte_spinlock_t print_spinlock;
475 uint32_t ops_enq_retries_comp;
476 uint32_t ops_deq_retries_comp;
478 uint32_t ops_enq_retries_decomp;
479 uint32_t ops_deq_retries_decomp;
481 uint32_t duration_setup_per_op;
483 uint32_t duration_enq_per_op_comp;
484 uint32_t duration_deq_per_op_comp;
486 uint32_t duration_enq_per_op_decomp;
487 uint32_t duration_deq_per_op_decomp;
489 ctx->ver.mem.lcore_id = lcore;
492 * printing information about current compression thread
494 if (rte_atomic16_test_and_set(&ctx->ver.mem.print_info_once))
500 " queue pair id: %u\n",
502 ctx->ver.options->driver_name,
503 rte_compressdev_name_get(ctx->ver.mem.dev_id),
505 rte_compressdev_socket_id(ctx->ver.mem.dev_id),
509 * First the verification part is needed
511 if (cperf_verify_test_runner(&ctx->ver))
515 * Run the tests twice, discarding the first performance
516 * results, before the cache is warmed up
519 /* C O M P R E S S */
520 for (i = 0; i < 2; i++) {
521 if (main_loop(ctx, RTE_COMP_COMPRESS) < 0)
525 ops_enq_retries_comp = ctx->ops_enq_retries;
526 ops_deq_retries_comp = ctx->ops_deq_retries;
528 duration_enq_per_op_comp = ctx->duration_enq /
529 (ctx->ver.mem.total_bufs * test_data->num_iter);
530 duration_deq_per_op_comp = ctx->duration_deq /
531 (ctx->ver.mem.total_bufs * test_data->num_iter);
533 /* D E C O M P R E S S */
534 for (i = 0; i < 2; i++) {
535 if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0)
539 ops_enq_retries_decomp = ctx->ops_enq_retries;
540 ops_deq_retries_decomp = ctx->ops_deq_retries;
542 duration_enq_per_op_decomp = ctx->duration_enq /
543 (ctx->ver.mem.total_bufs * test_data->num_iter);
544 duration_deq_per_op_decomp = ctx->duration_deq /
545 (ctx->ver.mem.total_bufs * test_data->num_iter);
547 duration_setup_per_op = ctx->duration_op /
548 (ctx->ver.mem.total_bufs * test_data->num_iter);
550 /* R E P O R T processing */
551 if (rte_atomic16_test_and_set(&display_once)) {
553 rte_spinlock_lock(&print_spinlock);
555 printf("\nLegend for the table\n"
556 " - Retries section: number of retries for the following operations:\n"
557 " [C-e] - compression enqueue\n"
558 " [C-d] - compression dequeue\n"
559 " [D-e] - decompression enqueue\n"
560 " [D-d] - decompression dequeue\n"
561 " - Cycles section: number of cycles per 'op' for the following operations:\n"
562 " setup/op - memory allocation, op configuration and memory dealocation\n"
563 " [C-e] - compression enqueue\n"
564 " [C-d] - compression dequeue\n"
565 " [D-e] - decompression enqueue\n"
566 " [D-d] - decompression dequeue\n\n");
568 printf("\n%12s%6s%12s%17s",
569 "lcore id", "Level", "Comp size", "Comp ratio [%]");
571 printf(" |%10s %6s %8s %6s %8s",
576 printf(" |%9s %9s %9s %9s %9s %9s\n",
582 rte_spinlock_unlock(&print_spinlock);
585 rte_spinlock_lock(&print_spinlock);
591 ctx->ver.mem.lcore_id,
593 ctx->ver.comp_data_sz,
596 printf(" |%10s %6u %8u %6u %8u",
598 ops_enq_retries_comp,
599 ops_deq_retries_comp,
600 ops_enq_retries_decomp,
601 ops_deq_retries_decomp);
603 printf(" |%9s %9u %9u %9u %9u %9u\n",
605 duration_setup_per_op,
606 duration_enq_per_op_comp,
607 duration_deq_per_op_comp,
608 duration_enq_per_op_decomp,
609 duration_deq_per_op_decomp);
611 rte_spinlock_unlock(&print_spinlock);