1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
5 #include <rte_malloc.h>
8 #include <rte_cycles.h>
9 #include "rte_spinlock.h"
10 #include <rte_compressdev.h>
12 #include "comp_perf_test_cyclecount.h"
14 struct cperf_cyclecount_ctx {
15 struct cperf_verify_ctx ver;
17 uint32_t ops_enq_retries;
18 uint32_t ops_deq_retries;
21 uint64_t duration_enq;
22 uint64_t duration_deq;
26 cperf_cyclecount_test_destructor(void *arg)
28 struct cperf_cyclecount_ctx *ctx = arg;
31 comp_perf_free_memory(ctx->ver.options, &ctx->ver.mem);
37 cperf_cyclecount_test_constructor(uint8_t dev_id, uint16_t qp_id,
38 struct comp_test_data *options)
40 struct cperf_cyclecount_ctx *ctx = NULL;
42 ctx = rte_malloc(NULL, sizeof(struct cperf_cyclecount_ctx), 0);
47 ctx->ver.mem.dev_id = dev_id;
48 ctx->ver.mem.qp_id = qp_id;
49 ctx->ver.options = options;
50 ctx->ver.silent = 1; /* ver. part will be silent */
52 if (!comp_perf_allocate_memory(ctx->ver.options, &ctx->ver.mem)
53 && !prepare_bufs(ctx->ver.options, &ctx->ver.mem))
56 cperf_cyclecount_test_destructor(ctx);
61 cperf_cyclecount_op_setup(struct rte_comp_op **ops,
62 struct cperf_cyclecount_ctx *ctx,
63 struct rte_mbuf **input_bufs,
64 struct rte_mbuf **output_bufs,
68 struct comp_test_data *test_data = ctx->ver.options;
69 struct cperf_mem_resources *mem = &ctx->ver.mem;
71 uint32_t i, iter, num_iter;
75 num_iter = test_data->num_iter;
77 for (iter = 0; iter < num_iter; iter++) {
78 uint32_t remaining_ops = mem->total_bufs;
79 uint32_t total_enq_ops = 0;
83 while (remaining_ops > 0) {
84 uint16_t num_ops = RTE_MIN(remaining_ops,
88 /* Allocate compression operations */
89 if (ops_needed && rte_mempool_get_bulk(
94 "Cyclecount: could not allocate enough operations\n");
99 for (i = 0; i < ops_needed; i++) {
101 /* Calculate next buffer to attach */
103 uint32_t buf_id = total_enq_ops + i;
106 /* Reset all data in output buffers */
107 struct rte_mbuf *m = output_bufs[buf_id];
109 m->pkt_len = out_seg_sz * m->nb_segs;
111 m->data_len = m->buf_len - m->data_off;
114 ops[op_id]->m_src = input_bufs[buf_id];
115 ops[op_id]->m_dst = output_bufs[buf_id];
116 ops[op_id]->src.offset = 0;
117 ops[op_id]->src.length =
118 rte_pktmbuf_pkt_len(input_bufs[buf_id]);
119 ops[op_id]->dst.offset = 0;
120 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
121 ops[op_id]->input_chksum = buf_id;
122 ops[op_id]->private_xform = priv_xform;
125 /* E N Q U E U I N G */
126 /* assuming that all ops are enqueued */
127 /* instead of the real enqueue operation */
130 remaining_ops -= num_enq;
131 total_enq_ops += num_enq;
133 /* D E Q U E U I N G */
134 /* assuming that all ops dequeued */
135 /* instead of the real dequeue operation */
138 rte_mempool_put_bulk(mem->op_pool,
139 (void **)ops, num_deq);
144 rte_mempool_put_bulk(mem->op_pool, (void **)ops, ops_needed);
151 main_loop(struct cperf_cyclecount_ctx *ctx, enum rte_comp_xform_type type)
153 struct comp_test_data *test_data = ctx->ver.options;
154 struct cperf_mem_resources *mem = &ctx->ver.mem;
155 uint8_t dev_id = mem->dev_id;
156 uint32_t i, iter, num_iter;
157 struct rte_comp_op **ops, **deq_ops;
158 void *priv_xform = NULL;
159 struct rte_comp_xform xform;
160 struct rte_mbuf **input_bufs, **output_bufs;
165 uint64_t tsc_start, tsc_end, tsc_duration;
167 if (test_data == NULL || !test_data->burst_sz) {
168 RTE_LOG(ERR, USER1, "Unknown burst size\n");
171 ctx->duration_enq = 0;
172 ctx->duration_deq = 0;
173 ctx->ops_enq_retries = 0;
174 ctx->ops_deq_retries = 0;
176 /* one array for both enqueue and dequeue */
177 ops = rte_zmalloc_socket(NULL,
178 (test_data->burst_sz + mem->total_bufs) *
179 sizeof(struct rte_comp_op *),
184 "Can't allocate memory for ops structures\n");
188 deq_ops = &ops[test_data->burst_sz];
190 if (type == RTE_COMP_COMPRESS) {
191 xform = (struct rte_comp_xform) {
192 .type = RTE_COMP_COMPRESS,
194 .algo = RTE_COMP_ALGO_DEFLATE,
195 .deflate.huffman = test_data->huffman_enc,
196 .level = test_data->level,
197 .window_size = test_data->window_sz,
198 .chksum = RTE_COMP_CHECKSUM_NONE,
199 .hash_algo = RTE_COMP_HASH_ALGO_NONE
202 input_bufs = mem->decomp_bufs;
203 output_bufs = mem->comp_bufs;
204 out_seg_sz = test_data->out_seg_sz;
206 xform = (struct rte_comp_xform) {
207 .type = RTE_COMP_DECOMPRESS,
209 .algo = RTE_COMP_ALGO_DEFLATE,
210 .chksum = RTE_COMP_CHECKSUM_NONE,
211 .window_size = test_data->window_sz,
212 .hash_algo = RTE_COMP_HASH_ALGO_NONE
215 input_bufs = mem->comp_bufs;
216 output_bufs = mem->decomp_bufs;
217 out_seg_sz = test_data->seg_sz;
220 /* Create private xform */
221 if (rte_compressdev_private_xform_create(dev_id, &xform,
223 RTE_LOG(ERR, USER1, "Private xform could not be created\n");
228 tsc_start = rte_rdtsc_precise();
229 ret = cperf_cyclecount_op_setup(ops,
236 tsc_end = rte_rdtsc_precise();
238 /* ret value check postponed a bit to cancel extra 'if' bias */
240 RTE_LOG(ERR, USER1, "Setup function failed\n");
245 tsc_duration = tsc_end - tsc_start;
246 ctx->duration_op = tsc_duration;
248 num_iter = test_data->num_iter;
249 for (iter = 0; iter < num_iter; iter++) {
250 uint32_t total_ops = mem->total_bufs;
251 uint32_t remaining_ops = mem->total_bufs;
252 uint32_t total_deq_ops = 0;
253 uint32_t total_enq_ops = 0;
254 uint16_t ops_unused = 0;
255 uint16_t num_enq = 0;
256 uint16_t num_deq = 0;
258 while (remaining_ops > 0) {
259 uint16_t num_ops = RTE_MIN(remaining_ops,
260 test_data->burst_sz);
261 uint16_t ops_needed = num_ops - ops_unused;
264 * Move the unused operations from the previous
265 * enqueue_burst call to the front, to maintain order
267 if ((ops_unused > 0) && (num_enq > 0)) {
269 ops_unused * sizeof(struct rte_comp_op *);
271 memmove(ops, &ops[num_enq], nb_b_to_mov);
274 /* Allocate compression operations */
275 if (ops_needed && rte_mempool_get_bulk(
277 (void **)&ops[ops_unused],
280 "Could not allocate enough operations\n");
284 allocated += ops_needed;
286 for (i = 0; i < ops_needed; i++) {
288 * Calculate next buffer to attach to operation
290 uint32_t buf_id = total_enq_ops + i +
292 uint16_t op_id = ops_unused + i;
293 /* Reset all data in output buffers */
294 struct rte_mbuf *m = output_bufs[buf_id];
296 m->pkt_len = out_seg_sz * m->nb_segs;
298 m->data_len = m->buf_len - m->data_off;
301 ops[op_id]->m_src = input_bufs[buf_id];
302 ops[op_id]->m_dst = output_bufs[buf_id];
303 ops[op_id]->src.offset = 0;
304 ops[op_id]->src.length =
305 rte_pktmbuf_pkt_len(input_bufs[buf_id]);
306 ops[op_id]->dst.offset = 0;
307 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
308 ops[op_id]->input_chksum = buf_id;
309 ops[op_id]->private_xform = priv_xform;
312 if (unlikely(test_data->perf_comp_force_stop))
315 tsc_start = rte_rdtsc_precise();
316 num_enq = rte_compressdev_enqueue_burst(dev_id,
319 tsc_end = rte_rdtsc_precise();
320 tsc_duration = tsc_end - tsc_start;
321 ctx->duration_enq += tsc_duration;
323 if (num_enq < num_ops)
324 ctx->ops_enq_retries++;
326 if (test_data->cyclecount_delay)
327 rte_delay_us_block(test_data->cyclecount_delay);
330 struct rte_compressdev_stats stats;
332 rte_compressdev_stats_get(dev_id, &stats);
333 if (stats.enqueue_err_count) {
339 ops_unused = num_ops - num_enq;
340 remaining_ops -= num_enq;
341 total_enq_ops += num_enq;
343 tsc_start = rte_rdtsc_precise();
344 num_deq = rte_compressdev_dequeue_burst(dev_id,
348 tsc_end = rte_rdtsc_precise();
349 tsc_duration = tsc_end - tsc_start;
350 ctx->duration_deq += tsc_duration;
352 if (num_deq < allocated)
353 ctx->ops_deq_retries++;
355 total_deq_ops += num_deq;
357 if (iter == num_iter - 1) {
358 for (i = 0; i < num_deq; i++) {
359 struct rte_comp_op *op = deq_ops[i];
362 RTE_COMP_OP_STATUS_SUCCESS) {
363 RTE_LOG(ERR, USER1, "Some operations were not successful\n");
367 struct rte_mbuf *m = op->m_dst;
369 m->pkt_len = op->produced;
370 uint32_t remaining_data = op->produced;
371 uint16_t data_to_append;
373 while (remaining_data > 0) {
375 RTE_MIN(remaining_data,
377 m->data_len = data_to_append;
384 rte_mempool_put_bulk(mem->op_pool,
385 (void **)deq_ops, num_deq);
386 allocated -= num_deq;
389 /* Dequeue the last operations */
390 while (total_deq_ops < total_ops) {
391 if (unlikely(test_data->perf_comp_force_stop))
394 tsc_start = rte_rdtsc_precise();
395 num_deq = rte_compressdev_dequeue_burst(dev_id,
398 test_data->burst_sz);
399 tsc_end = rte_rdtsc_precise();
400 tsc_duration = tsc_end - tsc_start;
401 ctx->duration_deq += tsc_duration;
402 ctx->ops_deq_retries++;
405 struct rte_compressdev_stats stats;
407 rte_compressdev_stats_get(dev_id, &stats);
408 if (stats.dequeue_err_count) {
413 total_deq_ops += num_deq;
415 if (iter == num_iter - 1) {
416 for (i = 0; i < num_deq; i++) {
417 struct rte_comp_op *op = deq_ops[i];
420 RTE_COMP_OP_STATUS_SUCCESS) {
421 RTE_LOG(ERR, USER1, "Some operations were not successful\n");
425 struct rte_mbuf *m = op->m_dst;
427 m->pkt_len = op->produced;
428 uint32_t remaining_data = op->produced;
429 uint16_t data_to_append;
431 while (remaining_data > 0) {
433 RTE_MIN(remaining_data,
435 m->data_len = data_to_append;
442 rte_mempool_put_bulk(mem->op_pool,
443 (void **)deq_ops, num_deq);
444 allocated -= num_deq;
451 rte_mempool_put_bulk(mem->op_pool, (void **)ops, allocated);
452 rte_compressdev_private_xform_free(dev_id, priv_xform);
455 if (test_data->perf_comp_force_stop) {
457 "lcore: %d Perf. test has been aborted by user\n",
465 cperf_cyclecount_test_runner(void *test_ctx)
467 struct cperf_cyclecount_ctx *ctx = test_ctx;
468 struct comp_test_data *test_data = ctx->ver.options;
469 uint32_t lcore = rte_lcore_id();
470 static uint16_t display_once;
471 static rte_spinlock_t print_spinlock;
474 uint32_t ops_enq_retries_comp;
475 uint32_t ops_deq_retries_comp;
477 uint32_t ops_enq_retries_decomp;
478 uint32_t ops_deq_retries_decomp;
480 uint32_t duration_setup_per_op;
482 uint32_t duration_enq_per_op_comp;
483 uint32_t duration_deq_per_op_comp;
485 uint32_t duration_enq_per_op_decomp;
486 uint32_t duration_deq_per_op_decomp;
488 ctx->ver.mem.lcore_id = lcore;
492 * printing information about current compression thread
494 if (__atomic_compare_exchange_n(&ctx->ver.mem.print_info_once, &exp,
495 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
501 " queue pair id: %u\n",
503 ctx->ver.options->driver_name,
504 rte_compressdev_name_get(ctx->ver.mem.dev_id),
506 rte_compressdev_socket_id(ctx->ver.mem.dev_id),
510 * First the verification part is needed
512 if (cperf_verify_test_runner(&ctx->ver))
516 * Run the tests twice, discarding the first performance
517 * results, before the cache is warmed up
520 /* C O M P R E S S */
521 for (i = 0; i < 2; i++) {
522 if (main_loop(ctx, RTE_COMP_COMPRESS) < 0)
526 ops_enq_retries_comp = ctx->ops_enq_retries;
527 ops_deq_retries_comp = ctx->ops_deq_retries;
529 duration_enq_per_op_comp = ctx->duration_enq /
530 (ctx->ver.mem.total_bufs * test_data->num_iter);
531 duration_deq_per_op_comp = ctx->duration_deq /
532 (ctx->ver.mem.total_bufs * test_data->num_iter);
534 /* D E C O M P R E S S */
535 for (i = 0; i < 2; i++) {
536 if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0)
540 ops_enq_retries_decomp = ctx->ops_enq_retries;
541 ops_deq_retries_decomp = ctx->ops_deq_retries;
543 duration_enq_per_op_decomp = ctx->duration_enq /
544 (ctx->ver.mem.total_bufs * test_data->num_iter);
545 duration_deq_per_op_decomp = ctx->duration_deq /
546 (ctx->ver.mem.total_bufs * test_data->num_iter);
548 duration_setup_per_op = ctx->duration_op /
549 (ctx->ver.mem.total_bufs * test_data->num_iter);
551 /* R E P O R T processing */
552 rte_spinlock_lock(&print_spinlock);
554 if (display_once == 0) {
557 printf("\nLegend for the table\n"
558 " - Retries section: number of retries for the following operations:\n"
559 " [C-e] - compression enqueue\n"
560 " [C-d] - compression dequeue\n"
561 " [D-e] - decompression enqueue\n"
562 " [D-d] - decompression dequeue\n"
563 " - Cycles section: number of cycles per 'op' for the following operations:\n"
564 " setup/op - memory allocation, op configuration and memory dealocation\n"
565 " [C-e] - compression enqueue\n"
566 " [C-d] - compression dequeue\n"
567 " [D-e] - decompression enqueue\n"
568 " [D-d] - decompression dequeue\n\n");
570 printf("\n%12s%6s%12s%17s",
571 "lcore id", "Level", "Comp size", "Comp ratio [%]");
573 printf(" |%10s %6s %8s %6s %8s",
578 printf(" |%9s %9s %9s %9s %9s %9s\n",
589 ctx->ver.mem.lcore_id,
591 ctx->ver.comp_data_sz,
594 printf(" |%10s %6u %8u %6u %8u",
596 ops_enq_retries_comp,
597 ops_deq_retries_comp,
598 ops_enq_retries_decomp,
599 ops_deq_retries_decomp);
601 printf(" |%9s %9u %9u %9u %9u %9u\n",
603 duration_setup_per_op,
604 duration_enq_per_op_comp,
605 duration_deq_per_op_comp,
606 duration_enq_per_op_decomp,
607 duration_deq_per_op_decomp);
609 rte_spinlock_unlock(&print_spinlock);