app/test-compress-perf/comp_perf_test_verify.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_eal.h>
   7 #include <rte_log.h>
   8 #include <rte_compressdev.h>
   9
  10 #include "comp_perf_test_verify.h"
  11 #include "comp_perf_test_common.h"
  12
  13 void
  14 cperf_verify_test_destructor(void *arg)
  15 {
  16         if (arg) {
  17                 comp_perf_free_memory(&((struct cperf_verify_ctx *)arg)->mem);
  18                 rte_free(arg);
  19         }
  20 }
  21
  22 void *
  23 cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id,
  24                 struct comp_test_data *options)
  25 {
  26         struct cperf_verify_ctx *ctx = NULL;
  27
  28         ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0);
  29
  30         if (ctx == NULL)
  31                 return NULL;
  32
  33         ctx->mem.dev_id = dev_id;
  34         ctx->mem.qp_id = qp_id;
  35         ctx->options = options;
  36
  37         if (!comp_perf_allocate_memory(ctx->options, &ctx->mem) &&
  38                         !prepare_bufs(ctx->options, &ctx->mem))
  39                 return ctx;
  40
  41         cperf_verify_test_destructor(ctx);
  42         return NULL;
  43 }
  44
  45 static int
  46 main_loop(struct cperf_verify_ctx *ctx, enum rte_comp_xform_type type)
  47 {
  48         struct comp_test_data *test_data = ctx->options;
  49         uint8_t *output_data_ptr;
  50         size_t *output_data_sz;
  51         struct cperf_mem_resources *mem = &ctx->mem;
  52
  53         uint8_t dev_id = mem->dev_id;
  54         uint32_t i, iter, num_iter;
  55         struct rte_comp_op **ops, **deq_ops;
  56         void *priv_xform = NULL;
  57         struct rte_comp_xform xform;
  58         size_t output_size = 0;
  59         struct rte_mbuf **input_bufs, **output_bufs;
  60         int res = 0;
  61         int allocated = 0;
  62         uint32_t out_seg_sz;
  63
  64         if (test_data == NULL || !test_data->burst_sz) {
  65                 RTE_LOG(ERR, USER1,
  66                         "Unknown burst size\n");
  67                 return -1;
  68         }
  69
  70         ops = rte_zmalloc_socket(NULL,
  71                 2 * mem->total_bufs * sizeof(struct rte_comp_op *),
  72                 0, rte_socket_id());
  73
  74         if (ops == NULL) {
  75                 RTE_LOG(ERR, USER1,
  76                         "Can't allocate memory for ops strucures\n");
  77                 return -1;
  78         }
  79
  80         deq_ops = &ops[mem->total_bufs];
  81
  82         if (type == RTE_COMP_COMPRESS) {
  83                 xform = (struct rte_comp_xform) {
  84                         .type = RTE_COMP_COMPRESS,
  85                         .compress = {
  86                                 .algo = RTE_COMP_ALGO_DEFLATE,
  87                                 .deflate.huffman = test_data->huffman_enc,
  88                                 .level = test_data->level,
  89                                 .window_size = test_data->window_sz,
  90                                 .chksum = RTE_COMP_CHECKSUM_NONE,
  91                                 .hash_algo = RTE_COMP_HASH_ALGO_NONE
  92                         }
  93                 };
  94                 output_data_ptr = ctx->mem.compressed_data;
  95                 output_data_sz = &ctx->comp_data_sz;
  96                 input_bufs = mem->decomp_bufs;
  97                 output_bufs = mem->comp_bufs;
  98                 out_seg_sz = test_data->out_seg_sz;
  99         } else {
 100                 xform = (struct rte_comp_xform) {
 101                         .type = RTE_COMP_DECOMPRESS,
 102                         .decompress = {
 103                                 .algo = RTE_COMP_ALGO_DEFLATE,
 104                                 .chksum = RTE_COMP_CHECKSUM_NONE,
 105                                 .window_size = test_data->window_sz,
 106                                 .hash_algo = RTE_COMP_HASH_ALGO_NONE
 107                         }
 108                 };
 109                 output_data_ptr = ctx->mem.decompressed_data;
 110                 output_data_sz = &ctx->decomp_data_sz;
 111                 input_bufs = mem->comp_bufs;
 112                 output_bufs = mem->decomp_bufs;
 113                 out_seg_sz = test_data->seg_sz;
 114         }
 115
 116         /* Create private xform */
 117         if (rte_compressdev_private_xform_create(dev_id, &xform,
 118                         &priv_xform) < 0) {
 119                 RTE_LOG(ERR, USER1, "Private xform could not be created\n");
 120                 res = -1;
 121                 goto end;
 122         }
 123
 124         num_iter = 1;
 125
 126         for (iter = 0; iter < num_iter; iter++) {
 127                 uint32_t total_ops = mem->total_bufs;
 128                 uint32_t remaining_ops = mem->total_bufs;
 129                 uint32_t total_deq_ops = 0;
 130                 uint32_t total_enq_ops = 0;
 131                 uint16_t ops_unused = 0;
 132                 uint16_t num_enq = 0;
 133                 uint16_t num_deq = 0;
 134
 135                 output_size = 0;
 136
 137                 while (remaining_ops > 0) {
 138                         uint16_t num_ops = RTE_MIN(remaining_ops,
 139                                                    test_data->burst_sz);
 140                         uint16_t ops_needed = num_ops - ops_unused;
 141
 142                         /*
 143                          * Move the unused operations from the previous
 144                          * enqueue_burst call to the front, to maintain order
 145                          */
 146                         if ((ops_unused > 0) && (num_enq > 0)) {
 147                                 size_t nb_b_to_mov =
 148                                       ops_unused * sizeof(struct rte_comp_op *);
 149
 150                                 memmove(ops, &ops[num_enq], nb_b_to_mov);
 151                         }
 152
 153                         /* Allocate compression operations */
 154                         if (ops_needed && !rte_comp_op_bulk_alloc(
 155                                                 mem->op_pool,
 156                                                 &ops[ops_unused],
 157                                                 ops_needed)) {
 158                                 RTE_LOG(ERR, USER1,
 159                                       "Could not allocate enough operations\n");
 160                                 res = -1;
 161                                 goto end;
 162                         }
 163                         allocated += ops_needed;
 164
 165                         for (i = 0; i < ops_needed; i++) {
 166                                 /*
 167                                  * Calculate next buffer to attach to operation
 168                                  */
 169                                 uint32_t buf_id = total_enq_ops + i +
 170                                                 ops_unused;
 171                                 uint16_t op_id = ops_unused + i;
 172                                 /* Reset all data in output buffers */
 173                                 struct rte_mbuf *m = output_bufs[buf_id];
 174
 175                                 m->pkt_len = out_seg_sz * m->nb_segs;
 176                                 while (m) {
 177                                         m->data_len = m->buf_len - m->data_off;
 178                                         m = m->next;
 179                                 }
 180                                 ops[op_id]->m_src = input_bufs[buf_id];
 181                                 ops[op_id]->m_dst = output_bufs[buf_id];
 182                                 ops[op_id]->src.offset = 0;
 183                                 ops[op_id]->src.length =
 184                                         rte_pktmbuf_pkt_len(input_bufs[buf_id]);
 185                                 ops[op_id]->dst.offset = 0;
 186                                 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
 187                                 ops[op_id]->input_chksum = buf_id;
 188                                 ops[op_id]->private_xform = priv_xform;
 189                         }
 190
 191                         num_enq = rte_compressdev_enqueue_burst(dev_id,
 192                                                                 mem->qp_id, ops,
 193                                                                 num_ops);
 194                         if (num_enq == 0) {
 195                                 struct rte_compressdev_stats stats;
 196
 197                                 rte_compressdev_stats_get(dev_id, &stats);
 198                                 if (stats.enqueue_err_count) {
 199                                         res = -1;
 200                                         goto end;
 201                                 }
 202                         }
 203
 204                         ops_unused = num_ops - num_enq;
 205                         remaining_ops -= num_enq;
 206                         total_enq_ops += num_enq;
 207
 208                         num_deq = rte_compressdev_dequeue_burst(dev_id,
 209                                                            mem->qp_id,
 210                                                            deq_ops,
 211                                                            test_data->burst_sz);
 212                         total_deq_ops += num_deq;
 213
 214                         for (i = 0; i < num_deq; i++) {
 215                                 struct rte_comp_op *op = deq_ops[i];
 216
 217                                 if (op->status != RTE_COMP_OP_STATUS_SUCCESS) {
 218                                         RTE_LOG(ERR, USER1,
 219                                                 "Some operations were not successful\n");
 220                                         goto end;
 221                                 }
 222
 223                                 const void *read_data_addr =
 224                                                 rte_pktmbuf_read(op->m_dst, 0,
 225                                                 op->produced, output_data_ptr);
 226                                 if (read_data_addr == NULL) {
 227                                         RTE_LOG(ERR, USER1,
 228                                                 "Could not copy buffer in destination\n");
 229                                         res = -1;
 230                                         goto end;
 231                                 }
 232
 233                                 if (read_data_addr != output_data_ptr)
 234                                         rte_memcpy(output_data_ptr,
 235                                                    rte_pktmbuf_mtod(op->m_dst,
 236                                                                     uint8_t *),
 237                                                    op->produced);
 238                                 output_data_ptr += op->produced;
 239                                 output_size += op->produced;
 240
 241                         }
 242
 243
 244                         if (iter == num_iter - 1) {
 245                                 for (i = 0; i < num_deq; i++) {
 246                                         struct rte_comp_op *op = deq_ops[i];
 247                                         struct rte_mbuf *m = op->m_dst;
 248
 249                                         m->pkt_len = op->produced;
 250                                         uint32_t remaining_data = op->produced;
 251                                         uint16_t data_to_append;
 252
 253                                         while (remaining_data > 0) {
 254                                                 data_to_append =
 255                                                         RTE_MIN(remaining_data,
 256                                                         out_seg_sz);
 257                                                 m->data_len = data_to_append;
 258                                                 remaining_data -=
 259                                                                 data_to_append;
 260                                                 m = m->next;
 261                                         }
 262                                 }
 263                         }
 264                         rte_mempool_put_bulk(mem->op_pool,
 265                                              (void **)deq_ops, num_deq);
 266                         allocated -= num_deq;
 267                 }
 268
 269                 /* Dequeue the last operations */
 270                 while (total_deq_ops < total_ops) {
 271                         num_deq = rte_compressdev_dequeue_burst(dev_id,
 272                                                         mem->qp_id,
 273                                                         deq_ops,
 274                                                         test_data->burst_sz);
 275                         if (num_deq == 0) {
 276                                 struct rte_compressdev_stats stats;
 277
 278                                 rte_compressdev_stats_get(dev_id, &stats);
 279                                 if (stats.dequeue_err_count) {
 280                                         res = -1;
 281                                         goto end;
 282                                 }
 283                         }
 284
 285                         total_deq_ops += num_deq;
 286
 287                         for (i = 0; i < num_deq; i++) {
 288                                 struct rte_comp_op *op = deq_ops[i];
 289
 290                                 if (op->status != RTE_COMP_OP_STATUS_SUCCESS) {
 291                                         RTE_LOG(ERR, USER1,
 292                                                 "Some operations were not successful\n");
 293                                         goto end;
 294                                 }
 295
 296                                 const void *read_data_addr =
 297                                                 rte_pktmbuf_read(op->m_dst,
 298                                                                  op->dst.offset,
 299                                                 op->produced, output_data_ptr);
 300                                 if (read_data_addr == NULL) {
 301                                         RTE_LOG(ERR, USER1,
 302                                                 "Could not copy buffer in destination\n");
 303                                         res = -1;
 304                                         goto end;
 305                                 }
 306
 307                                 if (read_data_addr != output_data_ptr)
 308                                         rte_memcpy(output_data_ptr,
 309                                                    rte_pktmbuf_mtod(
 310                                                         op->m_dst, uint8_t *),
 311                                                    op->produced);
 312                                 output_data_ptr += op->produced;
 313                                 output_size += op->produced;
 314
 315                         }
 316
 317                         if (iter == num_iter - 1) {
 318                                 for (i = 0; i < num_deq; i++) {
 319                                         struct rte_comp_op *op = deq_ops[i];
 320                                         struct rte_mbuf *m = op->m_dst;
 321
 322                                         m->pkt_len = op->produced;
 323                                         uint32_t remaining_data = op->produced;
 324                                         uint16_t data_to_append;
 325
 326                                         while (remaining_data > 0) {
 327                                                 data_to_append =
 328                                                 RTE_MIN(remaining_data,
 329                                                         out_seg_sz);
 330                                                 m->data_len = data_to_append;
 331                                                 remaining_data -=
 332                                                                 data_to_append;
 333                                                 m = m->next;
 334                                         }
 335                                 }
 336                         }
 337                         rte_mempool_put_bulk(mem->op_pool,
 338                                              (void **)deq_ops, num_deq);
 339                         allocated -= num_deq;
 340                 }
 341         }
 342
 343         if (output_data_sz)
 344                 *output_data_sz = output_size;
 345 end:
 346         rte_mempool_put_bulk(mem->op_pool, (void **)ops, allocated);
 347         rte_compressdev_private_xform_free(dev_id, priv_xform);
 348         rte_free(ops);
 349         return res;
 350 }
 351
 352 int
 353 cperf_verify_test_runner(void *test_ctx)
 354 {
 355         struct cperf_verify_ctx *ctx = test_ctx;
 356         struct comp_test_data *test_data = ctx->options;
 357         int ret = EXIT_SUCCESS;
 358         static rte_atomic16_t display_once = RTE_ATOMIC16_INIT(0);
 359         uint32_t lcore = rte_lcore_id();
 360
 361         ctx->mem.lcore_id = lcore;
 362
 363         test_data->ratio = 0;
 364
 365         if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) {
 366                 ret = EXIT_FAILURE;
 367                 goto end;
 368         }
 369
 370         if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) {
 371                 ret = EXIT_FAILURE;
 372                 goto end;
 373         }
 374
 375         if (ctx->decomp_data_sz != test_data->input_data_sz) {
 376                 RTE_LOG(ERR, USER1,
 377            "Decompressed data length not equal to input data length\n");
 378                 RTE_LOG(ERR, USER1,
 379                         "Decompressed size = %zu, expected = %zu\n",
 380                         ctx->decomp_data_sz, test_data->input_data_sz);
 381                 ret = EXIT_FAILURE;
 382                 goto end;
 383         } else {
 384                 if (memcmp(ctx->mem.decompressed_data,
 385                                 test_data->input_data,
 386                                 test_data->input_data_sz) != 0) {
 387                         RTE_LOG(ERR, USER1,
 388                     "Decompressed data is not the same as file data\n");
 389                         ret = EXIT_FAILURE;
 390                         goto end;
 391                 }
 392         }
 393
 394         ctx->ratio = (double) ctx->comp_data_sz /
 395                         test_data->input_data_sz * 100;
 396
 397         if (!ctx->silent) {
 398                 if (rte_atomic16_test_and_set(&display_once)) {
 399                         printf("%12s%6s%12s%17s\n",
 400                             "lcore id", "Level", "Comp size", "Comp ratio [%]");
 401                 }
 402                 printf("%12u%6u%12zu%17.2f\n",
 403                        ctx->mem.lcore_id,
 404                        test_data->level, ctx->comp_data_sz, ctx->ratio);
 405         }
 406
 407 end:
 408         return ret;
 409 }