app/test-compress-perf/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_eal.h>
   7 #include <rte_log.h>
   8 #include <rte_cycles.h>
   9 #include <rte_compressdev.h>
  10
  11 #include "comp_perf_options.h"
  12
  13 #define NUM_MAX_XFORMS 16
  14 #define NUM_MAX_INFLIGHT_OPS 512
  15 #define EXPANSE_RATIO 1.05
  16 #define MIN_COMPRESSED_BUF_SIZE 8
  17
  18 #define DIV_CEIL(a, b)  ((a) / (b) + ((a) % (b) != 0))
  19
  20 /* Cleanup state machine */
  21 static enum cleanup_st {
  22         ST_CLEAR = 0,
  23         ST_TEST_DATA,
  24         ST_COMPDEV,
  25         ST_INPUT_DATA,
  26         ST_MEMORY_ALLOC,
  27         ST_PREPARE_BUF,
  28         ST_DURING_TEST
  29 } cleanup = ST_CLEAR;
  30
  31 static int
  32 param_range_check(uint16_t size, const struct rte_param_log2_range *range)
  33 {
  34         unsigned int next_size;
  35
  36         /* Check lower/upper bounds */
  37         if (size < range->min)
  38                 return -1;
  39
  40         if (size > range->max)
  41                 return -1;
  42
  43         /* If range is actually only one value, size is correct */
  44         if (range->increment == 0)
  45                 return 0;
  46
  47         /* Check if value is one of the supported sizes */
  48         for (next_size = range->min; next_size <= range->max;
  49                         next_size += range->increment)
  50                 if (size == next_size)
  51                         return 0;
  52
  53         return -1;
  54 }
  55
  56 static int
  57 comp_perf_check_capabilities(struct comp_test_data *test_data)
  58 {
  59         const struct rte_compressdev_capabilities *cap;
  60
  61         cap = rte_compressdev_capability_get(test_data->cdev_id,
  62                                              RTE_COMP_ALGO_DEFLATE);
  63
  64         if (cap == NULL) {
  65                 RTE_LOG(ERR, USER1,
  66                         "Compress device does not support DEFLATE\n");
  67                 return -1;
  68         }
  69
  70         uint64_t comp_flags = cap->comp_feature_flags;
  71
  72         /* Huffman enconding */
  73         if (test_data->huffman_enc == RTE_COMP_HUFFMAN_FIXED &&
  74                         (comp_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0) {
  75                 RTE_LOG(ERR, USER1,
  76                         "Compress device does not supported Fixed Huffman\n");
  77                 return -1;
  78         }
  79
  80         if (test_data->huffman_enc == RTE_COMP_HUFFMAN_DYNAMIC &&
  81                         (comp_flags & RTE_COMP_FF_HUFFMAN_DYNAMIC) == 0) {
  82                 RTE_LOG(ERR, USER1,
  83                         "Compress device does not supported Dynamic Huffman\n");
  84                 return -1;
  85         }
  86
  87         /* Window size */
  88         if (test_data->window_sz != -1) {
  89                 if (param_range_check(test_data->window_sz, &cap->window_size)
  90                                 < 0) {
  91                         RTE_LOG(ERR, USER1,
  92                                 "Compress device does not support "
  93                                 "this window size\n");
  94                         return -1;
  95                 }
  96         } else
  97                 /* Set window size to PMD maximum if none was specified */
  98                 test_data->window_sz = cap->window_size.max;
  99
 100         /* Check if chained mbufs is supported */
 101         if (test_data->max_sgl_segs > 1  &&
 102                         (comp_flags & RTE_COMP_FF_OOP_SGL_IN_SGL_OUT) == 0) {
 103                 RTE_LOG(INFO, USER1, "Compress device does not support "
 104                                 "chained mbufs. Max SGL segments set to 1\n");
 105                 test_data->max_sgl_segs = 1;
 106         }
 107
 108         /* Level 0 support */
 109         if (test_data->level.min == 0 &&
 110                         (comp_flags & RTE_COMP_FF_NONCOMPRESSED_BLOCKS) == 0) {
 111                 RTE_LOG(ERR, USER1, "Compress device does not support "
 112                                 "level 0 (no compression)\n");
 113                 return -1;
 114         }
 115
 116         return 0;
 117 }
 118
 119 static int
 120 comp_perf_allocate_memory(struct comp_test_data *test_data)
 121 {
 122         /* Number of segments for input and output
 123          * (compression and decompression)
 124          */
 125         uint32_t total_segs = DIV_CEIL(test_data->input_data_sz,
 126                         test_data->seg_sz);
 127         test_data->comp_buf_pool = rte_pktmbuf_pool_create("comp_buf_pool",
 128                                 total_segs,
 129                                 0, 0, test_data->seg_sz + RTE_PKTMBUF_HEADROOM,
 130                                 rte_socket_id());
 131         if (test_data->comp_buf_pool == NULL) {
 132                 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n");
 133                 return -1;
 134         }
 135
 136         cleanup = ST_MEMORY_ALLOC;
 137         test_data->decomp_buf_pool = rte_pktmbuf_pool_create("decomp_buf_pool",
 138                                 total_segs,
 139                                 0, 0, test_data->seg_sz + RTE_PKTMBUF_HEADROOM,
 140                                 rte_socket_id());
 141         if (test_data->decomp_buf_pool == NULL) {
 142                 RTE_LOG(ERR, USER1, "Mbuf mempool could not be created\n");
 143                 return -1;
 144         }
 145
 146         test_data->total_bufs = DIV_CEIL(total_segs, test_data->max_sgl_segs);
 147
 148         test_data->op_pool = rte_comp_op_pool_create("op_pool",
 149                                   test_data->total_bufs,
 150                                   0, 0, rte_socket_id());
 151         if (test_data->op_pool == NULL) {
 152                 RTE_LOG(ERR, USER1, "Comp op mempool could not be created\n");
 153                 return -1;
 154         }
 155
 156         /*
 157          * Compressed data might be a bit larger than input data,
 158          * if data cannot be compressed
 159          */
 160         test_data->compressed_data = rte_zmalloc_socket(NULL,
 161                                 test_data->input_data_sz * EXPANSE_RATIO
 162                                                 + MIN_COMPRESSED_BUF_SIZE, 0,
 163                                 rte_socket_id());
 164         if (test_data->compressed_data == NULL) {
 165                 RTE_LOG(ERR, USER1, "Memory to hold the data from the input "
 166                                 "file could not be allocated\n");
 167                 return -1;
 168         }
 169
 170         test_data->decompressed_data = rte_zmalloc_socket(NULL,
 171                                 test_data->input_data_sz, 0,
 172                                 rte_socket_id());
 173         if (test_data->decompressed_data == NULL) {
 174                 RTE_LOG(ERR, USER1, "Memory to hold the data from the input "
 175                                 "file could not be allocated\n");
 176                 return -1;
 177         }
 178
 179         test_data->comp_bufs = rte_zmalloc_socket(NULL,
 180                         test_data->total_bufs * sizeof(struct rte_mbuf *),
 181                         0, rte_socket_id());
 182         if (test_data->comp_bufs == NULL) {
 183                 RTE_LOG(ERR, USER1, "Memory to hold the compression mbufs"
 184                                 " could not be allocated\n");
 185                 return -1;
 186         }
 187
 188         test_data->decomp_bufs = rte_zmalloc_socket(NULL,
 189                         test_data->total_bufs * sizeof(struct rte_mbuf *),
 190                         0, rte_socket_id());
 191         if (test_data->decomp_bufs == NULL) {
 192                 RTE_LOG(ERR, USER1, "Memory to hold the decompression mbufs"
 193                                 " could not be allocated\n");
 194                 return -1;
 195         }
 196         return 0;
 197 }
 198
 199 static int
 200 comp_perf_dump_input_data(struct comp_test_data *test_data)
 201 {
 202         FILE *f = fopen(test_data->input_file, "r");
 203         int ret = -1;
 204
 205         if (f == NULL) {
 206                 RTE_LOG(ERR, USER1, "Input file could not be opened\n");
 207                 return -1;
 208         }
 209
 210         if (fseek(f, 0, SEEK_END) != 0) {
 211                 RTE_LOG(ERR, USER1, "Size of input could not be calculated\n");
 212                 goto end;
 213         }
 214         size_t actual_file_sz = ftell(f);
 215         /* If extended input data size has not been set,
 216          * input data size = file size
 217          */
 218
 219         if (test_data->input_data_sz == 0)
 220                 test_data->input_data_sz = actual_file_sz;
 221
 222         if (fseek(f, 0, SEEK_SET) != 0) {
 223                 RTE_LOG(ERR, USER1, "Size of input could not be calculated\n");
 224                 goto end;
 225         }
 226
 227         test_data->input_data = rte_zmalloc_socket(NULL,
 228                                 test_data->input_data_sz, 0, rte_socket_id());
 229
 230         if (test_data->input_data == NULL) {
 231                 RTE_LOG(ERR, USER1, "Memory to hold the data from the input "
 232                                 "file could not be allocated\n");
 233                 goto end;
 234         }
 235
 236         size_t remaining_data = test_data->input_data_sz;
 237         uint8_t *data = test_data->input_data;
 238
 239         while (remaining_data > 0) {
 240                 size_t data_to_read = RTE_MIN(remaining_data, actual_file_sz);
 241
 242                 if (fread(data, data_to_read, 1, f) != 1) {
 243                         RTE_LOG(ERR, USER1, "Input file could not be read\n");
 244                         goto end;
 245                 }
 246                 if (fseek(f, 0, SEEK_SET) != 0) {
 247                         RTE_LOG(ERR, USER1,
 248                                 "Size of input could not be calculated\n");
 249                         goto end;
 250                 }
 251                 remaining_data -= data_to_read;
 252                 data += data_to_read;
 253         }
 254
 255         if (test_data->input_data_sz > actual_file_sz)
 256                 RTE_LOG(INFO, USER1,
 257                   "%zu bytes read from file %s, extending the file %.2f times\n",
 258                         test_data->input_data_sz, test_data->input_file,
 259                         (double)test_data->input_data_sz/actual_file_sz);
 260         else
 261                 RTE_LOG(INFO, USER1,
 262                         "%zu bytes read from file %s\n",
 263                         test_data->input_data_sz, test_data->input_file);
 264
 265         ret = 0;
 266
 267 end:
 268         fclose(f);
 269         return ret;
 270 }
 271
 272 static int
 273 comp_perf_initialize_compressdev(struct comp_test_data *test_data)
 274 {
 275         uint8_t enabled_cdev_count;
 276         uint8_t enabled_cdevs[RTE_COMPRESS_MAX_DEVS];
 277
 278         enabled_cdev_count = rte_compressdev_devices_get(test_data->driver_name,
 279                         enabled_cdevs, RTE_COMPRESS_MAX_DEVS);
 280         if (enabled_cdev_count == 0) {
 281                 RTE_LOG(ERR, USER1, "No compress devices type %s available\n",
 282                                 test_data->driver_name);
 283                 return -EINVAL;
 284         }
 285
 286         if (enabled_cdev_count > 1)
 287                 RTE_LOG(INFO, USER1,
 288                         "Only the first compress device will be used\n");
 289
 290         test_data->cdev_id = enabled_cdevs[0];
 291
 292         if (comp_perf_check_capabilities(test_data) < 0)
 293                 return -1;
 294
 295         /* Configure compressdev (one device, one queue pair) */
 296         struct rte_compressdev_config config = {
 297                 .socket_id = rte_socket_id(),
 298                 .nb_queue_pairs = 1,
 299                 .max_nb_priv_xforms = NUM_MAX_XFORMS,
 300                 .max_nb_streams = 0
 301         };
 302
 303         if (rte_compressdev_configure(test_data->cdev_id, &config) < 0) {
 304                 RTE_LOG(ERR, USER1, "Device configuration failed\n");
 305                 return -1;
 306         }
 307
 308         if (rte_compressdev_queue_pair_setup(test_data->cdev_id, 0,
 309                         NUM_MAX_INFLIGHT_OPS, rte_socket_id()) < 0) {
 310                 RTE_LOG(ERR, USER1, "Queue pair setup failed\n");
 311                 return -1;
 312         }
 313
 314         if (rte_compressdev_start(test_data->cdev_id) < 0) {
 315                 RTE_LOG(ERR, USER1, "Device could not be started\n");
 316                 return -1;
 317         }
 318
 319         return 0;
 320 }
 321
 322 static int
 323 prepare_bufs(struct comp_test_data *test_data)
 324 {
 325         uint32_t remaining_data = test_data->input_data_sz;
 326         uint8_t *input_data_ptr = test_data->input_data;
 327         size_t data_sz;
 328         uint8_t *data_addr;
 329         uint32_t i, j;
 330
 331         for (i = 0; i < test_data->total_bufs; i++) {
 332                 /* Allocate data in input mbuf and copy data from input file */
 333                 test_data->decomp_bufs[i] =
 334                         rte_pktmbuf_alloc(test_data->decomp_buf_pool);
 335                 if (test_data->decomp_bufs[i] == NULL) {
 336                         RTE_LOG(ERR, USER1, "Could not allocate mbuf\n");
 337                         return -1;
 338                 }
 339
 340                 cleanup = ST_PREPARE_BUF;
 341                 data_sz = RTE_MIN(remaining_data, test_data->seg_sz);
 342                 data_addr = (uint8_t *) rte_pktmbuf_append(
 343                                         test_data->decomp_bufs[i], data_sz);
 344                 if (data_addr == NULL) {
 345                         RTE_LOG(ERR, USER1, "Could not append data\n");
 346                         return -1;
 347                 }
 348                 rte_memcpy(data_addr, input_data_ptr, data_sz);
 349
 350                 input_data_ptr += data_sz;
 351                 remaining_data -= data_sz;
 352
 353                 /* Already one segment in the mbuf */
 354                 uint16_t segs_per_mbuf = 1;
 355
 356                 /* Chain mbufs if needed for input mbufs */
 357                 while (segs_per_mbuf < test_data->max_sgl_segs
 358                                 && remaining_data > 0) {
 359                         struct rte_mbuf *next_seg =
 360                                 rte_pktmbuf_alloc(test_data->decomp_buf_pool);
 361
 362                         if (next_seg == NULL) {
 363                                 RTE_LOG(ERR, USER1,
 364                                         "Could not allocate mbuf\n");
 365                                 return -1;
 366                         }
 367
 368                         data_sz = RTE_MIN(remaining_data, test_data->seg_sz);
 369                         data_addr = (uint8_t *)rte_pktmbuf_append(next_seg,
 370                                 data_sz);
 371
 372                         if (data_addr == NULL) {
 373                                 RTE_LOG(ERR, USER1, "Could not append data\n");
 374                                 return -1;
 375                         }
 376
 377                         rte_memcpy(data_addr, input_data_ptr, data_sz);
 378                         input_data_ptr += data_sz;
 379                         remaining_data -= data_sz;
 380
 381                         if (rte_pktmbuf_chain(test_data->decomp_bufs[i],
 382                                         next_seg) < 0) {
 383                                 RTE_LOG(ERR, USER1, "Could not chain mbufs\n");
 384                                 return -1;
 385                         }
 386                         segs_per_mbuf++;
 387                 }
 388
 389                 /* Allocate data in output mbuf */
 390                 test_data->comp_bufs[i] =
 391                         rte_pktmbuf_alloc(test_data->comp_buf_pool);
 392                 if (test_data->comp_bufs[i] == NULL) {
 393                         RTE_LOG(ERR, USER1, "Could not allocate mbuf\n");
 394                         return -1;
 395                 }
 396                 data_addr = (uint8_t *) rte_pktmbuf_append(
 397                                         test_data->comp_bufs[i],
 398                                         test_data->seg_sz);
 399                 if (data_addr == NULL) {
 400                         RTE_LOG(ERR, USER1, "Could not append data\n");
 401                         return -1;
 402                 }
 403
 404                 /* Chain mbufs if needed for output mbufs */
 405                 for (j = 1; j < segs_per_mbuf; j++) {
 406                         struct rte_mbuf *next_seg =
 407                                 rte_pktmbuf_alloc(test_data->comp_buf_pool);
 408
 409                         if (next_seg == NULL) {
 410                                 RTE_LOG(ERR, USER1,
 411                                         "Could not allocate mbuf\n");
 412                                 return -1;
 413                         }
 414
 415                         data_addr = (uint8_t *)rte_pktmbuf_append(next_seg,
 416                                 test_data->seg_sz);
 417
 418                         if (data_addr == NULL) {
 419                                 RTE_LOG(ERR, USER1, "Could not append data\n");
 420                                 return -1;
 421                         }
 422
 423                         if (rte_pktmbuf_chain(test_data->comp_bufs[i],
 424                                         next_seg) < 0) {
 425                                 RTE_LOG(ERR, USER1, "Could not chain mbufs\n");
 426                                 return -1;
 427                         }
 428                 }
 429         }
 430
 431         return 0;
 432 }
 433
 434 static void
 435 free_bufs(struct comp_test_data *test_data)
 436 {
 437         uint32_t i;
 438
 439         for (i = 0; i < test_data->total_bufs; i++) {
 440                 rte_pktmbuf_free(test_data->comp_bufs[i]);
 441                 rte_pktmbuf_free(test_data->decomp_bufs[i]);
 442         }
 443 }
 444
 445 static int
 446 main_loop(struct comp_test_data *test_data, uint8_t level,
 447                         enum rte_comp_xform_type type,
 448                         uint8_t *output_data_ptr,
 449                         size_t *output_data_sz,
 450                         unsigned int benchmarking)
 451 {
 452         uint8_t dev_id = test_data->cdev_id;
 453         uint32_t i, iter, num_iter;
 454         struct rte_comp_op **ops, **deq_ops;
 455         void *priv_xform = NULL;
 456         struct rte_comp_xform xform;
 457         size_t output_size = 0;
 458         struct rte_mbuf **input_bufs, **output_bufs;
 459         int res = 0;
 460         int allocated = 0;
 461
 462         if (test_data == NULL || !test_data->burst_sz) {
 463                 RTE_LOG(ERR, USER1,
 464                         "Unknown burst size\n");
 465                 return -1;
 466         }
 467
 468         ops = rte_zmalloc_socket(NULL,
 469                 2 * test_data->total_bufs * sizeof(struct rte_comp_op *),
 470                 0, rte_socket_id());
 471
 472         if (ops == NULL) {
 473                 RTE_LOG(ERR, USER1,
 474                         "Can't allocate memory for ops strucures\n");
 475                 return -1;
 476         }
 477
 478         deq_ops = &ops[test_data->total_bufs];
 479
 480         if (type == RTE_COMP_COMPRESS) {
 481                 xform = (struct rte_comp_xform) {
 482                         .type = RTE_COMP_COMPRESS,
 483                         .compress = {
 484                                 .algo = RTE_COMP_ALGO_DEFLATE,
 485                                 .deflate.huffman = test_data->huffman_enc,
 486                                 .level = level,
 487                                 .window_size = test_data->window_sz,
 488                                 .chksum = RTE_COMP_CHECKSUM_NONE,
 489                                 .hash_algo = RTE_COMP_HASH_ALGO_NONE
 490                         }
 491                 };
 492                 input_bufs = test_data->decomp_bufs;
 493                 output_bufs = test_data->comp_bufs;
 494         } else {
 495                 xform = (struct rte_comp_xform) {
 496                         .type = RTE_COMP_DECOMPRESS,
 497                         .decompress = {
 498                                 .algo = RTE_COMP_ALGO_DEFLATE,
 499                                 .chksum = RTE_COMP_CHECKSUM_NONE,
 500                                 .window_size = test_data->window_sz,
 501                                 .hash_algo = RTE_COMP_HASH_ALGO_NONE
 502                         }
 503                 };
 504                 input_bufs = test_data->comp_bufs;
 505                 output_bufs = test_data->decomp_bufs;
 506         }
 507
 508         /* Create private xform */
 509         if (rte_compressdev_private_xform_create(dev_id, &xform,
 510                         &priv_xform) < 0) {
 511                 RTE_LOG(ERR, USER1, "Private xform could not be created\n");
 512                 res = -1;
 513                 goto end;
 514         }
 515
 516         uint64_t tsc_start, tsc_end, tsc_duration;
 517
 518         tsc_start = tsc_end = tsc_duration = 0;
 519         if (benchmarking) {
 520                 tsc_start = rte_rdtsc();
 521                 num_iter = test_data->num_iter;
 522         } else
 523                 num_iter = 1;
 524
 525         for (iter = 0; iter < num_iter; iter++) {
 526                 uint32_t total_ops = test_data->total_bufs;
 527                 uint32_t remaining_ops = test_data->total_bufs;
 528                 uint32_t total_deq_ops = 0;
 529                 uint32_t total_enq_ops = 0;
 530                 uint16_t ops_unused = 0;
 531                 uint16_t num_enq = 0;
 532                 uint16_t num_deq = 0;
 533
 534                 output_size = 0;
 535
 536                 while (remaining_ops > 0) {
 537                         uint16_t num_ops = RTE_MIN(remaining_ops,
 538                                                    test_data->burst_sz);
 539                         uint16_t ops_needed = num_ops - ops_unused;
 540
 541                         /*
 542                          * Move the unused operations from the previous
 543                          * enqueue_burst call to the front, to maintain order
 544                          */
 545                         if ((ops_unused > 0) && (num_enq > 0)) {
 546                                 size_t nb_b_to_mov =
 547                                       ops_unused * sizeof(struct rte_comp_op *);
 548
 549                                 memmove(ops, &ops[num_enq], nb_b_to_mov);
 550                         }
 551
 552                         /* Allocate compression operations */
 553                         if (ops_needed && !rte_comp_op_bulk_alloc(
 554                                                 test_data->op_pool,
 555                                                 &ops[ops_unused],
 556                                                 ops_needed)) {
 557                                 RTE_LOG(ERR, USER1,
 558                                       "Could not allocate enough operations\n");
 559                                 res = -1;
 560                                 goto end;
 561                         }
 562                         allocated += ops_needed;
 563
 564                         for (i = 0; i < ops_needed; i++) {
 565                                 /*
 566                                  * Calculate next buffer to attach to operation
 567                                  */
 568                                 uint32_t buf_id = total_enq_ops + i +
 569                                                 ops_unused;
 570                                 uint16_t op_id = ops_unused + i;
 571                                 /* Reset all data in output buffers */
 572                                 struct rte_mbuf *m = output_bufs[buf_id];
 573
 574                                 m->pkt_len = test_data->seg_sz * m->nb_segs;
 575                                 while (m) {
 576                                         m->data_len = m->buf_len - m->data_off;
 577                                         m = m->next;
 578                                 }
 579                                 ops[op_id]->m_src = input_bufs[buf_id];
 580                                 ops[op_id]->m_dst = output_bufs[buf_id];
 581                                 ops[op_id]->src.offset = 0;
 582                                 ops[op_id]->src.length =
 583                                         rte_pktmbuf_pkt_len(input_bufs[buf_id]);
 584                                 ops[op_id]->dst.offset = 0;
 585                                 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
 586                                 ops[op_id]->input_chksum = buf_id;
 587                                 ops[op_id]->private_xform = priv_xform;
 588                         }
 589
 590                         num_enq = rte_compressdev_enqueue_burst(dev_id, 0, ops,
 591                                                                 num_ops);
 592                         ops_unused = num_ops - num_enq;
 593                         remaining_ops -= num_enq;
 594                         total_enq_ops += num_enq;
 595
 596                         num_deq = rte_compressdev_dequeue_burst(dev_id, 0,
 597                                                            deq_ops,
 598                                                            test_data->burst_sz);
 599                         total_deq_ops += num_deq;
 600                         if (benchmarking == 0) {
 601                                 for (i = 0; i < num_deq; i++) {
 602                                         struct rte_comp_op *op = deq_ops[i];
 603                                         const void *read_data_addr =
 604                                                 rte_pktmbuf_read(op->m_dst, 0,
 605                                                 op->produced, output_data_ptr);
 606                                         if (read_data_addr == NULL) {
 607                                                 RTE_LOG(ERR, USER1,
 608                                       "Could not copy buffer in destination\n");
 609                                                 res = -1;
 610                                                 goto end;
 611                                         }
 612
 613                                         if (read_data_addr != output_data_ptr)
 614                                                 rte_memcpy(output_data_ptr,
 615                                                         rte_pktmbuf_mtod(
 616                                                           op->m_dst, uint8_t *),
 617                                                         op->produced);
 618                                         output_data_ptr += op->produced;
 619                                         output_size += op->produced;
 620
 621                                 }
 622                         }
 623
 624                         if (iter == num_iter - 1) {
 625                                 for (i = 0; i < num_deq; i++) {
 626                                         struct rte_comp_op *op = deq_ops[i];
 627                                         struct rte_mbuf *m = op->m_dst;
 628
 629                                         m->pkt_len = op->produced;
 630                                         uint32_t remaining_data = op->produced;
 631                                         uint16_t data_to_append;
 632
 633                                         while (remaining_data > 0) {
 634                                                 data_to_append =
 635                                                         RTE_MIN(remaining_data,
 636                                                              test_data->seg_sz);
 637                                                 m->data_len = data_to_append;
 638                                                 remaining_data -=
 639                                                                 data_to_append;
 640                                                 m = m->next;
 641                                         }
 642                                 }
 643                         }
 644                         rte_mempool_put_bulk(test_data->op_pool,
 645                                              (void **)deq_ops, num_deq);
 646                         allocated -= num_deq;
 647                 }
 648
 649                 /* Dequeue the last operations */
 650                 while (total_deq_ops < total_ops) {
 651                         num_deq = rte_compressdev_dequeue_burst(dev_id, 0,
 652                                                 deq_ops, test_data->burst_sz);
 653                         total_deq_ops += num_deq;
 654                         if (benchmarking == 0) {
 655                                 for (i = 0; i < num_deq; i++) {
 656                                         struct rte_comp_op *op = deq_ops[i];
 657                                         const void *read_data_addr =
 658                                                 rte_pktmbuf_read(op->m_dst,
 659                                                         op->dst.offset,
 660                                                         op->produced,
 661                                                         output_data_ptr);
 662                                         if (read_data_addr == NULL) {
 663                                                 RTE_LOG(ERR, USER1,
 664                                       "Could not copy buffer in destination\n");
 665                                                 res = -1;
 666                                                 goto end;
 667                                         }
 668
 669                                         if (read_data_addr != output_data_ptr)
 670                                                 rte_memcpy(output_data_ptr,
 671                                                         rte_pktmbuf_mtod(
 672                                                         op->m_dst, uint8_t *),
 673                                                         op->produced);
 674                                         output_data_ptr += op->produced;
 675                                         output_size += op->produced;
 676
 677                                 }
 678                         }
 679
 680                         if (iter == num_iter - 1) {
 681                                 for (i = 0; i < num_deq; i++) {
 682                                         struct rte_comp_op *op = deq_ops[i];
 683                                         struct rte_mbuf *m = op->m_dst;
 684
 685                                         m->pkt_len = op->produced;
 686                                         uint32_t remaining_data = op->produced;
 687                                         uint16_t data_to_append;
 688
 689                                         while (remaining_data > 0) {
 690                                                 data_to_append =
 691                                                 RTE_MIN(remaining_data,
 692                                                         test_data->seg_sz);
 693                                                 m->data_len = data_to_append;
 694                                                 remaining_data -=
 695                                                                 data_to_append;
 696                                                 m = m->next;
 697                                         }
 698                                 }
 699                         }
 700                         rte_mempool_put_bulk(test_data->op_pool,
 701                                              (void **)deq_ops, num_deq);
 702                         allocated -= num_deq;
 703                 }
 704         }
 705
 706         if (benchmarking) {
 707                 tsc_end = rte_rdtsc();
 708                 tsc_duration = tsc_end - tsc_start;
 709
 710                 if (type == RTE_COMP_COMPRESS)
 711                         test_data->comp_tsc_duration[level] =
 712                                         tsc_duration / num_iter;
 713                 else
 714                         test_data->decomp_tsc_duration[level] =
 715                                         tsc_duration / num_iter;
 716         }
 717
 718         if (benchmarking == 0 && output_data_sz)
 719                 *output_data_sz = output_size;
 720 end:
 721         rte_mempool_put_bulk(test_data->op_pool, (void **)ops, allocated);
 722         rte_compressdev_private_xform_free(dev_id, priv_xform);
 723         rte_free(ops);
 724         return res;
 725 }
 726
 727 int
 728 main(int argc, char **argv)
 729 {
 730         uint8_t level, level_idx = 0;
 731         int ret, i;
 732         struct comp_test_data *test_data;
 733
 734         /* Initialise DPDK EAL */
 735         ret = rte_eal_init(argc, argv);
 736         if (ret < 0)
 737                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments!\n");
 738         argc -= ret;
 739         argv += ret;
 740
 741         test_data = rte_zmalloc_socket(NULL, sizeof(struct comp_test_data),
 742                                         0, rte_socket_id());
 743
 744         if (test_data == NULL)
 745                 rte_exit(EXIT_FAILURE, "Cannot reserve memory in socket %d\n",
 746                                 rte_socket_id());
 747
 748         cleanup = ST_TEST_DATA;
 749         comp_perf_options_default(test_data);
 750
 751         if (comp_perf_options_parse(test_data, argc, argv) < 0) {
 752                 RTE_LOG(ERR, USER1,
 753                         "Parsing one or more user options failed\n");
 754                 ret = EXIT_FAILURE;
 755                 goto end;
 756         }
 757
 758         if (comp_perf_options_check(test_data) < 0) {
 759                 ret = EXIT_FAILURE;
 760                 goto end;
 761         }
 762
 763         if (comp_perf_initialize_compressdev(test_data) < 0) {
 764                 ret = EXIT_FAILURE;
 765                 goto end;
 766         }
 767
 768         cleanup = ST_COMPDEV;
 769         if (comp_perf_dump_input_data(test_data) < 0) {
 770                 ret = EXIT_FAILURE;
 771                 goto end;
 772         }
 773
 774         cleanup = ST_INPUT_DATA;
 775         if (comp_perf_allocate_memory(test_data) < 0) {
 776                 ret = EXIT_FAILURE;
 777                 goto end;
 778         }
 779
 780         if (prepare_bufs(test_data) < 0) {
 781                 ret = EXIT_FAILURE;
 782                 goto end;
 783         }
 784
 785         if (test_data->level.inc != 0)
 786                 level = test_data->level.min;
 787         else
 788                 level = test_data->level.list[0];
 789
 790         size_t comp_data_sz;
 791         size_t decomp_data_sz;
 792
 793         printf("Burst size = %u\n", test_data->burst_sz);
 794         printf("File size = %zu\n", test_data->input_data_sz);
 795
 796         printf("%6s%12s%17s%19s%21s%15s%21s%23s%16s\n",
 797                 "Level", "Comp size", "Comp ratio [%]",
 798                 "Comp [Cycles/it]", "Comp [Cycles/Byte]", "Comp [Gbps]",
 799                 "Decomp [Cycles/it]", "Decomp [Cycles/Byte]", "Decomp [Gbps]");
 800
 801         cleanup = ST_DURING_TEST;
 802         while (level <= test_data->level.max) {
 803                 /*
 804                  * Run a first iteration, to verify compression and
 805                  * get the compression ratio for the level
 806                  */
 807                 if (main_loop(test_data, level, RTE_COMP_COMPRESS,
 808                               test_data->compressed_data,
 809                               &comp_data_sz, 0) < 0) {
 810                         ret = EXIT_FAILURE;
 811                         goto end;
 812                 }
 813
 814                 if (main_loop(test_data, level, RTE_COMP_DECOMPRESS,
 815                               test_data->decompressed_data,
 816                               &decomp_data_sz, 0) < 0) {
 817                         ret = EXIT_FAILURE;
 818                         goto end;
 819                 }
 820
 821                 if (decomp_data_sz != test_data->input_data_sz) {
 822                         RTE_LOG(ERR, USER1,
 823                    "Decompressed data length not equal to input data length\n");
 824                         RTE_LOG(ERR, USER1,
 825                                 "Decompressed size = %zu, expected = %zu\n",
 826                                 decomp_data_sz, test_data->input_data_sz);
 827                         ret = EXIT_FAILURE;
 828                         goto end;
 829                 } else {
 830                         if (memcmp(test_data->decompressed_data,
 831                                         test_data->input_data,
 832                                         test_data->input_data_sz) != 0) {
 833                                 RTE_LOG(ERR, USER1,
 834                             "Decompressed data is not the same as file data\n");
 835                                 ret = EXIT_FAILURE;
 836                                 goto end;
 837                         }
 838                 }
 839
 840                 double ratio = (double) comp_data_sz /
 841                                                 test_data->input_data_sz * 100;
 842
 843                 /*
 844                  * Run the tests twice, discarding the first performance
 845                  * results, before the cache is warmed up
 846                  */
 847                 for (i = 0; i < 2; i++) {
 848                         if (main_loop(test_data, level, RTE_COMP_COMPRESS,
 849                                         NULL, NULL, 1) < 0) {
 850                                 ret = EXIT_FAILURE;
 851                                 goto end;
 852                         }
 853                 }
 854
 855                 for (i = 0; i < 2; i++) {
 856                         if (main_loop(test_data, level, RTE_COMP_DECOMPRESS,
 857                                         NULL, NULL, 1) < 0) {
 858                                 ret = EXIT_FAILURE;
 859                                 goto end;
 860                         }
 861                 }
 862
 863                 uint64_t comp_tsc_duration =
 864                                 test_data->comp_tsc_duration[level];
 865                 double comp_tsc_byte = (double)comp_tsc_duration /
 866                                                 test_data->input_data_sz;
 867                 double comp_gbps = rte_get_tsc_hz() / comp_tsc_byte * 8 /
 868                                 1000000000;
 869                 uint64_t decomp_tsc_duration =
 870                                 test_data->decomp_tsc_duration[level];
 871                 double decomp_tsc_byte = (double)decomp_tsc_duration /
 872                                                 test_data->input_data_sz;
 873                 double decomp_gbps = rte_get_tsc_hz() / decomp_tsc_byte * 8 /
 874                                 1000000000;
 875
 876                 printf("%6u%12zu%17.2f%19"PRIu64"%21.2f"
 877                                         "%15.2f%21"PRIu64"%23.2f%16.2f\n",
 878                        level, comp_data_sz, ratio, comp_tsc_duration,
 879                        comp_tsc_byte, comp_gbps, decomp_tsc_duration,
 880                        decomp_tsc_byte, decomp_gbps);
 881
 882                 if (test_data->level.inc != 0)
 883                         level += test_data->level.inc;
 884                 else {
 885                         if (++level_idx == test_data->level.count)
 886                                 break;
 887                         level = test_data->level.list[level_idx];
 888                 }
 889         }
 890
 891         ret = EXIT_SUCCESS;
 892
 893 end:
 894         switch (cleanup) {
 895
 896         case ST_DURING_TEST:
 897         case ST_PREPARE_BUF:
 898                 free_bufs(test_data);
 899                 /* fallthrough */
 900         case ST_MEMORY_ALLOC:
 901                 rte_free(test_data->decomp_bufs);
 902                 rte_free(test_data->comp_bufs);
 903                 rte_free(test_data->decompressed_data);
 904                 rte_free(test_data->compressed_data);
 905                 rte_mempool_free(test_data->op_pool);
 906                 rte_mempool_free(test_data->decomp_buf_pool);
 907                 rte_mempool_free(test_data->comp_buf_pool);
 908                 /* fallthrough */
 909         case ST_INPUT_DATA:
 910                 rte_free(test_data->input_data);
 911                 /* fallthrough */
 912         case ST_COMPDEV:
 913                 if (test_data->cdev_id != -1)
 914                         rte_compressdev_stop(test_data->cdev_id);
 915                 /* fallthrough */
 916         case ST_TEST_DATA:
 917                 rte_free(test_data);
 918                 /* fallthrough */
 919         case ST_CLEAR:
 920         default:
 921                 i = rte_eal_cleanup();
 922                 if (i) {
 923                         RTE_LOG(ERR, USER1,
 924                                 "Error from rte_eal_cleanup(), %d\n", i);
 925                         ret = i;
 926                 }
 927                 break;
 928         }
 929         return ret;
 930 }