test/test/test_compressdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <string.h>
   5 #include <zlib.h>
   6 #include <math.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_malloc.h>
  10 #include <rte_mempool.h>
  11 #include <rte_mbuf.h>
  12 #include <rte_compressdev.h>
  13
  14 #include "test_compressdev_test_buffer.h"
  15 #include "test.h"
  16
  17 #define DEFAULT_WINDOW_SIZE 15
  18 #define DEFAULT_MEM_LEVEL 8
  19 #define MAX_DEQD_RETRIES 10
  20 #define DEQUEUE_WAIT_TIME 10000
  21
  22 /*
  23  * 30% extra size for compressed data compared to original data,
  24  * in case data size cannot be reduced and it is actually bigger
  25  * due to the compress block headers
  26  */
  27 #define COMPRESS_BUF_SIZE_RATIO 1.3
  28 #define NUM_MBUFS 16
  29 #define NUM_OPS 16
  30 #define NUM_MAX_XFORMS 16
  31 #define NUM_MAX_INFLIGHT_OPS 128
  32 #define CACHE_SIZE 0
  33
  34 const char *
  35 huffman_type_strings[] = {
  36         [RTE_COMP_HUFFMAN_DEFAULT]      = "PMD default",
  37         [RTE_COMP_HUFFMAN_FIXED]        = "Fixed",
  38         [RTE_COMP_HUFFMAN_DYNAMIC]      = "Dynamic"
  39 };
  40
  41 enum zlib_direction {
  42         ZLIB_NONE,
  43         ZLIB_COMPRESS,
  44         ZLIB_DECOMPRESS,
  45         ZLIB_ALL
  46 };
  47
  48 struct priv_op_data {
  49         uint16_t orig_idx;
  50 };
  51
  52 struct comp_testsuite_params {
  53         struct rte_mempool *mbuf_pool;
  54         struct rte_mempool *op_pool;
  55         struct rte_comp_xform *def_comp_xform;
  56         struct rte_comp_xform *def_decomp_xform;
  57 };
  58
  59 static struct comp_testsuite_params testsuite_params = { 0 };
  60
  61 static void
  62 testsuite_teardown(void)
  63 {
  64         struct comp_testsuite_params *ts_params = &testsuite_params;
  65
  66         rte_mempool_free(ts_params->mbuf_pool);
  67         rte_mempool_free(ts_params->op_pool);
  68         rte_free(ts_params->def_comp_xform);
  69         rte_free(ts_params->def_decomp_xform);
  70 }
  71
  72 static int
  73 testsuite_setup(void)
  74 {
  75         struct comp_testsuite_params *ts_params = &testsuite_params;
  76         unsigned int i;
  77
  78         if (rte_compressdev_count() == 0) {
  79                 RTE_LOG(ERR, USER1, "Need at least one compress device\n");
  80                 return TEST_FAILED;
  81         }
  82
  83         uint32_t max_buf_size = 0;
  84         for (i = 0; i < RTE_DIM(compress_test_bufs); i++)
  85                 max_buf_size = RTE_MAX(max_buf_size,
  86                                 strlen(compress_test_bufs[i]) + 1);
  87
  88         max_buf_size *= COMPRESS_BUF_SIZE_RATIO;
  89         /*
  90          * Buffers to be used in compression and decompression.
  91          * Since decompressed data might be larger than
  92          * compressed data (due to block header),
  93          * buffers should be big enough for both cases.
  94          */
  95         ts_params->mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool",
  96                         NUM_MBUFS,
  97                         CACHE_SIZE, 0,
  98                         max_buf_size + RTE_PKTMBUF_HEADROOM,
  99                         rte_socket_id());
 100         if (ts_params->mbuf_pool == NULL) {
 101                 RTE_LOG(ERR, USER1, "Large mbuf pool could not be created\n");
 102                 return TEST_FAILED;
 103         }
 104
 105         ts_params->op_pool = rte_comp_op_pool_create("op_pool", NUM_OPS,
 106                                 0, sizeof(struct priv_op_data),
 107                                 rte_socket_id());
 108         if (ts_params->op_pool == NULL) {
 109                 RTE_LOG(ERR, USER1, "Operation pool could not be created\n");
 110                 goto exit;
 111         }
 112
 113         ts_params->def_comp_xform =
 114                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 115         if (ts_params->def_comp_xform == NULL) {
 116                 RTE_LOG(ERR, USER1,
 117                         "Default compress xform could not be created\n");
 118                 goto exit;
 119         }
 120         ts_params->def_decomp_xform =
 121                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 122         if (ts_params->def_decomp_xform == NULL) {
 123                 RTE_LOG(ERR, USER1,
 124                         "Default decompress xform could not be created\n");
 125                 goto exit;
 126         }
 127
 128         /* Initializes default values for compress/decompress xforms */
 129         ts_params->def_comp_xform->type = RTE_COMP_COMPRESS;
 130         ts_params->def_comp_xform->compress.algo = RTE_COMP_ALGO_DEFLATE,
 131         ts_params->def_comp_xform->compress.deflate.huffman =
 132                                                 RTE_COMP_HUFFMAN_DEFAULT;
 133         ts_params->def_comp_xform->compress.level = RTE_COMP_LEVEL_PMD_DEFAULT;
 134         ts_params->def_comp_xform->compress.chksum = RTE_COMP_CHECKSUM_NONE;
 135         ts_params->def_comp_xform->compress.window_size = DEFAULT_WINDOW_SIZE;
 136
 137         ts_params->def_decomp_xform->type = RTE_COMP_DECOMPRESS;
 138         ts_params->def_decomp_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE,
 139         ts_params->def_decomp_xform->decompress.chksum = RTE_COMP_CHECKSUM_NONE;
 140         ts_params->def_decomp_xform->decompress.window_size = DEFAULT_WINDOW_SIZE;
 141
 142         return TEST_SUCCESS;
 143
 144 exit:
 145         testsuite_teardown();
 146
 147         return TEST_FAILED;
 148 }
 149
 150 static int
 151 generic_ut_setup(void)
 152 {
 153         /* Configure compressdev (one device, one queue pair) */
 154         struct rte_compressdev_config config = {
 155                 .socket_id = rte_socket_id(),
 156                 .nb_queue_pairs = 1,
 157                 .max_nb_priv_xforms = NUM_MAX_XFORMS,
 158                 .max_nb_streams = 0
 159         };
 160
 161         if (rte_compressdev_configure(0, &config) < 0) {
 162                 RTE_LOG(ERR, USER1, "Device configuration failed\n");
 163                 return -1;
 164         }
 165
 166         if (rte_compressdev_queue_pair_setup(0, 0, NUM_MAX_INFLIGHT_OPS,
 167                         rte_socket_id()) < 0) {
 168                 RTE_LOG(ERR, USER1, "Queue pair setup failed\n");
 169                 return -1;
 170         }
 171
 172         if (rte_compressdev_start(0) < 0) {
 173                 RTE_LOG(ERR, USER1, "Device could not be started\n");
 174                 return -1;
 175         }
 176
 177         return 0;
 178 }
 179
 180 static void
 181 generic_ut_teardown(void)
 182 {
 183         rte_compressdev_stop(0);
 184         if (rte_compressdev_close(0) < 0)
 185                 RTE_LOG(ERR, USER1, "Device could not be closed\n");
 186 }
 187
 188 static int
 189 test_compressdev_invalid_configuration(void)
 190 {
 191         struct rte_compressdev_config invalid_config;
 192         struct rte_compressdev_config valid_config = {
 193                 .socket_id = rte_socket_id(),
 194                 .nb_queue_pairs = 1,
 195                 .max_nb_priv_xforms = NUM_MAX_XFORMS,
 196                 .max_nb_streams = 0
 197         };
 198         struct rte_compressdev_info dev_info;
 199
 200         /* Invalid configuration with 0 queue pairs */
 201         memcpy(&invalid_config, &valid_config,
 202                         sizeof(struct rte_compressdev_config));
 203         invalid_config.nb_queue_pairs = 0;
 204
 205         TEST_ASSERT_FAIL(rte_compressdev_configure(0, &invalid_config),
 206                         "Device configuration was successful "
 207                         "with no queue pairs (invalid)\n");
 208
 209         /*
 210          * Invalid configuration with too many queue pairs
 211          * (if there is an actual maximum number of queue pairs)
 212          */
 213         rte_compressdev_info_get(0, &dev_info);
 214         if (dev_info.max_nb_queue_pairs != 0) {
 215                 memcpy(&invalid_config, &valid_config,
 216                         sizeof(struct rte_compressdev_config));
 217                 invalid_config.nb_queue_pairs = dev_info.max_nb_queue_pairs + 1;
 218
 219                 TEST_ASSERT_FAIL(rte_compressdev_configure(0, &invalid_config),
 220                                 "Device configuration was successful "
 221                                 "with too many queue pairs (invalid)\n");
 222         }
 223
 224         /* Invalid queue pair setup, with no number of queue pairs set */
 225         TEST_ASSERT_FAIL(rte_compressdev_queue_pair_setup(0, 0,
 226                                 NUM_MAX_INFLIGHT_OPS, rte_socket_id()),
 227                         "Queue pair setup was successful "
 228                         "with no queue pairs set (invalid)\n");
 229
 230         return TEST_SUCCESS;
 231 }
 232
 233 static int
 234 compare_buffers(const char *buffer1, uint32_t buffer1_len,
 235                 const char *buffer2, uint32_t buffer2_len)
 236 {
 237         if (buffer1_len != buffer2_len) {
 238                 RTE_LOG(ERR, USER1, "Buffer lengths are different\n");
 239                 return -1;
 240         }
 241
 242         if (memcmp(buffer1, buffer2, buffer1_len) != 0) {
 243                 RTE_LOG(ERR, USER1, "Buffers are different\n");
 244                 return -1;
 245         }
 246
 247         return 0;
 248 }
 249
 250 /*
 251  * Maps compressdev and Zlib flush flags
 252  */
 253 static int
 254 map_zlib_flush_flag(enum rte_comp_flush_flag flag)
 255 {
 256         switch (flag) {
 257         case RTE_COMP_FLUSH_NONE:
 258                 return Z_NO_FLUSH;
 259         case RTE_COMP_FLUSH_SYNC:
 260                 return Z_SYNC_FLUSH;
 261         case RTE_COMP_FLUSH_FULL:
 262                 return Z_FULL_FLUSH;
 263         case RTE_COMP_FLUSH_FINAL:
 264                 return Z_FINISH;
 265         /*
 266          * There should be only the values above,
 267          * so this should never happen
 268          */
 269         default:
 270                 return -1;
 271         }
 272 }
 273
 274 static int
 275 compress_zlib(struct rte_comp_op *op,
 276                 const struct rte_comp_xform *xform, int mem_level)
 277 {
 278         z_stream stream;
 279         int zlib_flush;
 280         int strategy, window_bits, comp_level;
 281         int ret = -1;
 282
 283         /* initialize zlib stream */
 284         stream.zalloc = Z_NULL;
 285         stream.zfree = Z_NULL;
 286         stream.opaque = Z_NULL;
 287
 288         if (xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
 289                 strategy = Z_FIXED;
 290         else
 291                 strategy = Z_DEFAULT_STRATEGY;
 292
 293         /*
 294          * Window bits is the base two logarithm of the window size (in bytes).
 295          * When doing raw DEFLATE, this number will be negative.
 296          */
 297         window_bits = -(xform->compress.window_size);
 298
 299         comp_level = xform->compress.level;
 300
 301         if (comp_level != RTE_COMP_LEVEL_NONE)
 302                 ret = deflateInit2(&stream, comp_level, Z_DEFLATED,
 303                         window_bits, mem_level, strategy);
 304         else
 305                 ret = deflateInit(&stream, Z_NO_COMPRESSION);
 306
 307         if (ret != Z_OK) {
 308                 printf("Zlib deflate could not be initialized\n");
 309                 goto exit;
 310         }
 311
 312         /* Assuming stateless operation */
 313         stream.avail_in = op->src.length;
 314         stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
 315         stream.avail_out = op->m_dst->data_len;
 316         stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
 317
 318         /* Stateless operation, all buffer will be compressed in one go */
 319         zlib_flush = map_zlib_flush_flag(op->flush_flag);
 320         ret = deflate(&stream, zlib_flush);
 321
 322         if (stream.avail_in != 0) {
 323                 RTE_LOG(ERR, USER1, "Buffer could not be read entirely\n");
 324                 goto exit;
 325         }
 326
 327         if (ret != Z_STREAM_END)
 328                 goto exit;
 329
 330         op->consumed = op->src.length - stream.avail_in;
 331         op->produced = op->m_dst->data_len - stream.avail_out;
 332         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 333
 334         deflateReset(&stream);
 335
 336         ret = 0;
 337 exit:
 338         deflateEnd(&stream);
 339
 340         return ret;
 341 }
 342
 343 static int
 344 decompress_zlib(struct rte_comp_op *op,
 345                 const struct rte_comp_xform *xform)
 346 {
 347         z_stream stream;
 348         int window_bits;
 349         int zlib_flush;
 350         int ret = TEST_FAILED;
 351
 352         /* initialize zlib stream */
 353         stream.zalloc = Z_NULL;
 354         stream.zfree = Z_NULL;
 355         stream.opaque = Z_NULL;
 356
 357         /*
 358          * Window bits is the base two logarithm of the window size (in bytes).
 359          * When doing raw DEFLATE, this number will be negative.
 360          */
 361         window_bits = -(xform->decompress.window_size);
 362
 363         ret = inflateInit2(&stream, window_bits);
 364
 365         if (ret != Z_OK) {
 366                 printf("Zlib deflate could not be initialized\n");
 367                 goto exit;
 368         }
 369
 370         /* Assuming stateless operation */
 371         stream.avail_in = op->src.length;
 372         stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
 373         stream.avail_out = op->m_dst->data_len;
 374         stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
 375
 376         /* Stateless operation, all buffer will be compressed in one go */
 377         zlib_flush = map_zlib_flush_flag(op->flush_flag);
 378         ret = inflate(&stream, zlib_flush);
 379
 380         if (stream.avail_in != 0) {
 381                 RTE_LOG(ERR, USER1, "Buffer could not be read entirely\n");
 382                 goto exit;
 383         }
 384
 385         if (ret != Z_STREAM_END)
 386                 goto exit;
 387
 388         op->consumed = op->src.length - stream.avail_in;
 389         op->produced = op->m_dst->data_len - stream.avail_out;
 390         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 391
 392         inflateReset(&stream);
 393
 394         ret = 0;
 395 exit:
 396         inflateEnd(&stream);
 397
 398         return ret;
 399 }
 400
 401 /*
 402  * Compresses and decompresses buffer with compressdev API and Zlib API
 403  */
 404 static int
 405 test_deflate_comp_decomp(const char * const test_bufs[],
 406                 unsigned int num_bufs,
 407                 uint16_t buf_idx[],
 408                 struct rte_comp_xform *compress_xforms[],
 409                 struct rte_comp_xform *decompress_xforms[],
 410                 unsigned int num_xforms,
 411                 enum rte_comp_op_type state,
 412                 enum zlib_direction zlib_dir)
 413 {
 414         struct comp_testsuite_params *ts_params = &testsuite_params;
 415         int ret_status = -1;
 416         int ret;
 417         struct rte_mbuf *uncomp_bufs[num_bufs];
 418         struct rte_mbuf *comp_bufs[num_bufs];
 419         struct rte_comp_op *ops[num_bufs];
 420         struct rte_comp_op *ops_processed[num_bufs];
 421         void *priv_xforms[num_bufs];
 422         uint16_t num_enqd, num_deqd, num_total_deqd;
 423         uint16_t num_priv_xforms = 0;
 424         unsigned int deqd_retries = 0;
 425         struct priv_op_data *priv_data;
 426         char *data_ptr;
 427         unsigned int i;
 428         const struct rte_compressdev_capabilities *capa =
 429                 rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 430
 431         /* Initialize all arrays to NULL */
 432         memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
 433         memset(comp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
 434         memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
 435         memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
 436         memset(priv_xforms, 0, sizeof(void *) * num_bufs);
 437
 438         /* Prepare the source mbufs with the data */
 439         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, uncomp_bufs, num_bufs);
 440         if (ret < 0) {
 441                 RTE_LOG(ERR, USER1,
 442                         "Source mbufs could not be allocated "
 443                         "from the mempool\n");
 444                 goto exit;
 445         }
 446
 447         for (i = 0; i < num_bufs; i++) {
 448                 data_ptr = rte_pktmbuf_append(uncomp_bufs[i],
 449                                         strlen(test_bufs[i]) + 1);
 450                 snprintf(data_ptr, strlen(test_bufs[i]) + 1, "%s",
 451                                         test_bufs[i]);
 452         }
 453
 454         /* Prepare the destination mbufs */
 455         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, comp_bufs, num_bufs);
 456         if (ret < 0) {
 457                 RTE_LOG(ERR, USER1,
 458                         "Destination mbufs could not be allocated "
 459                         "from the mempool\n");
 460                 goto exit;
 461         }
 462
 463         for (i = 0; i < num_bufs; i++)
 464                 rte_pktmbuf_append(comp_bufs[i],
 465                         strlen(test_bufs[i]) * COMPRESS_BUF_SIZE_RATIO);
 466
 467         /* Build the compression operations */
 468         ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
 469         if (ret < 0) {
 470                 RTE_LOG(ERR, USER1,
 471                         "Compress operations could not be allocated "
 472                         "from the mempool\n");
 473                 goto exit;
 474         }
 475
 476         for (i = 0; i < num_bufs; i++) {
 477                 ops[i]->m_src = uncomp_bufs[i];
 478                 ops[i]->m_dst = comp_bufs[i];
 479                 ops[i]->src.offset = 0;
 480                 ops[i]->src.length = rte_pktmbuf_pkt_len(uncomp_bufs[i]);
 481                 ops[i]->dst.offset = 0;
 482                 if (state == RTE_COMP_OP_STATELESS) {
 483                         ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
 484                 } else {
 485                         RTE_LOG(ERR, USER1,
 486                                 "Stateful operations are not supported "
 487                                 "in these tests yet\n");
 488                         goto exit;
 489                 }
 490                 ops[i]->input_chksum = 0;
 491                 /*
 492                  * Store original operation index in private data,
 493                  * since ordering does not have to be maintained,
 494                  * when dequeueing from compressdev, so a comparison
 495                  * at the end of the test can be done.
 496                  */
 497                 priv_data = (struct priv_op_data *) (ops[i] + 1);
 498                 priv_data->orig_idx = i;
 499         }
 500
 501         /* Compress data (either with Zlib API or compressdev API */
 502         if (zlib_dir == ZLIB_COMPRESS || zlib_dir == ZLIB_ALL) {
 503                 for (i = 0; i < num_bufs; i++) {
 504                         const struct rte_comp_xform *compress_xform =
 505                                 compress_xforms[i % num_xforms];
 506                         ret = compress_zlib(ops[i], compress_xform,
 507                                         DEFAULT_MEM_LEVEL);
 508                         if (ret < 0)
 509                                 goto exit;
 510
 511                         ops_processed[i] = ops[i];
 512                 }
 513         } else {
 514                 /* Create compress private xform data */
 515                 for (i = 0; i < num_xforms; i++) {
 516                         ret = rte_compressdev_private_xform_create(0,
 517                                 (const struct rte_comp_xform *)compress_xforms[i],
 518                                 &priv_xforms[i]);
 519                         if (ret < 0) {
 520                                 RTE_LOG(ERR, USER1,
 521                                         "Compression private xform "
 522                                         "could not be created\n");
 523                                 goto exit;
 524                         }
 525                         num_priv_xforms++;
 526                 }
 527
 528                 if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 529                         /* Attach shareable private xform data to ops */
 530                         for (i = 0; i < num_bufs; i++)
 531                                 ops[i]->private_xform = priv_xforms[i % num_xforms];
 532                 } else {
 533                         /* Create rest of the private xforms for the other ops */
 534                         for (i = num_xforms; i < num_bufs; i++) {
 535                                 ret = rte_compressdev_private_xform_create(0,
 536                                         compress_xforms[i % num_xforms],
 537                                         &priv_xforms[i]);
 538                                 if (ret < 0) {
 539                                         RTE_LOG(ERR, USER1,
 540                                                 "Compression private xform "
 541                                                 "could not be created\n");
 542                                         goto exit;
 543                                 }
 544                                 num_priv_xforms++;
 545                         }
 546
 547                         /* Attach non shareable private xform data to ops */
 548                         for (i = 0; i < num_bufs; i++)
 549                                 ops[i]->private_xform = priv_xforms[i];
 550                 }
 551
 552                 /* Enqueue and dequeue all operations */
 553                 num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
 554                 if (num_enqd < num_bufs) {
 555                         RTE_LOG(ERR, USER1,
 556                                 "The operations could not be enqueued\n");
 557                         goto exit;
 558                 }
 559
 560                 num_total_deqd = 0;
 561                 do {
 562                         /*
 563                          * If retrying a dequeue call, wait for 10 ms to allow
 564                          * enough time to the driver to process the operations
 565                          */
 566                         if (deqd_retries != 0) {
 567                                 /*
 568                                  * Avoid infinite loop if not all the
 569                                  * operations get out of the device
 570                                  */
 571                                 if (deqd_retries == MAX_DEQD_RETRIES) {
 572                                         RTE_LOG(ERR, USER1,
 573                                                 "Not all operations could be "
 574                                                 "dequeued\n");
 575                                         goto exit;
 576                                 }
 577                                 usleep(DEQUEUE_WAIT_TIME);
 578                         }
 579                         num_deqd = rte_compressdev_dequeue_burst(0, 0,
 580                                         &ops_processed[num_total_deqd], num_bufs);
 581                         num_total_deqd += num_deqd;
 582                         deqd_retries++;
 583                 } while (num_total_deqd < num_enqd);
 584
 585                 deqd_retries = 0;
 586
 587                 /* Free compress private xforms */
 588                 for (i = 0; i < num_priv_xforms; i++) {
 589                         rte_compressdev_private_xform_free(0, priv_xforms[i]);
 590                         priv_xforms[i] = NULL;
 591                 }
 592                 num_priv_xforms = 0;
 593         }
 594
 595         for (i = 0; i < num_bufs; i++) {
 596                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 597                 uint16_t xform_idx = priv_data->orig_idx % num_xforms;
 598                 const struct rte_comp_compress_xform *compress_xform =
 599                                 &compress_xforms[xform_idx]->compress;
 600                 enum rte_comp_huffman huffman_type =
 601                         compress_xform->deflate.huffman;
 602                 RTE_LOG(DEBUG, USER1, "Buffer %u compressed from %u to %u bytes "
 603                         "(level = %d, huffman = %s)\n",
 604                         buf_idx[priv_data->orig_idx],
 605                         ops_processed[i]->consumed, ops_processed[i]->produced,
 606                         compress_xform->level,
 607                         huffman_type_strings[huffman_type]);
 608                 RTE_LOG(DEBUG, USER1, "Compression ratio = %.2f",
 609                         (float)ops_processed[i]->produced /
 610                         ops_processed[i]->consumed * 100);
 611                 ops[i] = NULL;
 612         }
 613
 614         /*
 615          * Check operation status and free source mbufs (destination mbuf and
 616          * compress operation information is needed for the decompression stage)
 617          */
 618         for (i = 0; i < num_bufs; i++) {
 619                 if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 620                         RTE_LOG(ERR, USER1,
 621                                 "Some operations were not successful\n");
 622                         goto exit;
 623                 }
 624                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 625                 rte_pktmbuf_free(uncomp_bufs[priv_data->orig_idx]);
 626                 uncomp_bufs[priv_data->orig_idx] = NULL;
 627         }
 628
 629         /* Allocate buffers for decompressed data */
 630         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, uncomp_bufs, num_bufs);
 631         if (ret < 0) {
 632                 RTE_LOG(ERR, USER1,
 633                         "Destination mbufs could not be allocated "
 634                         "from the mempool\n");
 635                 goto exit;
 636         }
 637
 638         for (i = 0; i < num_bufs; i++) {
 639                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 640                 rte_pktmbuf_append(uncomp_bufs[i],
 641                                 strlen(test_bufs[priv_data->orig_idx]) + 1);
 642         }
 643
 644         /* Build the decompression operations */
 645         ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
 646         if (ret < 0) {
 647                 RTE_LOG(ERR, USER1,
 648                         "Decompress operations could not be allocated "
 649                         "from the mempool\n");
 650                 goto exit;
 651         }
 652
 653         /* Source buffer is the compressed data from the previous operations */
 654         for (i = 0; i < num_bufs; i++) {
 655                 ops[i]->m_src = ops_processed[i]->m_dst;
 656                 ops[i]->m_dst = uncomp_bufs[i];
 657                 ops[i]->src.offset = 0;
 658                 /*
 659                  * Set the length of the compressed data to the
 660                  * number of bytes that were produced in the previous stage
 661                  */
 662                 ops[i]->src.length = ops_processed[i]->produced;
 663                 ops[i]->dst.offset = 0;
 664                 if (state == RTE_COMP_OP_STATELESS) {
 665                         ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
 666                 } else {
 667                         RTE_LOG(ERR, USER1,
 668                                 "Stateful operations are not supported "
 669                                 "in these tests yet\n");
 670                         goto exit;
 671                 }
 672                 ops[i]->input_chksum = 0;
 673                 /*
 674                  * Copy private data from previous operations,
 675                  * to keep the pointer to the original buffer
 676                  */
 677                 memcpy(ops[i] + 1, ops_processed[i] + 1,
 678                                 sizeof(struct priv_op_data));
 679         }
 680
 681         /*
 682          * Free the previous compress operations,
 683          * as it is not needed anymore
 684          */
 685         for (i = 0; i < num_bufs; i++) {
 686                 rte_comp_op_free(ops_processed[i]);
 687                 ops_processed[i] = NULL;
 688         }
 689
 690         /* Decompress data (either with Zlib API or compressdev API */
 691         if (zlib_dir == ZLIB_DECOMPRESS || zlib_dir == ZLIB_ALL) {
 692                 for (i = 0; i < num_bufs; i++) {
 693                         priv_data = (struct priv_op_data *)(ops[i] + 1);
 694                         uint16_t xform_idx = priv_data->orig_idx % num_xforms;
 695                         const struct rte_comp_xform *decompress_xform =
 696                                 decompress_xforms[xform_idx];
 697
 698                         ret = decompress_zlib(ops[i], decompress_xform);
 699                         if (ret < 0)
 700                                 goto exit;
 701
 702                         ops_processed[i] = ops[i];
 703                 }
 704         } else {
 705                 /* Create decompress private xform data */
 706                 for (i = 0; i < num_xforms; i++) {
 707                         ret = rte_compressdev_private_xform_create(0,
 708                                 (const struct rte_comp_xform *)decompress_xforms[i],
 709                                 &priv_xforms[i]);
 710                         if (ret < 0) {
 711                                 RTE_LOG(ERR, USER1,
 712                                         "Decompression private xform "
 713                                         "could not be created\n");
 714                                 goto exit;
 715                         }
 716                         num_priv_xforms++;
 717                 }
 718
 719                 if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 720                         /* Attach shareable private xform data to ops */
 721                         for (i = 0; i < num_bufs; i++) {
 722                                 priv_data = (struct priv_op_data *)(ops[i] + 1);
 723                                 uint16_t xform_idx = priv_data->orig_idx %
 724                                                                 num_xforms;
 725                                 ops[i]->private_xform = priv_xforms[xform_idx];
 726                         }
 727                 } else {
 728                         /* Create rest of the private xforms for the other ops */
 729                         for (i = num_xforms; i < num_bufs; i++) {
 730                                 ret = rte_compressdev_private_xform_create(0,
 731                                         decompress_xforms[i % num_xforms],
 732                                         &priv_xforms[i]);
 733                                 if (ret < 0) {
 734                                         RTE_LOG(ERR, USER1,
 735                                                 "Decompression private xform "
 736                                                 "could not be created\n");
 737                                         goto exit;
 738                                 }
 739                                 num_priv_xforms++;
 740                         }
 741
 742                         /* Attach non shareable private xform data to ops */
 743                         for (i = 0; i < num_bufs; i++) {
 744                                 priv_data = (struct priv_op_data *)(ops[i] + 1);
 745                                 uint16_t xform_idx = priv_data->orig_idx;
 746                                 ops[i]->private_xform = priv_xforms[xform_idx];
 747                         }
 748                 }
 749
 750                 /* Enqueue and dequeue all operations */
 751                 num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
 752                 if (num_enqd < num_bufs) {
 753                         RTE_LOG(ERR, USER1,
 754                                 "The operations could not be enqueued\n");
 755                         goto exit;
 756                 }
 757
 758                 num_total_deqd = 0;
 759                 do {
 760                         /*
 761                          * If retrying a dequeue call, wait for 10 ms to allow
 762                          * enough time to the driver to process the operations
 763                          */
 764                         if (deqd_retries != 0) {
 765                                 /*
 766                                  * Avoid infinite loop if not all the
 767                                  * operations get out of the device
 768                                  */
 769                                 if (deqd_retries == MAX_DEQD_RETRIES) {
 770                                         RTE_LOG(ERR, USER1,
 771                                                 "Not all operations could be "
 772                                                 "dequeued\n");
 773                                         goto exit;
 774                                 }
 775                                 usleep(DEQUEUE_WAIT_TIME);
 776                         }
 777                         num_deqd = rte_compressdev_dequeue_burst(0, 0,
 778                                         &ops_processed[num_total_deqd], num_bufs);
 779                         num_total_deqd += num_deqd;
 780                         deqd_retries++;
 781                 } while (num_total_deqd < num_enqd);
 782
 783                 deqd_retries = 0;
 784         }
 785
 786         for (i = 0; i < num_bufs; i++) {
 787                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 788                 RTE_LOG(DEBUG, USER1, "Buffer %u decompressed from %u to %u bytes\n",
 789                         buf_idx[priv_data->orig_idx],
 790                         ops_processed[i]->consumed, ops_processed[i]->produced);
 791                 ops[i] = NULL;
 792         }
 793
 794         /*
 795          * Check operation status and free source mbuf (destination mbuf and
 796          * compress operation information is still needed)
 797          */
 798         for (i = 0; i < num_bufs; i++) {
 799                 if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 800                         RTE_LOG(ERR, USER1,
 801                                 "Some operations were not successful\n");
 802                         goto exit;
 803                 }
 804                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 805                 rte_pktmbuf_free(comp_bufs[priv_data->orig_idx]);
 806                 comp_bufs[priv_data->orig_idx] = NULL;
 807         }
 808
 809         /*
 810          * Compare the original stream with the decompressed stream
 811          * (in size and the data)
 812          */
 813         for (i = 0; i < num_bufs; i++) {
 814                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 815                 const char *buf1 = test_bufs[priv_data->orig_idx];
 816                 const char *buf2 = rte_pktmbuf_mtod(ops_processed[i]->m_dst,
 817                                 const char *);
 818
 819                 if (compare_buffers(buf1, strlen(buf1) + 1,
 820                                 buf2, ops_processed[i]->produced) < 0)
 821                         goto exit;
 822         }
 823
 824         ret_status = 0;
 825
 826 exit:
 827         /* Free resources */
 828         for (i = 0; i < num_bufs; i++) {
 829                 rte_pktmbuf_free(uncomp_bufs[i]);
 830                 rte_pktmbuf_free(comp_bufs[i]);
 831                 rte_comp_op_free(ops[i]);
 832                 rte_comp_op_free(ops_processed[i]);
 833         }
 834         for (i = 0; i < num_priv_xforms; i++) {
 835                 if (priv_xforms[i] != NULL)
 836                         rte_compressdev_private_xform_free(0, priv_xforms[i]);
 837         }
 838
 839         return ret_status;
 840 }
 841
 842 static int
 843 test_compressdev_deflate_stateless_fixed(void)
 844 {
 845         struct comp_testsuite_params *ts_params = &testsuite_params;
 846         const char *test_buffer;
 847         uint16_t i;
 848         int ret;
 849         struct rte_comp_xform *compress_xform =
 850                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 851
 852         if (compress_xform == NULL) {
 853                 RTE_LOG(ERR, USER1,
 854                         "Compress xform could not be created\n");
 855                 ret = TEST_FAILED;
 856                 goto exit;
 857         }
 858
 859         memcpy(compress_xform, ts_params->def_comp_xform,
 860                         sizeof(struct rte_comp_xform));
 861         compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
 862
 863         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
 864                 test_buffer = compress_test_bufs[i];
 865
 866                 /* Compress with compressdev, decompress with Zlib */
 867                 if (test_deflate_comp_decomp(&test_buffer, 1,
 868                                 &i,
 869                                 &compress_xform,
 870                                 &ts_params->def_decomp_xform,
 871                                 1,
 872                                 RTE_COMP_OP_STATELESS,
 873                                 ZLIB_DECOMPRESS) < 0) {
 874                         ret = TEST_FAILED;
 875                         goto exit;
 876                 }
 877
 878                 /* Compress with Zlib, decompress with compressdev */
 879                 if (test_deflate_comp_decomp(&test_buffer, 1,
 880                                 &i,
 881                                 &compress_xform,
 882                                 &ts_params->def_decomp_xform,
 883                                 1,
 884                                 RTE_COMP_OP_STATELESS,
 885                                 ZLIB_COMPRESS) < 0) {
 886                         ret = TEST_FAILED;
 887                         goto exit;
 888                 }
 889         }
 890
 891         ret = TEST_SUCCESS;
 892
 893 exit:
 894         rte_free(compress_xform);
 895         return ret;
 896 }
 897
 898 static int
 899 test_compressdev_deflate_stateless_dynamic(void)
 900 {
 901         struct comp_testsuite_params *ts_params = &testsuite_params;
 902         const char *test_buffer;
 903         uint16_t i;
 904         int ret;
 905         struct rte_comp_xform *compress_xform =
 906                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 907
 908         if (compress_xform == NULL) {
 909                 RTE_LOG(ERR, USER1,
 910                         "Compress xform could not be created\n");
 911                 ret = TEST_FAILED;
 912                 goto exit;
 913         }
 914
 915         memcpy(compress_xform, ts_params->def_comp_xform,
 916                         sizeof(struct rte_comp_xform));
 917         compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_DYNAMIC;
 918
 919         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
 920                 test_buffer = compress_test_bufs[i];
 921
 922                 /* Compress with compressdev, decompress with Zlib */
 923                 if (test_deflate_comp_decomp(&test_buffer, 1,
 924                                 &i,
 925                                 &compress_xform,
 926                                 &ts_params->def_decomp_xform,
 927                                 1,
 928                                 RTE_COMP_OP_STATELESS,
 929                                 ZLIB_DECOMPRESS) < 0) {
 930                         ret = TEST_FAILED;
 931                         goto exit;
 932                 }
 933
 934                 /* Compress with Zlib, decompress with compressdev */
 935                 if (test_deflate_comp_decomp(&test_buffer, 1,
 936                                 &i,
 937                                 &compress_xform,
 938                                 &ts_params->def_decomp_xform,
 939                                 1,
 940                                 RTE_COMP_OP_STATELESS,
 941                                 ZLIB_COMPRESS) < 0) {
 942                         ret = TEST_FAILED;
 943                         goto exit;
 944                 }
 945         }
 946
 947         ret = TEST_SUCCESS;
 948
 949 exit:
 950         rte_free(compress_xform);
 951         return ret;
 952 }
 953
 954 static int
 955 test_compressdev_deflate_stateless_multi_op(void)
 956 {
 957         struct comp_testsuite_params *ts_params = &testsuite_params;
 958         uint16_t num_bufs = RTE_DIM(compress_test_bufs);
 959         uint16_t buf_idx[num_bufs];
 960         uint16_t i;
 961
 962         for (i = 0; i < num_bufs; i++)
 963                 buf_idx[i] = i;
 964
 965         /* Compress with compressdev, decompress with Zlib */
 966         if (test_deflate_comp_decomp(compress_test_bufs, num_bufs,
 967                         buf_idx,
 968                         &ts_params->def_comp_xform,
 969                         &ts_params->def_decomp_xform,
 970                         1,
 971                         RTE_COMP_OP_STATELESS,
 972                         ZLIB_DECOMPRESS) < 0)
 973                 return TEST_FAILED;
 974
 975         /* Compress with Zlib, decompress with compressdev */
 976         if (test_deflate_comp_decomp(compress_test_bufs, num_bufs,
 977                         buf_idx,
 978                         &ts_params->def_comp_xform,
 979                         &ts_params->def_decomp_xform,
 980                         1,
 981                         RTE_COMP_OP_STATELESS,
 982                         ZLIB_COMPRESS) < 0)
 983                 return TEST_FAILED;
 984
 985         return TEST_SUCCESS;
 986 }
 987
 988 static int
 989 test_compressdev_deflate_stateless_multi_level(void)
 990 {
 991         struct comp_testsuite_params *ts_params = &testsuite_params;
 992         const char *test_buffer;
 993         unsigned int level;
 994         uint16_t i;
 995         int ret;
 996         struct rte_comp_xform *compress_xform =
 997                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 998
 999         if (compress_xform == NULL) {
1000                 RTE_LOG(ERR, USER1,
1001                         "Compress xform could not be created\n");
1002                 ret = TEST_FAILED;
1003                 goto exit;
1004         }
1005
1006         memcpy(compress_xform, ts_params->def_comp_xform,
1007                         sizeof(struct rte_comp_xform));
1008
1009         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
1010                 test_buffer = compress_test_bufs[i];
1011                 for (level = RTE_COMP_LEVEL_MIN; level <= RTE_COMP_LEVEL_MAX;
1012                                 level++) {
1013                         compress_xform->compress.level = level;
1014                         /* Compress with compressdev, decompress with Zlib */
1015                         if (test_deflate_comp_decomp(&test_buffer, 1,
1016                                         &i,
1017                                         &compress_xform,
1018                                         &ts_params->def_decomp_xform,
1019                                         1,
1020                                         RTE_COMP_OP_STATELESS,
1021                                         ZLIB_DECOMPRESS) < 0) {
1022                                 ret = TEST_FAILED;
1023                                 goto exit;
1024                         }
1025                 }
1026         }
1027
1028         ret = TEST_SUCCESS;
1029
1030 exit:
1031         rte_free(compress_xform);
1032         return ret;
1033 }
1034
1035 #define NUM_XFORMS 3
1036 static int
1037 test_compressdev_deflate_stateless_multi_xform(void)
1038 {
1039         struct comp_testsuite_params *ts_params = &testsuite_params;
1040         uint16_t num_bufs = NUM_XFORMS;
1041         struct rte_comp_xform *compress_xforms[NUM_XFORMS] = {NULL};
1042         struct rte_comp_xform *decompress_xforms[NUM_XFORMS] = {NULL};
1043         const char *test_buffers[NUM_XFORMS];
1044         uint16_t i;
1045         unsigned int level = RTE_COMP_LEVEL_MIN;
1046         uint16_t buf_idx[num_bufs];
1047
1048         int ret;
1049
1050         /* Create multiple xforms with various levels */
1051         for (i = 0; i < NUM_XFORMS; i++) {
1052                 compress_xforms[i] = rte_malloc(NULL,
1053                                 sizeof(struct rte_comp_xform), 0);
1054                 if (compress_xforms[i] == NULL) {
1055                         RTE_LOG(ERR, USER1,
1056                                 "Compress xform could not be created\n");
1057                         ret = TEST_FAILED;
1058                         goto exit;
1059                 }
1060
1061                 memcpy(compress_xforms[i], ts_params->def_comp_xform,
1062                                 sizeof(struct rte_comp_xform));
1063                 compress_xforms[i]->compress.level = level;
1064                 level++;
1065
1066                 decompress_xforms[i] = rte_malloc(NULL,
1067                                 sizeof(struct rte_comp_xform), 0);
1068                 if (decompress_xforms[i] == NULL) {
1069                         RTE_LOG(ERR, USER1,
1070                                 "Decompress xform could not be created\n");
1071                         ret = TEST_FAILED;
1072                         goto exit;
1073                 }
1074
1075                 memcpy(decompress_xforms[i], ts_params->def_decomp_xform,
1076                                 sizeof(struct rte_comp_xform));
1077         }
1078
1079         for (i = 0; i < NUM_XFORMS; i++) {
1080                 buf_idx[i] = 0;
1081                 /* Use the same buffer in all sessions */
1082                 test_buffers[i] = compress_test_bufs[0];
1083         }
1084         /* Compress with compressdev, decompress with Zlib */
1085         if (test_deflate_comp_decomp(test_buffers, num_bufs,
1086                         buf_idx,
1087                         compress_xforms,
1088                         decompress_xforms,
1089                         NUM_XFORMS,
1090                         RTE_COMP_OP_STATELESS,
1091                         ZLIB_DECOMPRESS) < 0) {
1092                 ret = TEST_FAILED;
1093                 goto exit;
1094         }
1095
1096         ret = TEST_SUCCESS;
1097 exit:
1098         for (i = 0; i < NUM_XFORMS; i++) {
1099                 rte_free(compress_xforms[i]);
1100                 rte_free(decompress_xforms[i]);
1101         }
1102
1103         return ret;
1104 }
1105
1106 static struct unit_test_suite compressdev_testsuite  = {
1107         .suite_name = "compressdev unit test suite",
1108         .setup = testsuite_setup,
1109         .teardown = testsuite_teardown,
1110         .unit_test_cases = {
1111                 TEST_CASE_ST(NULL, NULL,
1112                         test_compressdev_invalid_configuration),
1113                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1114                         test_compressdev_deflate_stateless_fixed),
1115                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1116                         test_compressdev_deflate_stateless_dynamic),
1117                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1118                         test_compressdev_deflate_stateless_multi_op),
1119                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1120                         test_compressdev_deflate_stateless_multi_level),
1121                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1122                         test_compressdev_deflate_stateless_multi_xform),
1123                 TEST_CASES_END() /**< NULL terminate unit test array */
1124         }
1125 };
1126
1127 static int
1128 test_compressdev(void)
1129 {
1130         return unit_test_suite_runner(&compressdev_testsuite);
1131 }
1132
1133 REGISTER_TEST_COMMAND(compressdev_autotest, test_compressdev);