test/test/test_compressdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <string.h>
   5 #include <zlib.h>
   6 #include <math.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_malloc.h>
  10 #include <rte_mempool.h>
  11 #include <rte_mbuf.h>
  12 #include <rte_compressdev.h>
  13
  14 #include "test_compressdev_test_buffer.h"
  15 #include "test.h"
  16
  17 #define DEFAULT_WINDOW_SIZE 15
  18 #define DEFAULT_MEM_LEVEL 8
  19 #define MAX_DEQD_RETRIES 10
  20 #define DEQUEUE_WAIT_TIME 10000
  21
  22 /*
  23  * 30% extra size for compressed data compared to original data,
  24  * in case data size cannot be reduced and it is actually bigger
  25  * due to the compress block headers
  26  */
  27 #define COMPRESS_BUF_SIZE_RATIO 1.3
  28 #define NUM_MBUFS 16
  29 #define NUM_OPS 16
  30 #define NUM_MAX_XFORMS 16
  31 #define NUM_MAX_INFLIGHT_OPS 128
  32 #define CACHE_SIZE 0
  33
  34 const char *
  35 huffman_type_strings[] = {
  36         [RTE_COMP_HUFFMAN_DEFAULT]      = "PMD default",
  37         [RTE_COMP_HUFFMAN_FIXED]        = "Fixed",
  38         [RTE_COMP_HUFFMAN_DYNAMIC]      = "Dynamic"
  39 };
  40
  41 enum zlib_direction {
  42         ZLIB_NONE,
  43         ZLIB_COMPRESS,
  44         ZLIB_DECOMPRESS,
  45         ZLIB_ALL
  46 };
  47
  48 struct priv_op_data {
  49         uint16_t orig_idx;
  50 };
  51
  52 struct comp_testsuite_params {
  53         struct rte_mempool *mbuf_pool;
  54         struct rte_mempool *op_pool;
  55         struct rte_comp_xform *def_comp_xform;
  56         struct rte_comp_xform *def_decomp_xform;
  57 };
  58
  59 static struct comp_testsuite_params testsuite_params = { 0 };
  60
  61 static void
  62 testsuite_teardown(void)
  63 {
  64         struct comp_testsuite_params *ts_params = &testsuite_params;
  65
  66         rte_mempool_free(ts_params->mbuf_pool);
  67         rte_mempool_free(ts_params->op_pool);
  68         rte_free(ts_params->def_comp_xform);
  69         rte_free(ts_params->def_decomp_xform);
  70 }
  71
  72 static int
  73 testsuite_setup(void)
  74 {
  75         struct comp_testsuite_params *ts_params = &testsuite_params;
  76         unsigned int i;
  77
  78         if (rte_compressdev_count() == 0) {
  79                 RTE_LOG(ERR, USER1, "Need at least one compress device\n");
  80                 return TEST_FAILED;
  81         }
  82
  83         uint32_t max_buf_size = 0;
  84         for (i = 0; i < RTE_DIM(compress_test_bufs); i++)
  85                 max_buf_size = RTE_MAX(max_buf_size,
  86                                 strlen(compress_test_bufs[i]) + 1);
  87
  88         max_buf_size *= COMPRESS_BUF_SIZE_RATIO;
  89         /*
  90          * Buffers to be used in compression and decompression.
  91          * Since decompressed data might be larger than
  92          * compressed data (due to block header),
  93          * buffers should be big enough for both cases.
  94          */
  95         ts_params->mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool",
  96                         NUM_MBUFS,
  97                         CACHE_SIZE, 0,
  98                         max_buf_size + RTE_PKTMBUF_HEADROOM,
  99                         rte_socket_id());
 100         if (ts_params->mbuf_pool == NULL) {
 101                 RTE_LOG(ERR, USER1, "Large mbuf pool could not be created\n");
 102                 return TEST_FAILED;
 103         }
 104
 105         ts_params->op_pool = rte_comp_op_pool_create("op_pool", NUM_OPS,
 106                                 0, sizeof(struct priv_op_data),
 107                                 rte_socket_id());
 108         if (ts_params->op_pool == NULL) {
 109                 RTE_LOG(ERR, USER1, "Operation pool could not be created\n");
 110                 goto exit;
 111         }
 112
 113         ts_params->def_comp_xform =
 114                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 115         if (ts_params->def_comp_xform == NULL) {
 116                 RTE_LOG(ERR, USER1,
 117                         "Default compress xform could not be created\n");
 118                 goto exit;
 119         }
 120         ts_params->def_decomp_xform =
 121                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 122         if (ts_params->def_decomp_xform == NULL) {
 123                 RTE_LOG(ERR, USER1,
 124                         "Default decompress xform could not be created\n");
 125                 goto exit;
 126         }
 127
 128         /* Initializes default values for compress/decompress xforms */
 129         ts_params->def_comp_xform->type = RTE_COMP_COMPRESS;
 130         ts_params->def_comp_xform->compress.algo = RTE_COMP_ALGO_DEFLATE,
 131         ts_params->def_comp_xform->compress.deflate.huffman =
 132                                                 RTE_COMP_HUFFMAN_DEFAULT;
 133         ts_params->def_comp_xform->compress.level = RTE_COMP_LEVEL_PMD_DEFAULT;
 134         ts_params->def_comp_xform->compress.chksum = RTE_COMP_CHECKSUM_NONE;
 135         ts_params->def_comp_xform->compress.window_size = DEFAULT_WINDOW_SIZE;
 136
 137         ts_params->def_decomp_xform->type = RTE_COMP_DECOMPRESS;
 138         ts_params->def_decomp_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE,
 139         ts_params->def_decomp_xform->decompress.chksum = RTE_COMP_CHECKSUM_NONE;
 140         ts_params->def_decomp_xform->decompress.window_size = DEFAULT_WINDOW_SIZE;
 141
 142         return TEST_SUCCESS;
 143
 144 exit:
 145         testsuite_teardown();
 146
 147         return TEST_FAILED;
 148 }
 149
 150 static int
 151 generic_ut_setup(void)
 152 {
 153         /* Configure compressdev (one device, one queue pair) */
 154         struct rte_compressdev_config config = {
 155                 .socket_id = rte_socket_id(),
 156                 .nb_queue_pairs = 1,
 157                 .max_nb_priv_xforms = NUM_MAX_XFORMS,
 158                 .max_nb_streams = 0
 159         };
 160
 161         if (rte_compressdev_configure(0, &config) < 0) {
 162                 RTE_LOG(ERR, USER1, "Device configuration failed\n");
 163                 return -1;
 164         }
 165
 166         if (rte_compressdev_queue_pair_setup(0, 0, NUM_MAX_INFLIGHT_OPS,
 167                         rte_socket_id()) < 0) {
 168                 RTE_LOG(ERR, USER1, "Queue pair setup failed\n");
 169                 return -1;
 170         }
 171
 172         if (rte_compressdev_start(0) < 0) {
 173                 RTE_LOG(ERR, USER1, "Device could not be started\n");
 174                 return -1;
 175         }
 176
 177         return 0;
 178 }
 179
 180 static void
 181 generic_ut_teardown(void)
 182 {
 183         rte_compressdev_stop(0);
 184         if (rte_compressdev_close(0) < 0)
 185                 RTE_LOG(ERR, USER1, "Device could not be closed\n");
 186 }
 187
 188 static int
 189 compare_buffers(const char *buffer1, uint32_t buffer1_len,
 190                 const char *buffer2, uint32_t buffer2_len)
 191 {
 192         if (buffer1_len != buffer2_len) {
 193                 RTE_LOG(ERR, USER1, "Buffer lengths are different\n");
 194                 return -1;
 195         }
 196
 197         if (memcmp(buffer1, buffer2, buffer1_len) != 0) {
 198                 RTE_LOG(ERR, USER1, "Buffers are different\n");
 199                 return -1;
 200         }
 201
 202         return 0;
 203 }
 204
 205 /*
 206  * Maps compressdev and Zlib flush flags
 207  */
 208 static int
 209 map_zlib_flush_flag(enum rte_comp_flush_flag flag)
 210 {
 211         switch (flag) {
 212         case RTE_COMP_FLUSH_NONE:
 213                 return Z_NO_FLUSH;
 214         case RTE_COMP_FLUSH_SYNC:
 215                 return Z_SYNC_FLUSH;
 216         case RTE_COMP_FLUSH_FULL:
 217                 return Z_FULL_FLUSH;
 218         case RTE_COMP_FLUSH_FINAL:
 219                 return Z_FINISH;
 220         /*
 221          * There should be only the values above,
 222          * so this should never happen
 223          */
 224         default:
 225                 return -1;
 226         }
 227 }
 228
 229 static int
 230 compress_zlib(struct rte_comp_op *op,
 231                 const struct rte_comp_xform *xform, int mem_level)
 232 {
 233         z_stream stream;
 234         int zlib_flush;
 235         int strategy, window_bits, comp_level;
 236         int ret = -1;
 237
 238         /* initialize zlib stream */
 239         stream.zalloc = Z_NULL;
 240         stream.zfree = Z_NULL;
 241         stream.opaque = Z_NULL;
 242
 243         if (xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
 244                 strategy = Z_FIXED;
 245         else
 246                 strategy = Z_DEFAULT_STRATEGY;
 247
 248         /*
 249          * Window bits is the base two logarithm of the window size (in bytes).
 250          * When doing raw DEFLATE, this number will be negative.
 251          */
 252         window_bits = -(xform->compress.window_size);
 253
 254         comp_level = xform->compress.level;
 255
 256         if (comp_level != RTE_COMP_LEVEL_NONE)
 257                 ret = deflateInit2(&stream, comp_level, Z_DEFLATED,
 258                         window_bits, mem_level, strategy);
 259         else
 260                 ret = deflateInit(&stream, Z_NO_COMPRESSION);
 261
 262         if (ret != Z_OK) {
 263                 printf("Zlib deflate could not be initialized\n");
 264                 goto exit;
 265         }
 266
 267         /* Assuming stateless operation */
 268         stream.avail_in = op->src.length;
 269         stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
 270         stream.avail_out = op->m_dst->data_len;
 271         stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
 272
 273         /* Stateless operation, all buffer will be compressed in one go */
 274         zlib_flush = map_zlib_flush_flag(op->flush_flag);
 275         ret = deflate(&stream, zlib_flush);
 276
 277         if (stream.avail_in != 0) {
 278                 RTE_LOG(ERR, USER1, "Buffer could not be read entirely\n");
 279                 goto exit;
 280         }
 281
 282         if (ret != Z_STREAM_END)
 283                 goto exit;
 284
 285         op->consumed = op->src.length - stream.avail_in;
 286         op->produced = op->m_dst->data_len - stream.avail_out;
 287         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 288
 289         deflateReset(&stream);
 290
 291         ret = 0;
 292 exit:
 293         deflateEnd(&stream);
 294
 295         return ret;
 296 }
 297
 298 static int
 299 decompress_zlib(struct rte_comp_op *op,
 300                 const struct rte_comp_xform *xform)
 301 {
 302         z_stream stream;
 303         int window_bits;
 304         int zlib_flush;
 305         int ret = TEST_FAILED;
 306
 307         /* initialize zlib stream */
 308         stream.zalloc = Z_NULL;
 309         stream.zfree = Z_NULL;
 310         stream.opaque = Z_NULL;
 311
 312         /*
 313          * Window bits is the base two logarithm of the window size (in bytes).
 314          * When doing raw DEFLATE, this number will be negative.
 315          */
 316         window_bits = -(xform->decompress.window_size);
 317
 318         ret = inflateInit2(&stream, window_bits);
 319
 320         if (ret != Z_OK) {
 321                 printf("Zlib deflate could not be initialized\n");
 322                 goto exit;
 323         }
 324
 325         /* Assuming stateless operation */
 326         stream.avail_in = op->src.length;
 327         stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
 328         stream.avail_out = op->m_dst->data_len;
 329         stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
 330
 331         /* Stateless operation, all buffer will be compressed in one go */
 332         zlib_flush = map_zlib_flush_flag(op->flush_flag);
 333         ret = inflate(&stream, zlib_flush);
 334
 335         if (stream.avail_in != 0) {
 336                 RTE_LOG(ERR, USER1, "Buffer could not be read entirely\n");
 337                 goto exit;
 338         }
 339
 340         if (ret != Z_STREAM_END)
 341                 goto exit;
 342
 343         op->consumed = op->src.length - stream.avail_in;
 344         op->produced = op->m_dst->data_len - stream.avail_out;
 345         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 346
 347         inflateReset(&stream);
 348
 349         ret = 0;
 350 exit:
 351         inflateEnd(&stream);
 352
 353         return ret;
 354 }
 355
 356 /*
 357  * Compresses and decompresses buffer with compressdev API and Zlib API
 358  */
 359 static int
 360 test_deflate_comp_decomp(const char * const test_bufs[],
 361                 unsigned int num_bufs,
 362                 uint16_t buf_idx[],
 363                 struct rte_comp_xform *compress_xforms[],
 364                 struct rte_comp_xform *decompress_xforms[],
 365                 unsigned int num_xforms,
 366                 enum rte_comp_op_type state,
 367                 enum zlib_direction zlib_dir)
 368 {
 369         struct comp_testsuite_params *ts_params = &testsuite_params;
 370         int ret_status = -1;
 371         int ret;
 372         struct rte_mbuf *uncomp_bufs[num_bufs];
 373         struct rte_mbuf *comp_bufs[num_bufs];
 374         struct rte_comp_op *ops[num_bufs];
 375         struct rte_comp_op *ops_processed[num_bufs];
 376         void *priv_xforms[num_bufs];
 377         uint16_t num_enqd, num_deqd, num_total_deqd;
 378         uint16_t num_priv_xforms = 0;
 379         unsigned int deqd_retries = 0;
 380         struct priv_op_data *priv_data;
 381         char *data_ptr;
 382         unsigned int i;
 383         const struct rte_compressdev_capabilities *capa =
 384                 rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 385
 386         /* Initialize all arrays to NULL */
 387         memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
 388         memset(comp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
 389         memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
 390         memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
 391         memset(priv_xforms, 0, sizeof(void *) * num_bufs);
 392
 393         /* Prepare the source mbufs with the data */
 394         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, uncomp_bufs, num_bufs);
 395         if (ret < 0) {
 396                 RTE_LOG(ERR, USER1,
 397                         "Source mbufs could not be allocated "
 398                         "from the mempool\n");
 399                 goto exit;
 400         }
 401
 402         for (i = 0; i < num_bufs; i++) {
 403                 data_ptr = rte_pktmbuf_append(uncomp_bufs[i],
 404                                         strlen(test_bufs[i]) + 1);
 405                 snprintf(data_ptr, strlen(test_bufs[i]) + 1, "%s",
 406                                         test_bufs[i]);
 407         }
 408
 409         /* Prepare the destination mbufs */
 410         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, comp_bufs, num_bufs);
 411         if (ret < 0) {
 412                 RTE_LOG(ERR, USER1,
 413                         "Destination mbufs could not be allocated "
 414                         "from the mempool\n");
 415                 goto exit;
 416         }
 417
 418         for (i = 0; i < num_bufs; i++)
 419                 rte_pktmbuf_append(comp_bufs[i],
 420                         strlen(test_bufs[i]) * COMPRESS_BUF_SIZE_RATIO);
 421
 422         /* Build the compression operations */
 423         ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
 424         if (ret < 0) {
 425                 RTE_LOG(ERR, USER1,
 426                         "Compress operations could not be allocated "
 427                         "from the mempool\n");
 428                 goto exit;
 429         }
 430
 431         for (i = 0; i < num_bufs; i++) {
 432                 ops[i]->m_src = uncomp_bufs[i];
 433                 ops[i]->m_dst = comp_bufs[i];
 434                 ops[i]->src.offset = 0;
 435                 ops[i]->src.length = rte_pktmbuf_pkt_len(uncomp_bufs[i]);
 436                 ops[i]->dst.offset = 0;
 437                 if (state == RTE_COMP_OP_STATELESS) {
 438                         ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
 439                 } else {
 440                         RTE_LOG(ERR, USER1,
 441                                 "Stateful operations are not supported "
 442                                 "in these tests yet\n");
 443                         goto exit;
 444                 }
 445                 ops[i]->input_chksum = 0;
 446                 /*
 447                  * Store original operation index in private data,
 448                  * since ordering does not have to be maintained,
 449                  * when dequeueing from compressdev, so a comparison
 450                  * at the end of the test can be done.
 451                  */
 452                 priv_data = (struct priv_op_data *) (ops[i] + 1);
 453                 priv_data->orig_idx = i;
 454         }
 455
 456         /* Compress data (either with Zlib API or compressdev API */
 457         if (zlib_dir == ZLIB_COMPRESS || zlib_dir == ZLIB_ALL) {
 458                 for (i = 0; i < num_bufs; i++) {
 459                         const struct rte_comp_xform *compress_xform =
 460                                 compress_xforms[i % num_xforms];
 461                         ret = compress_zlib(ops[i], compress_xform,
 462                                         DEFAULT_MEM_LEVEL);
 463                         if (ret < 0)
 464                                 goto exit;
 465
 466                         ops_processed[i] = ops[i];
 467                 }
 468         } else {
 469                 /* Create compress private xform data */
 470                 for (i = 0; i < num_xforms; i++) {
 471                         ret = rte_compressdev_private_xform_create(0,
 472                                 (const struct rte_comp_xform *)compress_xforms[i],
 473                                 &priv_xforms[i]);
 474                         if (ret < 0) {
 475                                 RTE_LOG(ERR, USER1,
 476                                         "Compression private xform "
 477                                         "could not be created\n");
 478                                 goto exit;
 479                         }
 480                         num_priv_xforms++;
 481                 }
 482
 483                 if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 484                         /* Attach shareable private xform data to ops */
 485                         for (i = 0; i < num_bufs; i++)
 486                                 ops[i]->private_xform = priv_xforms[i % num_xforms];
 487                 } else {
 488                         /* Create rest of the private xforms for the other ops */
 489                         for (i = num_xforms; i < num_bufs; i++) {
 490                                 ret = rte_compressdev_private_xform_create(0,
 491                                         compress_xforms[i % num_xforms],
 492                                         &priv_xforms[i]);
 493                                 if (ret < 0) {
 494                                         RTE_LOG(ERR, USER1,
 495                                                 "Compression private xform "
 496                                                 "could not be created\n");
 497                                         goto exit;
 498                                 }
 499                                 num_priv_xforms++;
 500                         }
 501
 502                         /* Attach non shareable private xform data to ops */
 503                         for (i = 0; i < num_bufs; i++)
 504                                 ops[i]->private_xform = priv_xforms[i];
 505                 }
 506
 507                 /* Enqueue and dequeue all operations */
 508                 num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
 509                 if (num_enqd < num_bufs) {
 510                         RTE_LOG(ERR, USER1,
 511                                 "The operations could not be enqueued\n");
 512                         goto exit;
 513                 }
 514
 515                 num_total_deqd = 0;
 516                 do {
 517                         /*
 518                          * If retrying a dequeue call, wait for 10 ms to allow
 519                          * enough time to the driver to process the operations
 520                          */
 521                         if (deqd_retries != 0) {
 522                                 /*
 523                                  * Avoid infinite loop if not all the
 524                                  * operations get out of the device
 525                                  */
 526                                 if (deqd_retries == MAX_DEQD_RETRIES) {
 527                                         RTE_LOG(ERR, USER1,
 528                                                 "Not all operations could be "
 529                                                 "dequeued\n");
 530                                         goto exit;
 531                                 }
 532                                 usleep(DEQUEUE_WAIT_TIME);
 533                         }
 534                         num_deqd = rte_compressdev_dequeue_burst(0, 0,
 535                                         &ops_processed[num_total_deqd], num_bufs);
 536                         num_total_deqd += num_deqd;
 537                         deqd_retries++;
 538                 } while (num_total_deqd < num_enqd);
 539
 540                 deqd_retries = 0;
 541
 542                 /* Free compress private xforms */
 543                 for (i = 0; i < num_priv_xforms; i++) {
 544                         rte_compressdev_private_xform_free(0, priv_xforms[i]);
 545                         priv_xforms[i] = NULL;
 546                 }
 547                 num_priv_xforms = 0;
 548         }
 549
 550         for (i = 0; i < num_bufs; i++) {
 551                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 552                 uint16_t xform_idx = priv_data->orig_idx % num_xforms;
 553                 const struct rte_comp_compress_xform *compress_xform =
 554                                 &compress_xforms[xform_idx]->compress;
 555                 enum rte_comp_huffman huffman_type =
 556                         compress_xform->deflate.huffman;
 557                 RTE_LOG(DEBUG, USER1, "Buffer %u compressed from %u to %u bytes "
 558                         "(level = %d, huffman = %s)\n",
 559                         buf_idx[priv_data->orig_idx],
 560                         ops_processed[i]->consumed, ops_processed[i]->produced,
 561                         compress_xform->level,
 562                         huffman_type_strings[huffman_type]);
 563                 RTE_LOG(DEBUG, USER1, "Compression ratio = %.2f",
 564                         (float)ops_processed[i]->produced /
 565                         ops_processed[i]->consumed * 100);
 566                 ops[i] = NULL;
 567         }
 568
 569         /*
 570          * Check operation status and free source mbufs (destination mbuf and
 571          * compress operation information is needed for the decompression stage)
 572          */
 573         for (i = 0; i < num_bufs; i++) {
 574                 if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 575                         RTE_LOG(ERR, USER1,
 576                                 "Some operations were not successful\n");
 577                         goto exit;
 578                 }
 579                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 580                 rte_pktmbuf_free(uncomp_bufs[priv_data->orig_idx]);
 581                 uncomp_bufs[priv_data->orig_idx] = NULL;
 582         }
 583
 584         /* Allocate buffers for decompressed data */
 585         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, uncomp_bufs, num_bufs);
 586         if (ret < 0) {
 587                 RTE_LOG(ERR, USER1,
 588                         "Destination mbufs could not be allocated "
 589                         "from the mempool\n");
 590                 goto exit;
 591         }
 592
 593         for (i = 0; i < num_bufs; i++) {
 594                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 595                 rte_pktmbuf_append(uncomp_bufs[i],
 596                                 strlen(test_bufs[priv_data->orig_idx]) + 1);
 597         }
 598
 599         /* Build the decompression operations */
 600         ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
 601         if (ret < 0) {
 602                 RTE_LOG(ERR, USER1,
 603                         "Decompress operations could not be allocated "
 604                         "from the mempool\n");
 605                 goto exit;
 606         }
 607
 608         /* Source buffer is the compressed data from the previous operations */
 609         for (i = 0; i < num_bufs; i++) {
 610                 ops[i]->m_src = ops_processed[i]->m_dst;
 611                 ops[i]->m_dst = uncomp_bufs[i];
 612                 ops[i]->src.offset = 0;
 613                 /*
 614                  * Set the length of the compressed data to the
 615                  * number of bytes that were produced in the previous stage
 616                  */
 617                 ops[i]->src.length = ops_processed[i]->produced;
 618                 ops[i]->dst.offset = 0;
 619                 if (state == RTE_COMP_OP_STATELESS) {
 620                         ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
 621                 } else {
 622                         RTE_LOG(ERR, USER1,
 623                                 "Stateful operations are not supported "
 624                                 "in these tests yet\n");
 625                         goto exit;
 626                 }
 627                 ops[i]->input_chksum = 0;
 628                 /*
 629                  * Copy private data from previous operations,
 630                  * to keep the pointer to the original buffer
 631                  */
 632                 memcpy(ops[i] + 1, ops_processed[i] + 1,
 633                                 sizeof(struct priv_op_data));
 634         }
 635
 636         /*
 637          * Free the previous compress operations,
 638          * as it is not needed anymore
 639          */
 640         for (i = 0; i < num_bufs; i++) {
 641                 rte_comp_op_free(ops_processed[i]);
 642                 ops_processed[i] = NULL;
 643         }
 644
 645         /* Decompress data (either with Zlib API or compressdev API */
 646         if (zlib_dir == ZLIB_DECOMPRESS || zlib_dir == ZLIB_ALL) {
 647                 for (i = 0; i < num_bufs; i++) {
 648                         priv_data = (struct priv_op_data *)(ops[i] + 1);
 649                         uint16_t xform_idx = priv_data->orig_idx % num_xforms;
 650                         const struct rte_comp_xform *decompress_xform =
 651                                 decompress_xforms[xform_idx];
 652
 653                         ret = decompress_zlib(ops[i], decompress_xform);
 654                         if (ret < 0)
 655                                 goto exit;
 656
 657                         ops_processed[i] = ops[i];
 658                 }
 659         } else {
 660                 /* Create decompress private xform data */
 661                 for (i = 0; i < num_xforms; i++) {
 662                         ret = rte_compressdev_private_xform_create(0,
 663                                 (const struct rte_comp_xform *)decompress_xforms[i],
 664                                 &priv_xforms[i]);
 665                         if (ret < 0) {
 666                                 RTE_LOG(ERR, USER1,
 667                                         "Decompression private xform "
 668                                         "could not be created\n");
 669                                 goto exit;
 670                         }
 671                         num_priv_xforms++;
 672                 }
 673
 674                 if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 675                         /* Attach shareable private xform data to ops */
 676                         for (i = 0; i < num_bufs; i++) {
 677                                 priv_data = (struct priv_op_data *)(ops[i] + 1);
 678                                 uint16_t xform_idx = priv_data->orig_idx %
 679                                                                 num_xforms;
 680                                 ops[i]->private_xform = priv_xforms[xform_idx];
 681                         }
 682                 } else {
 683                         /* Create rest of the private xforms for the other ops */
 684                         for (i = num_xforms; i < num_bufs; i++) {
 685                                 ret = rte_compressdev_private_xform_create(0,
 686                                         decompress_xforms[i % num_xforms],
 687                                         &priv_xforms[i]);
 688                                 if (ret < 0) {
 689                                         RTE_LOG(ERR, USER1,
 690                                                 "Decompression private xform "
 691                                                 "could not be created\n");
 692                                         goto exit;
 693                                 }
 694                                 num_priv_xforms++;
 695                         }
 696
 697                         /* Attach non shareable private xform data to ops */
 698                         for (i = 0; i < num_bufs; i++) {
 699                                 priv_data = (struct priv_op_data *) (ops[i] + 1);
 700                                 uint16_t xform_idx = priv_data->orig_idx;
 701                                 ops[i]->private_xform = priv_xforms[xform_idx];
 702                         }
 703                 }
 704
 705                 /* Enqueue and dequeue all operations */
 706                 num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
 707                 if (num_enqd < num_bufs) {
 708                         RTE_LOG(ERR, USER1,
 709                                 "The operations could not be enqueued\n");
 710                         goto exit;
 711                 }
 712
 713                 num_total_deqd = 0;
 714                 do {
 715                         /*
 716                          * If retrying a dequeue call, wait for 10 ms to allow
 717                          * enough time to the driver to process the operations
 718                          */
 719                         if (deqd_retries != 0) {
 720                                 /*
 721                                  * Avoid infinite loop if not all the
 722                                  * operations get out of the device
 723                                  */
 724                                 if (deqd_retries == MAX_DEQD_RETRIES) {
 725                                         RTE_LOG(ERR, USER1,
 726                                                 "Not all operations could be "
 727                                                 "dequeued\n");
 728                                         goto exit;
 729                                 }
 730                                 usleep(DEQUEUE_WAIT_TIME);
 731                         }
 732                         num_deqd = rte_compressdev_dequeue_burst(0, 0,
 733                                         &ops_processed[num_total_deqd], num_bufs);
 734                         num_total_deqd += num_deqd;
 735                         deqd_retries++;
 736                 } while (num_total_deqd < num_enqd);
 737
 738                 deqd_retries = 0;
 739         }
 740
 741         for (i = 0; i < num_bufs; i++) {
 742                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 743                 RTE_LOG(DEBUG, USER1, "Buffer %u decompressed from %u to %u bytes\n",
 744                         buf_idx[priv_data->orig_idx],
 745                         ops_processed[i]->consumed, ops_processed[i]->produced);
 746                 ops[i] = NULL;
 747         }
 748
 749         /*
 750          * Check operation status and free source mbuf (destination mbuf and
 751          * compress operation information is still needed)
 752          */
 753         for (i = 0; i < num_bufs; i++) {
 754                 if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 755                         RTE_LOG(ERR, USER1,
 756                                 "Some operations were not successful\n");
 757                         goto exit;
 758                 }
 759                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 760                 rte_pktmbuf_free(comp_bufs[priv_data->orig_idx]);
 761                 comp_bufs[priv_data->orig_idx] = NULL;
 762         }
 763
 764         /*
 765          * Compare the original stream with the decompressed stream
 766          * (in size and the data)
 767          */
 768         for (i = 0; i < num_bufs; i++) {
 769                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 770                 const char *buf1 = test_bufs[priv_data->orig_idx];
 771                 const char *buf2 = rte_pktmbuf_mtod(ops_processed[i]->m_dst,
 772                                 const char *);
 773
 774                 if (compare_buffers(buf1, strlen(buf1) + 1,
 775                                 buf2, ops_processed[i]->produced) < 0)
 776                         goto exit;
 777         }
 778
 779         ret_status = 0;
 780
 781 exit:
 782         /* Free resources */
 783         for (i = 0; i < num_bufs; i++) {
 784                 rte_pktmbuf_free(uncomp_bufs[i]);
 785                 rte_pktmbuf_free(comp_bufs[i]);
 786                 rte_comp_op_free(ops[i]);
 787                 rte_comp_op_free(ops_processed[i]);
 788         }
 789         for (i = 0; i < num_priv_xforms; i++) {
 790                 if (priv_xforms[i] != NULL)
 791                         rte_compressdev_private_xform_free(0, priv_xforms[i]);
 792         }
 793
 794         return ret_status;
 795 }
 796
 797 static int
 798 test_compressdev_deflate_stateless_fixed(void)
 799 {
 800         struct comp_testsuite_params *ts_params = &testsuite_params;
 801         const char *test_buffer;
 802         uint16_t i;
 803         int ret;
 804         struct rte_comp_xform *compress_xform =
 805                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 806
 807         if (compress_xform == NULL) {
 808                 RTE_LOG(ERR, USER1,
 809                         "Compress xform could not be created\n");
 810                 ret = TEST_FAILED;
 811                 goto exit;
 812         }
 813
 814         memcpy(compress_xform, ts_params->def_comp_xform,
 815                         sizeof(struct rte_comp_xform));
 816         compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
 817
 818         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
 819                 test_buffer = compress_test_bufs[i];
 820
 821                 /* Compress with compressdev, decompress with Zlib */
 822                 if (test_deflate_comp_decomp(&test_buffer, 1,
 823                                 &i,
 824                                 &compress_xform,
 825                                 &ts_params->def_decomp_xform,
 826                                 1,
 827                                 RTE_COMP_OP_STATELESS,
 828                                 ZLIB_DECOMPRESS) < 0) {
 829                         ret = TEST_FAILED;
 830                         goto exit;
 831                 }
 832
 833                 /* Compress with Zlib, decompress with compressdev */
 834                 if (test_deflate_comp_decomp(&test_buffer, 1,
 835                                 &i,
 836                                 &compress_xform,
 837                                 &ts_params->def_decomp_xform,
 838                                 1,
 839                                 RTE_COMP_OP_STATELESS,
 840                                 ZLIB_COMPRESS) < 0) {
 841                         ret = TEST_FAILED;
 842                         goto exit;
 843                 }
 844         }
 845
 846         ret = TEST_SUCCESS;
 847
 848 exit:
 849         rte_free(compress_xform);
 850         return ret;
 851 }
 852
 853 static int
 854 test_compressdev_deflate_stateless_dynamic(void)
 855 {
 856         struct comp_testsuite_params *ts_params = &testsuite_params;
 857         const char *test_buffer;
 858         uint16_t i;
 859         int ret;
 860         struct rte_comp_xform *compress_xform =
 861                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 862
 863         if (compress_xform == NULL) {
 864                 RTE_LOG(ERR, USER1,
 865                         "Compress xform could not be created\n");
 866                 ret = TEST_FAILED;
 867                 goto exit;
 868         }
 869
 870         memcpy(compress_xform, ts_params->def_comp_xform,
 871                         sizeof(struct rte_comp_xform));
 872         compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_DYNAMIC;
 873
 874         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
 875                 test_buffer = compress_test_bufs[i];
 876
 877                 /* Compress with compressdev, decompress with Zlib */
 878                 if (test_deflate_comp_decomp(&test_buffer, 1,
 879                                 &i,
 880                                 &compress_xform,
 881                                 &ts_params->def_decomp_xform,
 882                                 1,
 883                                 RTE_COMP_OP_STATELESS,
 884                                 ZLIB_DECOMPRESS) < 0) {
 885                         ret = TEST_FAILED;
 886                         goto exit;
 887                 }
 888
 889                 /* Compress with Zlib, decompress with compressdev */
 890                 if (test_deflate_comp_decomp(&test_buffer, 1,
 891                                 &i,
 892                                 &compress_xform,
 893                                 &ts_params->def_decomp_xform,
 894                                 1,
 895                                 RTE_COMP_OP_STATELESS,
 896                                 ZLIB_COMPRESS) < 0) {
 897                         ret = TEST_FAILED;
 898                         goto exit;
 899                 }
 900         }
 901
 902         ret = TEST_SUCCESS;
 903
 904 exit:
 905         rte_free(compress_xform);
 906         return ret;
 907 }
 908
 909 static int
 910 test_compressdev_deflate_stateless_multi_op(void)
 911 {
 912         struct comp_testsuite_params *ts_params = &testsuite_params;
 913         uint16_t num_bufs = RTE_DIM(compress_test_bufs);
 914         uint16_t buf_idx[num_bufs];
 915         uint16_t i;
 916
 917         for (i = 0; i < num_bufs; i++)
 918                 buf_idx[i] = i;
 919
 920         /* Compress with compressdev, decompress with Zlib */
 921         if (test_deflate_comp_decomp(compress_test_bufs, num_bufs,
 922                         buf_idx,
 923                         &ts_params->def_comp_xform,
 924                         &ts_params->def_decomp_xform,
 925                         1,
 926                         RTE_COMP_OP_STATELESS,
 927                         ZLIB_DECOMPRESS) < 0)
 928                 return TEST_FAILED;
 929
 930         /* Compress with Zlib, decompress with compressdev */
 931         if (test_deflate_comp_decomp(compress_test_bufs, num_bufs,
 932                         buf_idx,
 933                         &ts_params->def_comp_xform,
 934                         &ts_params->def_decomp_xform,
 935                         1,
 936                         RTE_COMP_OP_STATELESS,
 937                         ZLIB_COMPRESS) < 0)
 938                 return TEST_FAILED;
 939
 940         return TEST_SUCCESS;
 941 }
 942
 943 static int
 944 test_compressdev_deflate_stateless_multi_level(void)
 945 {
 946         struct comp_testsuite_params *ts_params = &testsuite_params;
 947         const char *test_buffer;
 948         unsigned int level;
 949         uint16_t i;
 950         int ret;
 951         struct rte_comp_xform *compress_xform =
 952                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 953
 954         if (compress_xform == NULL) {
 955                 RTE_LOG(ERR, USER1,
 956                         "Compress xform could not be created\n");
 957                 ret = TEST_FAILED;
 958                 goto exit;
 959         }
 960
 961         memcpy(compress_xform, ts_params->def_comp_xform,
 962                         sizeof(struct rte_comp_xform));
 963
 964         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
 965                 test_buffer = compress_test_bufs[i];
 966                 for (level = RTE_COMP_LEVEL_MIN; level <= RTE_COMP_LEVEL_MAX;
 967                                 level++) {
 968                         compress_xform->compress.level = level;
 969                         /* Compress with compressdev, decompress with Zlib */
 970                         if (test_deflate_comp_decomp(&test_buffer, 1,
 971                                         &i,
 972                                         &compress_xform,
 973                                         &ts_params->def_decomp_xform,
 974                                         1,
 975                                         RTE_COMP_OP_STATELESS,
 976                                         ZLIB_DECOMPRESS) < 0) {
 977                                 ret = TEST_FAILED;
 978                                 goto exit;
 979                         }
 980                 }
 981         }
 982
 983         ret = TEST_SUCCESS;
 984
 985 exit:
 986         rte_free(compress_xform);
 987         return ret;
 988 }
 989
 990 #define NUM_XFORMS 3
 991 static int
 992 test_compressdev_deflate_stateless_multi_xform(void)
 993 {
 994         struct comp_testsuite_params *ts_params = &testsuite_params;
 995         uint16_t num_bufs = NUM_XFORMS;
 996         struct rte_comp_xform *compress_xforms[NUM_XFORMS] = {NULL};
 997         struct rte_comp_xform *decompress_xforms[NUM_XFORMS] = {NULL};
 998         const char *test_buffers[NUM_XFORMS];
 999         uint16_t i;
1000         unsigned int level = RTE_COMP_LEVEL_MIN;
1001         uint16_t buf_idx[num_bufs];
1002
1003         int ret;
1004
1005         /* Create multiple xforms with various levels */
1006         for (i = 0; i < NUM_XFORMS; i++) {
1007                 compress_xforms[i] = rte_malloc(NULL,
1008                                 sizeof(struct rte_comp_xform), 0);
1009                 if (compress_xforms[i] == NULL) {
1010                         RTE_LOG(ERR, USER1,
1011                                 "Compress xform could not be created\n");
1012                         ret = TEST_FAILED;
1013                         goto exit;
1014                 }
1015
1016                 memcpy(compress_xforms[i], ts_params->def_comp_xform,
1017                                 sizeof(struct rte_comp_xform));
1018                 compress_xforms[i]->compress.level = level;
1019                 level++;
1020
1021                 decompress_xforms[i] = rte_malloc(NULL,
1022                                 sizeof(struct rte_comp_xform), 0);
1023                 if (decompress_xforms[i] == NULL) {
1024                         RTE_LOG(ERR, USER1,
1025                                 "Decompress xform could not be created\n");
1026                         ret = TEST_FAILED;
1027                         goto exit;
1028                 }
1029
1030                 memcpy(decompress_xforms[i], ts_params->def_decomp_xform,
1031                                 sizeof(struct rte_comp_xform));
1032         }
1033
1034         for (i = 0; i < NUM_XFORMS; i++) {
1035                 buf_idx[i] = 0;
1036                 /* Use the same buffer in all sessions */
1037                 test_buffers[i] = compress_test_bufs[0];
1038         }
1039         /* Compress with compressdev, decompress with Zlib */
1040         if (test_deflate_comp_decomp(test_buffers, num_bufs,
1041                         buf_idx,
1042                         compress_xforms,
1043                         decompress_xforms,
1044                         NUM_XFORMS,
1045                         RTE_COMP_OP_STATELESS,
1046                         ZLIB_DECOMPRESS) < 0) {
1047                 ret = TEST_FAILED;
1048                 goto exit;
1049         }
1050
1051         ret = TEST_SUCCESS;
1052 exit:
1053         for (i = 0; i < NUM_XFORMS; i++) {
1054                 rte_free(compress_xforms[i]);
1055                 rte_free(decompress_xforms[i]);
1056         }
1057
1058         return ret;
1059 }
1060
1061 static struct unit_test_suite compressdev_testsuite  = {
1062         .suite_name = "compressdev unit test suite",
1063         .setup = testsuite_setup,
1064         .teardown = testsuite_teardown,
1065         .unit_test_cases = {
1066                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1067                         test_compressdev_deflate_stateless_fixed),
1068                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1069                         test_compressdev_deflate_stateless_dynamic),
1070                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1071                         test_compressdev_deflate_stateless_multi_op),
1072                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1073                         test_compressdev_deflate_stateless_multi_level),
1074                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1075                         test_compressdev_deflate_stateless_multi_xform),
1076                 TEST_CASES_END() /**< NULL terminate unit test array */
1077         }
1078 };
1079
1080 static int
1081 test_compressdev(void)
1082 {
1083         return unit_test_suite_runner(&compressdev_testsuite);
1084 }
1085
1086 REGISTER_TEST_COMMAND(compressdev_autotest, test_compressdev);