test/compress: add multi xform test
[dpdk.git] / test / test / test_compressdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 Intel Corporation
3  */
4 #include <string.h>
5 #include <zlib.h>
6 #include <math.h>
7
8 #include <rte_cycles.h>
9 #include <rte_malloc.h>
10 #include <rte_mempool.h>
11 #include <rte_mbuf.h>
12 #include <rte_compressdev.h>
13
14 #include "test_compressdev_test_buffer.h"
15 #include "test.h"
16
17 #define DEFAULT_WINDOW_SIZE 15
18 #define DEFAULT_MEM_LEVEL 8
19 #define MAX_DEQD_RETRIES 10
20 #define DEQUEUE_WAIT_TIME 10000
21
22 /*
23  * 30% extra size for compressed data compared to original data,
24  * in case data size cannot be reduced and it is actually bigger
25  * due to the compress block headers
26  */
27 #define COMPRESS_BUF_SIZE_RATIO 1.3
28 #define NUM_MBUFS 16
29 #define NUM_OPS 16
30 #define NUM_MAX_XFORMS 16
31 #define NUM_MAX_INFLIGHT_OPS 128
32 #define CACHE_SIZE 0
33
34 const char *
35 huffman_type_strings[] = {
36         [RTE_COMP_HUFFMAN_DEFAULT]      = "PMD default",
37         [RTE_COMP_HUFFMAN_FIXED]        = "Fixed",
38         [RTE_COMP_HUFFMAN_DYNAMIC]      = "Dynamic"
39 };
40
41 enum zlib_direction {
42         ZLIB_NONE,
43         ZLIB_COMPRESS,
44         ZLIB_DECOMPRESS,
45         ZLIB_ALL
46 };
47
48 struct priv_op_data {
49         uint16_t orig_idx;
50 };
51
52 struct comp_testsuite_params {
53         struct rte_mempool *mbuf_pool;
54         struct rte_mempool *op_pool;
55         struct rte_comp_xform *def_comp_xform;
56         struct rte_comp_xform *def_decomp_xform;
57 };
58
59 static struct comp_testsuite_params testsuite_params = { 0 };
60
61 static void
62 testsuite_teardown(void)
63 {
64         struct comp_testsuite_params *ts_params = &testsuite_params;
65
66         rte_mempool_free(ts_params->mbuf_pool);
67         rte_mempool_free(ts_params->op_pool);
68         rte_free(ts_params->def_comp_xform);
69         rte_free(ts_params->def_decomp_xform);
70 }
71
72 static int
73 testsuite_setup(void)
74 {
75         struct comp_testsuite_params *ts_params = &testsuite_params;
76         unsigned int i;
77
78         if (rte_compressdev_count() == 0) {
79                 RTE_LOG(ERR, USER1, "Need at least one compress device\n");
80                 return TEST_FAILED;
81         }
82
83         uint32_t max_buf_size = 0;
84         for (i = 0; i < RTE_DIM(compress_test_bufs); i++)
85                 max_buf_size = RTE_MAX(max_buf_size,
86                                 strlen(compress_test_bufs[i]) + 1);
87
88         max_buf_size *= COMPRESS_BUF_SIZE_RATIO;
89         /*
90          * Buffers to be used in compression and decompression.
91          * Since decompressed data might be larger than
92          * compressed data (due to block header),
93          * buffers should be big enough for both cases.
94          */
95         ts_params->mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool",
96                         NUM_MBUFS,
97                         CACHE_SIZE, 0,
98                         max_buf_size + RTE_PKTMBUF_HEADROOM,
99                         rte_socket_id());
100         if (ts_params->mbuf_pool == NULL) {
101                 RTE_LOG(ERR, USER1, "Large mbuf pool could not be created\n");
102                 return TEST_FAILED;
103         }
104
105         ts_params->op_pool = rte_comp_op_pool_create("op_pool", NUM_OPS,
106                                 0, sizeof(struct priv_op_data),
107                                 rte_socket_id());
108         if (ts_params->op_pool == NULL) {
109                 RTE_LOG(ERR, USER1, "Operation pool could not be created\n");
110                 goto exit;
111         }
112
113         ts_params->def_comp_xform =
114                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
115         if (ts_params->def_comp_xform == NULL) {
116                 RTE_LOG(ERR, USER1,
117                         "Default compress xform could not be created\n");
118                 goto exit;
119         }
120         ts_params->def_decomp_xform =
121                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
122         if (ts_params->def_decomp_xform == NULL) {
123                 RTE_LOG(ERR, USER1,
124                         "Default decompress xform could not be created\n");
125                 goto exit;
126         }
127
128         /* Initializes default values for compress/decompress xforms */
129         ts_params->def_comp_xform->type = RTE_COMP_COMPRESS;
130         ts_params->def_comp_xform->compress.algo = RTE_COMP_ALGO_DEFLATE,
131         ts_params->def_comp_xform->compress.deflate.huffman =
132                                                 RTE_COMP_HUFFMAN_DEFAULT;
133         ts_params->def_comp_xform->compress.level = RTE_COMP_LEVEL_PMD_DEFAULT;
134         ts_params->def_comp_xform->compress.chksum = RTE_COMP_CHECKSUM_NONE;
135         ts_params->def_comp_xform->compress.window_size = DEFAULT_WINDOW_SIZE;
136
137         ts_params->def_decomp_xform->type = RTE_COMP_DECOMPRESS;
138         ts_params->def_decomp_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE,
139         ts_params->def_decomp_xform->decompress.chksum = RTE_COMP_CHECKSUM_NONE;
140         ts_params->def_decomp_xform->decompress.window_size = DEFAULT_WINDOW_SIZE;
141
142         return TEST_SUCCESS;
143
144 exit:
145         testsuite_teardown();
146
147         return TEST_FAILED;
148 }
149
150 static int
151 generic_ut_setup(void)
152 {
153         /* Configure compressdev (one device, one queue pair) */
154         struct rte_compressdev_config config = {
155                 .socket_id = rte_socket_id(),
156                 .nb_queue_pairs = 1,
157                 .max_nb_priv_xforms = NUM_MAX_XFORMS,
158                 .max_nb_streams = 0
159         };
160
161         if (rte_compressdev_configure(0, &config) < 0) {
162                 RTE_LOG(ERR, USER1, "Device configuration failed\n");
163                 return -1;
164         }
165
166         if (rte_compressdev_queue_pair_setup(0, 0, NUM_MAX_INFLIGHT_OPS,
167                         rte_socket_id()) < 0) {
168                 RTE_LOG(ERR, USER1, "Queue pair setup failed\n");
169                 return -1;
170         }
171
172         if (rte_compressdev_start(0) < 0) {
173                 RTE_LOG(ERR, USER1, "Device could not be started\n");
174                 return -1;
175         }
176
177         return 0;
178 }
179
180 static void
181 generic_ut_teardown(void)
182 {
183         rte_compressdev_stop(0);
184         if (rte_compressdev_close(0) < 0)
185                 RTE_LOG(ERR, USER1, "Device could not be closed\n");
186 }
187
188 static int
189 compare_buffers(const char *buffer1, uint32_t buffer1_len,
190                 const char *buffer2, uint32_t buffer2_len)
191 {
192         if (buffer1_len != buffer2_len) {
193                 RTE_LOG(ERR, USER1, "Buffer lengths are different\n");
194                 return -1;
195         }
196
197         if (memcmp(buffer1, buffer2, buffer1_len) != 0) {
198                 RTE_LOG(ERR, USER1, "Buffers are different\n");
199                 return -1;
200         }
201
202         return 0;
203 }
204
205 /*
206  * Maps compressdev and Zlib flush flags
207  */
208 static int
209 map_zlib_flush_flag(enum rte_comp_flush_flag flag)
210 {
211         switch (flag) {
212         case RTE_COMP_FLUSH_NONE:
213                 return Z_NO_FLUSH;
214         case RTE_COMP_FLUSH_SYNC:
215                 return Z_SYNC_FLUSH;
216         case RTE_COMP_FLUSH_FULL:
217                 return Z_FULL_FLUSH;
218         case RTE_COMP_FLUSH_FINAL:
219                 return Z_FINISH;
220         /*
221          * There should be only the values above,
222          * so this should never happen
223          */
224         default:
225                 return -1;
226         }
227 }
228
229 static int
230 compress_zlib(struct rte_comp_op *op,
231                 const struct rte_comp_xform *xform, int mem_level)
232 {
233         z_stream stream;
234         int zlib_flush;
235         int strategy, window_bits, comp_level;
236         int ret = -1;
237
238         /* initialize zlib stream */
239         stream.zalloc = Z_NULL;
240         stream.zfree = Z_NULL;
241         stream.opaque = Z_NULL;
242
243         if (xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
244                 strategy = Z_FIXED;
245         else
246                 strategy = Z_DEFAULT_STRATEGY;
247
248         /*
249          * Window bits is the base two logarithm of the window size (in bytes).
250          * When doing raw DEFLATE, this number will be negative.
251          */
252         window_bits = -(xform->compress.window_size);
253
254         comp_level = xform->compress.level;
255
256         if (comp_level != RTE_COMP_LEVEL_NONE)
257                 ret = deflateInit2(&stream, comp_level, Z_DEFLATED,
258                         window_bits, mem_level, strategy);
259         else
260                 ret = deflateInit(&stream, Z_NO_COMPRESSION);
261
262         if (ret != Z_OK) {
263                 printf("Zlib deflate could not be initialized\n");
264                 goto exit;
265         }
266
267         /* Assuming stateless operation */
268         stream.avail_in = op->src.length;
269         stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
270         stream.avail_out = op->m_dst->data_len;
271         stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
272
273         /* Stateless operation, all buffer will be compressed in one go */
274         zlib_flush = map_zlib_flush_flag(op->flush_flag);
275         ret = deflate(&stream, zlib_flush);
276
277         if (stream.avail_in != 0) {
278                 RTE_LOG(ERR, USER1, "Buffer could not be read entirely\n");
279                 goto exit;
280         }
281
282         if (ret != Z_STREAM_END)
283                 goto exit;
284
285         op->consumed = op->src.length - stream.avail_in;
286         op->produced = op->m_dst->data_len - stream.avail_out;
287         op->status = RTE_COMP_OP_STATUS_SUCCESS;
288
289         deflateReset(&stream);
290
291         ret = 0;
292 exit:
293         deflateEnd(&stream);
294
295         return ret;
296 }
297
298 static int
299 decompress_zlib(struct rte_comp_op *op,
300                 const struct rte_comp_xform *xform)
301 {
302         z_stream stream;
303         int window_bits;
304         int zlib_flush;
305         int ret = TEST_FAILED;
306
307         /* initialize zlib stream */
308         stream.zalloc = Z_NULL;
309         stream.zfree = Z_NULL;
310         stream.opaque = Z_NULL;
311
312         /*
313          * Window bits is the base two logarithm of the window size (in bytes).
314          * When doing raw DEFLATE, this number will be negative.
315          */
316         window_bits = -(xform->decompress.window_size);
317
318         ret = inflateInit2(&stream, window_bits);
319
320         if (ret != Z_OK) {
321                 printf("Zlib deflate could not be initialized\n");
322                 goto exit;
323         }
324
325         /* Assuming stateless operation */
326         stream.avail_in = op->src.length;
327         stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
328         stream.avail_out = op->m_dst->data_len;
329         stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
330
331         /* Stateless operation, all buffer will be compressed in one go */
332         zlib_flush = map_zlib_flush_flag(op->flush_flag);
333         ret = inflate(&stream, zlib_flush);
334
335         if (stream.avail_in != 0) {
336                 RTE_LOG(ERR, USER1, "Buffer could not be read entirely\n");
337                 goto exit;
338         }
339
340         if (ret != Z_STREAM_END)
341                 goto exit;
342
343         op->consumed = op->src.length - stream.avail_in;
344         op->produced = op->m_dst->data_len - stream.avail_out;
345         op->status = RTE_COMP_OP_STATUS_SUCCESS;
346
347         inflateReset(&stream);
348
349         ret = 0;
350 exit:
351         inflateEnd(&stream);
352
353         return ret;
354 }
355
356 /*
357  * Compresses and decompresses buffer with compressdev API and Zlib API
358  */
359 static int
360 test_deflate_comp_decomp(const char * const test_bufs[],
361                 unsigned int num_bufs,
362                 uint16_t buf_idx[],
363                 struct rte_comp_xform *compress_xforms[],
364                 struct rte_comp_xform *decompress_xforms[],
365                 unsigned int num_xforms,
366                 enum rte_comp_op_type state,
367                 enum zlib_direction zlib_dir)
368 {
369         struct comp_testsuite_params *ts_params = &testsuite_params;
370         int ret_status = -1;
371         int ret;
372         struct rte_mbuf *uncomp_bufs[num_bufs];
373         struct rte_mbuf *comp_bufs[num_bufs];
374         struct rte_comp_op *ops[num_bufs];
375         struct rte_comp_op *ops_processed[num_bufs];
376         void *priv_xforms[num_bufs];
377         uint16_t num_enqd, num_deqd, num_total_deqd;
378         uint16_t num_priv_xforms = 0;
379         unsigned int deqd_retries = 0;
380         struct priv_op_data *priv_data;
381         char *data_ptr;
382         unsigned int i;
383         const struct rte_compressdev_capabilities *capa =
384                 rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
385
386         /* Initialize all arrays to NULL */
387         memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
388         memset(comp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
389         memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
390         memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
391         memset(priv_xforms, 0, sizeof(void *) * num_bufs);
392
393         /* Prepare the source mbufs with the data */
394         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, uncomp_bufs, num_bufs);
395         if (ret < 0) {
396                 RTE_LOG(ERR, USER1,
397                         "Source mbufs could not be allocated "
398                         "from the mempool\n");
399                 goto exit;
400         }
401
402         for (i = 0; i < num_bufs; i++) {
403                 data_ptr = rte_pktmbuf_append(uncomp_bufs[i],
404                                         strlen(test_bufs[i]) + 1);
405                 snprintf(data_ptr, strlen(test_bufs[i]) + 1, "%s",
406                                         test_bufs[i]);
407         }
408
409         /* Prepare the destination mbufs */
410         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, comp_bufs, num_bufs);
411         if (ret < 0) {
412                 RTE_LOG(ERR, USER1,
413                         "Destination mbufs could not be allocated "
414                         "from the mempool\n");
415                 goto exit;
416         }
417
418         for (i = 0; i < num_bufs; i++)
419                 rte_pktmbuf_append(comp_bufs[i],
420                         strlen(test_bufs[i]) * COMPRESS_BUF_SIZE_RATIO);
421
422         /* Build the compression operations */
423         ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
424         if (ret < 0) {
425                 RTE_LOG(ERR, USER1,
426                         "Compress operations could not be allocated "
427                         "from the mempool\n");
428                 goto exit;
429         }
430
431         for (i = 0; i < num_bufs; i++) {
432                 ops[i]->m_src = uncomp_bufs[i];
433                 ops[i]->m_dst = comp_bufs[i];
434                 ops[i]->src.offset = 0;
435                 ops[i]->src.length = rte_pktmbuf_pkt_len(uncomp_bufs[i]);
436                 ops[i]->dst.offset = 0;
437                 if (state == RTE_COMP_OP_STATELESS) {
438                         ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
439                 } else {
440                         RTE_LOG(ERR, USER1,
441                                 "Stateful operations are not supported "
442                                 "in these tests yet\n");
443                         goto exit;
444                 }
445                 ops[i]->input_chksum = 0;
446                 /*
447                  * Store original operation index in private data,
448                  * since ordering does not have to be maintained,
449                  * when dequeueing from compressdev, so a comparison
450                  * at the end of the test can be done.
451                  */
452                 priv_data = (struct priv_op_data *) (ops[i] + 1);
453                 priv_data->orig_idx = i;
454         }
455
456         /* Compress data (either with Zlib API or compressdev API */
457         if (zlib_dir == ZLIB_COMPRESS || zlib_dir == ZLIB_ALL) {
458                 for (i = 0; i < num_bufs; i++) {
459                         const struct rte_comp_xform *compress_xform =
460                                 compress_xforms[i % num_xforms];
461                         ret = compress_zlib(ops[i], compress_xform,
462                                         DEFAULT_MEM_LEVEL);
463                         if (ret < 0)
464                                 goto exit;
465
466                         ops_processed[i] = ops[i];
467                 }
468         } else {
469                 /* Create compress private xform data */
470                 for (i = 0; i < num_xforms; i++) {
471                         ret = rte_compressdev_private_xform_create(0,
472                                 (const struct rte_comp_xform *)compress_xforms[i],
473                                 &priv_xforms[i]);
474                         if (ret < 0) {
475                                 RTE_LOG(ERR, USER1,
476                                         "Compression private xform "
477                                         "could not be created\n");
478                                 goto exit;
479                         }
480                         num_priv_xforms++;
481                 }
482
483                 if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
484                         /* Attach shareable private xform data to ops */
485                         for (i = 0; i < num_bufs; i++)
486                                 ops[i]->private_xform = priv_xforms[i % num_xforms];
487                 } else {
488                         /* Create rest of the private xforms for the other ops */
489                         for (i = num_xforms; i < num_bufs; i++) {
490                                 ret = rte_compressdev_private_xform_create(0,
491                                         compress_xforms[i % num_xforms],
492                                         &priv_xforms[i]);
493                                 if (ret < 0) {
494                                         RTE_LOG(ERR, USER1,
495                                                 "Compression private xform "
496                                                 "could not be created\n");
497                                         goto exit;
498                                 }
499                                 num_priv_xforms++;
500                         }
501
502                         /* Attach non shareable private xform data to ops */
503                         for (i = 0; i < num_bufs; i++)
504                                 ops[i]->private_xform = priv_xforms[i];
505                 }
506
507                 /* Enqueue and dequeue all operations */
508                 num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
509                 if (num_enqd < num_bufs) {
510                         RTE_LOG(ERR, USER1,
511                                 "The operations could not be enqueued\n");
512                         goto exit;
513                 }
514
515                 num_total_deqd = 0;
516                 do {
517                         /*
518                          * If retrying a dequeue call, wait for 10 ms to allow
519                          * enough time to the driver to process the operations
520                          */
521                         if (deqd_retries != 0) {
522                                 /*
523                                  * Avoid infinite loop if not all the
524                                  * operations get out of the device
525                                  */
526                                 if (deqd_retries == MAX_DEQD_RETRIES) {
527                                         RTE_LOG(ERR, USER1,
528                                                 "Not all operations could be "
529                                                 "dequeued\n");
530                                         goto exit;
531                                 }
532                                 usleep(DEQUEUE_WAIT_TIME);
533                         }
534                         num_deqd = rte_compressdev_dequeue_burst(0, 0,
535                                         &ops_processed[num_total_deqd], num_bufs);
536                         num_total_deqd += num_deqd;
537                         deqd_retries++;
538                 } while (num_total_deqd < num_enqd);
539
540                 deqd_retries = 0;
541
542                 /* Free compress private xforms */
543                 for (i = 0; i < num_priv_xforms; i++) {
544                         rte_compressdev_private_xform_free(0, priv_xforms[i]);
545                         priv_xforms[i] = NULL;
546                 }
547                 num_priv_xforms = 0;
548         }
549
550         for (i = 0; i < num_bufs; i++) {
551                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
552                 uint16_t xform_idx = priv_data->orig_idx % num_xforms;
553                 const struct rte_comp_compress_xform *compress_xform =
554                                 &compress_xforms[xform_idx]->compress;
555                 enum rte_comp_huffman huffman_type =
556                         compress_xform->deflate.huffman;
557                 RTE_LOG(DEBUG, USER1, "Buffer %u compressed from %u to %u bytes "
558                         "(level = %d, huffman = %s)\n",
559                         buf_idx[priv_data->orig_idx],
560                         ops_processed[i]->consumed, ops_processed[i]->produced,
561                         compress_xform->level,
562                         huffman_type_strings[huffman_type]);
563                 RTE_LOG(DEBUG, USER1, "Compression ratio = %.2f",
564                         (float)ops_processed[i]->produced /
565                         ops_processed[i]->consumed * 100);
566                 ops[i] = NULL;
567         }
568
569         /*
570          * Check operation status and free source mbufs (destination mbuf and
571          * compress operation information is needed for the decompression stage)
572          */
573         for (i = 0; i < num_bufs; i++) {
574                 if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
575                         RTE_LOG(ERR, USER1,
576                                 "Some operations were not successful\n");
577                         goto exit;
578                 }
579                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
580                 rte_pktmbuf_free(uncomp_bufs[priv_data->orig_idx]);
581                 uncomp_bufs[priv_data->orig_idx] = NULL;
582         }
583
584         /* Allocate buffers for decompressed data */
585         ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, uncomp_bufs, num_bufs);
586         if (ret < 0) {
587                 RTE_LOG(ERR, USER1,
588                         "Destination mbufs could not be allocated "
589                         "from the mempool\n");
590                 goto exit;
591         }
592
593         for (i = 0; i < num_bufs; i++) {
594                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
595                 rte_pktmbuf_append(uncomp_bufs[i],
596                                 strlen(test_bufs[priv_data->orig_idx]) + 1);
597         }
598
599         /* Build the decompression operations */
600         ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
601         if (ret < 0) {
602                 RTE_LOG(ERR, USER1,
603                         "Decompress operations could not be allocated "
604                         "from the mempool\n");
605                 goto exit;
606         }
607
608         /* Source buffer is the compressed data from the previous operations */
609         for (i = 0; i < num_bufs; i++) {
610                 ops[i]->m_src = ops_processed[i]->m_dst;
611                 ops[i]->m_dst = uncomp_bufs[i];
612                 ops[i]->src.offset = 0;
613                 /*
614                  * Set the length of the compressed data to the
615                  * number of bytes that were produced in the previous stage
616                  */
617                 ops[i]->src.length = ops_processed[i]->produced;
618                 ops[i]->dst.offset = 0;
619                 if (state == RTE_COMP_OP_STATELESS) {
620                         ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
621                 } else {
622                         RTE_LOG(ERR, USER1,
623                                 "Stateful operations are not supported "
624                                 "in these tests yet\n");
625                         goto exit;
626                 }
627                 ops[i]->input_chksum = 0;
628                 /*
629                  * Copy private data from previous operations,
630                  * to keep the pointer to the original buffer
631                  */
632                 memcpy(ops[i] + 1, ops_processed[i] + 1,
633                                 sizeof(struct priv_op_data));
634         }
635
636         /*
637          * Free the previous compress operations,
638          * as it is not needed anymore
639          */
640         for (i = 0; i < num_bufs; i++) {
641                 rte_comp_op_free(ops_processed[i]);
642                 ops_processed[i] = NULL;
643         }
644
645         /* Decompress data (either with Zlib API or compressdev API */
646         if (zlib_dir == ZLIB_DECOMPRESS || zlib_dir == ZLIB_ALL) {
647                 for (i = 0; i < num_bufs; i++) {
648                         priv_data = (struct priv_op_data *)(ops[i] + 1);
649                         uint16_t xform_idx = priv_data->orig_idx % num_xforms;
650                         const struct rte_comp_xform *decompress_xform =
651                                 decompress_xforms[xform_idx];
652
653                         ret = decompress_zlib(ops[i], decompress_xform);
654                         if (ret < 0)
655                                 goto exit;
656
657                         ops_processed[i] = ops[i];
658                 }
659         } else {
660                 /* Create decompress private xform data */
661                 for (i = 0; i < num_xforms; i++) {
662                         ret = rte_compressdev_private_xform_create(0,
663                                 (const struct rte_comp_xform *)decompress_xforms[i],
664                                 &priv_xforms[i]);
665                         if (ret < 0) {
666                                 RTE_LOG(ERR, USER1,
667                                         "Decompression private xform "
668                                         "could not be created\n");
669                                 goto exit;
670                         }
671                         num_priv_xforms++;
672                 }
673
674                 if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
675                         /* Attach shareable private xform data to ops */
676                         for (i = 0; i < num_bufs; i++) {
677                                 priv_data = (struct priv_op_data *)(ops[i] + 1);
678                                 uint16_t xform_idx = priv_data->orig_idx %
679                                                                 num_xforms;
680                                 ops[i]->private_xform = priv_xforms[xform_idx];
681                         }
682                 } else {
683                         /* Create rest of the private xforms for the other ops */
684                         for (i = num_xforms; i < num_bufs; i++) {
685                                 ret = rte_compressdev_private_xform_create(0,
686                                         decompress_xforms[i % num_xforms],
687                                         &priv_xforms[i]);
688                                 if (ret < 0) {
689                                         RTE_LOG(ERR, USER1,
690                                                 "Decompression private xform "
691                                                 "could not be created\n");
692                                         goto exit;
693                                 }
694                                 num_priv_xforms++;
695                         }
696
697                         /* Attach non shareable private xform data to ops */
698                         for (i = 0; i < num_bufs; i++) {
699                                 priv_data = (struct priv_op_data *) (ops[i] + 1);
700                                 uint16_t xform_idx = priv_data->orig_idx;
701                                 ops[i]->private_xform = priv_xforms[xform_idx];
702                         }
703                 }
704
705                 /* Enqueue and dequeue all operations */
706                 num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
707                 if (num_enqd < num_bufs) {
708                         RTE_LOG(ERR, USER1,
709                                 "The operations could not be enqueued\n");
710                         goto exit;
711                 }
712
713                 num_total_deqd = 0;
714                 do {
715                         /*
716                          * If retrying a dequeue call, wait for 10 ms to allow
717                          * enough time to the driver to process the operations
718                          */
719                         if (deqd_retries != 0) {
720                                 /*
721                                  * Avoid infinite loop if not all the
722                                  * operations get out of the device
723                                  */
724                                 if (deqd_retries == MAX_DEQD_RETRIES) {
725                                         RTE_LOG(ERR, USER1,
726                                                 "Not all operations could be "
727                                                 "dequeued\n");
728                                         goto exit;
729                                 }
730                                 usleep(DEQUEUE_WAIT_TIME);
731                         }
732                         num_deqd = rte_compressdev_dequeue_burst(0, 0,
733                                         &ops_processed[num_total_deqd], num_bufs);
734                         num_total_deqd += num_deqd;
735                         deqd_retries++;
736                 } while (num_total_deqd < num_enqd);
737
738                 deqd_retries = 0;
739         }
740
741         for (i = 0; i < num_bufs; i++) {
742                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
743                 RTE_LOG(DEBUG, USER1, "Buffer %u decompressed from %u to %u bytes\n",
744                         buf_idx[priv_data->orig_idx],
745                         ops_processed[i]->consumed, ops_processed[i]->produced);
746                 ops[i] = NULL;
747         }
748
749         /*
750          * Check operation status and free source mbuf (destination mbuf and
751          * compress operation information is still needed)
752          */
753         for (i = 0; i < num_bufs; i++) {
754                 if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
755                         RTE_LOG(ERR, USER1,
756                                 "Some operations were not successful\n");
757                         goto exit;
758                 }
759                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
760                 rte_pktmbuf_free(comp_bufs[priv_data->orig_idx]);
761                 comp_bufs[priv_data->orig_idx] = NULL;
762         }
763
764         /*
765          * Compare the original stream with the decompressed stream
766          * (in size and the data)
767          */
768         for (i = 0; i < num_bufs; i++) {
769                 priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
770                 const char *buf1 = test_bufs[priv_data->orig_idx];
771                 const char *buf2 = rte_pktmbuf_mtod(ops_processed[i]->m_dst,
772                                 const char *);
773
774                 if (compare_buffers(buf1, strlen(buf1) + 1,
775                                 buf2, ops_processed[i]->produced) < 0)
776                         goto exit;
777         }
778
779         ret_status = 0;
780
781 exit:
782         /* Free resources */
783         for (i = 0; i < num_bufs; i++) {
784                 rte_pktmbuf_free(uncomp_bufs[i]);
785                 rte_pktmbuf_free(comp_bufs[i]);
786                 rte_comp_op_free(ops[i]);
787                 rte_comp_op_free(ops_processed[i]);
788         }
789         for (i = 0; i < num_priv_xforms; i++) {
790                 if (priv_xforms[i] != NULL)
791                         rte_compressdev_private_xform_free(0, priv_xforms[i]);
792         }
793
794         return ret_status;
795 }
796
797 static int
798 test_compressdev_deflate_stateless_fixed(void)
799 {
800         struct comp_testsuite_params *ts_params = &testsuite_params;
801         const char *test_buffer;
802         uint16_t i;
803         int ret;
804         struct rte_comp_xform *compress_xform =
805                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
806
807         if (compress_xform == NULL) {
808                 RTE_LOG(ERR, USER1,
809                         "Compress xform could not be created\n");
810                 ret = TEST_FAILED;
811                 goto exit;
812         }
813
814         memcpy(compress_xform, ts_params->def_comp_xform,
815                         sizeof(struct rte_comp_xform));
816         compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
817
818         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
819                 test_buffer = compress_test_bufs[i];
820
821                 /* Compress with compressdev, decompress with Zlib */
822                 if (test_deflate_comp_decomp(&test_buffer, 1,
823                                 &i,
824                                 &compress_xform,
825                                 &ts_params->def_decomp_xform,
826                                 1,
827                                 RTE_COMP_OP_STATELESS,
828                                 ZLIB_DECOMPRESS) < 0) {
829                         ret = TEST_FAILED;
830                         goto exit;
831                 }
832
833                 /* Compress with Zlib, decompress with compressdev */
834                 if (test_deflate_comp_decomp(&test_buffer, 1,
835                                 &i,
836                                 &compress_xform,
837                                 &ts_params->def_decomp_xform,
838                                 1,
839                                 RTE_COMP_OP_STATELESS,
840                                 ZLIB_COMPRESS) < 0) {
841                         ret = TEST_FAILED;
842                         goto exit;
843                 }
844         }
845
846         ret = TEST_SUCCESS;
847
848 exit:
849         rte_free(compress_xform);
850         return ret;
851 }
852
853 static int
854 test_compressdev_deflate_stateless_dynamic(void)
855 {
856         struct comp_testsuite_params *ts_params = &testsuite_params;
857         const char *test_buffer;
858         uint16_t i;
859         int ret;
860         struct rte_comp_xform *compress_xform =
861                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
862
863         if (compress_xform == NULL) {
864                 RTE_LOG(ERR, USER1,
865                         "Compress xform could not be created\n");
866                 ret = TEST_FAILED;
867                 goto exit;
868         }
869
870         memcpy(compress_xform, ts_params->def_comp_xform,
871                         sizeof(struct rte_comp_xform));
872         compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_DYNAMIC;
873
874         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
875                 test_buffer = compress_test_bufs[i];
876
877                 /* Compress with compressdev, decompress with Zlib */
878                 if (test_deflate_comp_decomp(&test_buffer, 1,
879                                 &i,
880                                 &compress_xform,
881                                 &ts_params->def_decomp_xform,
882                                 1,
883                                 RTE_COMP_OP_STATELESS,
884                                 ZLIB_DECOMPRESS) < 0) {
885                         ret = TEST_FAILED;
886                         goto exit;
887                 }
888
889                 /* Compress with Zlib, decompress with compressdev */
890                 if (test_deflate_comp_decomp(&test_buffer, 1,
891                                 &i,
892                                 &compress_xform,
893                                 &ts_params->def_decomp_xform,
894                                 1,
895                                 RTE_COMP_OP_STATELESS,
896                                 ZLIB_COMPRESS) < 0) {
897                         ret = TEST_FAILED;
898                         goto exit;
899                 }
900         }
901
902         ret = TEST_SUCCESS;
903
904 exit:
905         rte_free(compress_xform);
906         return ret;
907 }
908
909 static int
910 test_compressdev_deflate_stateless_multi_op(void)
911 {
912         struct comp_testsuite_params *ts_params = &testsuite_params;
913         uint16_t num_bufs = RTE_DIM(compress_test_bufs);
914         uint16_t buf_idx[num_bufs];
915         uint16_t i;
916
917         for (i = 0; i < num_bufs; i++)
918                 buf_idx[i] = i;
919
920         /* Compress with compressdev, decompress with Zlib */
921         if (test_deflate_comp_decomp(compress_test_bufs, num_bufs,
922                         buf_idx,
923                         &ts_params->def_comp_xform,
924                         &ts_params->def_decomp_xform,
925                         1,
926                         RTE_COMP_OP_STATELESS,
927                         ZLIB_DECOMPRESS) < 0)
928                 return TEST_FAILED;
929
930         /* Compress with Zlib, decompress with compressdev */
931         if (test_deflate_comp_decomp(compress_test_bufs, num_bufs,
932                         buf_idx,
933                         &ts_params->def_comp_xform,
934                         &ts_params->def_decomp_xform,
935                         1,
936                         RTE_COMP_OP_STATELESS,
937                         ZLIB_COMPRESS) < 0)
938                 return TEST_FAILED;
939
940         return TEST_SUCCESS;
941 }
942
943 static int
944 test_compressdev_deflate_stateless_multi_level(void)
945 {
946         struct comp_testsuite_params *ts_params = &testsuite_params;
947         const char *test_buffer;
948         unsigned int level;
949         uint16_t i;
950         int ret;
951         struct rte_comp_xform *compress_xform =
952                         rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
953
954         if (compress_xform == NULL) {
955                 RTE_LOG(ERR, USER1,
956                         "Compress xform could not be created\n");
957                 ret = TEST_FAILED;
958                 goto exit;
959         }
960
961         memcpy(compress_xform, ts_params->def_comp_xform,
962                         sizeof(struct rte_comp_xform));
963
964         for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
965                 test_buffer = compress_test_bufs[i];
966                 for (level = RTE_COMP_LEVEL_MIN; level <= RTE_COMP_LEVEL_MAX;
967                                 level++) {
968                         compress_xform->compress.level = level;
969                         /* Compress with compressdev, decompress with Zlib */
970                         if (test_deflate_comp_decomp(&test_buffer, 1,
971                                         &i,
972                                         &compress_xform,
973                                         &ts_params->def_decomp_xform,
974                                         1,
975                                         RTE_COMP_OP_STATELESS,
976                                         ZLIB_DECOMPRESS) < 0) {
977                                 ret = TEST_FAILED;
978                                 goto exit;
979                         }
980                 }
981         }
982
983         ret = TEST_SUCCESS;
984
985 exit:
986         rte_free(compress_xform);
987         return ret;
988 }
989
990 #define NUM_XFORMS 3
991 static int
992 test_compressdev_deflate_stateless_multi_xform(void)
993 {
994         struct comp_testsuite_params *ts_params = &testsuite_params;
995         uint16_t num_bufs = NUM_XFORMS;
996         struct rte_comp_xform *compress_xforms[NUM_XFORMS] = {NULL};
997         struct rte_comp_xform *decompress_xforms[NUM_XFORMS] = {NULL};
998         const char *test_buffers[NUM_XFORMS];
999         uint16_t i;
1000         unsigned int level = RTE_COMP_LEVEL_MIN;
1001         uint16_t buf_idx[num_bufs];
1002
1003         int ret;
1004
1005         /* Create multiple xforms with various levels */
1006         for (i = 0; i < NUM_XFORMS; i++) {
1007                 compress_xforms[i] = rte_malloc(NULL,
1008                                 sizeof(struct rte_comp_xform), 0);
1009                 if (compress_xforms[i] == NULL) {
1010                         RTE_LOG(ERR, USER1,
1011                                 "Compress xform could not be created\n");
1012                         ret = TEST_FAILED;
1013                         goto exit;
1014                 }
1015
1016                 memcpy(compress_xforms[i], ts_params->def_comp_xform,
1017                                 sizeof(struct rte_comp_xform));
1018                 compress_xforms[i]->compress.level = level;
1019                 level++;
1020
1021                 decompress_xforms[i] = rte_malloc(NULL,
1022                                 sizeof(struct rte_comp_xform), 0);
1023                 if (decompress_xforms[i] == NULL) {
1024                         RTE_LOG(ERR, USER1,
1025                                 "Decompress xform could not be created\n");
1026                         ret = TEST_FAILED;
1027                         goto exit;
1028                 }
1029
1030                 memcpy(decompress_xforms[i], ts_params->def_decomp_xform,
1031                                 sizeof(struct rte_comp_xform));
1032         }
1033
1034         for (i = 0; i < NUM_XFORMS; i++) {
1035                 buf_idx[i] = 0;
1036                 /* Use the same buffer in all sessions */
1037                 test_buffers[i] = compress_test_bufs[0];
1038         }
1039         /* Compress with compressdev, decompress with Zlib */
1040         if (test_deflate_comp_decomp(test_buffers, num_bufs,
1041                         buf_idx,
1042                         compress_xforms,
1043                         decompress_xforms,
1044                         NUM_XFORMS,
1045                         RTE_COMP_OP_STATELESS,
1046                         ZLIB_DECOMPRESS) < 0) {
1047                 ret = TEST_FAILED;
1048                 goto exit;
1049         }
1050
1051         ret = TEST_SUCCESS;
1052 exit:
1053         for (i = 0; i < NUM_XFORMS; i++) {
1054                 rte_free(compress_xforms[i]);
1055                 rte_free(decompress_xforms[i]);
1056         }
1057
1058         return ret;
1059 }
1060
1061 static struct unit_test_suite compressdev_testsuite  = {
1062         .suite_name = "compressdev unit test suite",
1063         .setup = testsuite_setup,
1064         .teardown = testsuite_teardown,
1065         .unit_test_cases = {
1066                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1067                         test_compressdev_deflate_stateless_fixed),
1068                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1069                         test_compressdev_deflate_stateless_dynamic),
1070                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1071                         test_compressdev_deflate_stateless_multi_op),
1072                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1073                         test_compressdev_deflate_stateless_multi_level),
1074                 TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
1075                         test_compressdev_deflate_stateless_multi_xform),
1076                 TEST_CASES_END() /**< NULL terminate unit test array */
1077         }
1078 };
1079
1080 static int
1081 test_compressdev(void)
1082 {
1083         return unit_test_suite_runner(&compressdev_testsuite);
1084 }
1085
1086 REGISTER_TEST_COMMAND(compressdev_autotest, test_compressdev);