examples/power: fix FreeBSD meson lib dependency
[dpdk.git] / drivers / compress / isal / isal_compress_pmd.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 Intel Corporation
3  */
4 #include <isa-l.h>
5
6 #include <rte_bus_vdev.h>
7 #include <rte_common.h>
8 #include <rte_malloc.h>
9 #include <rte_mbuf.h>
10 #include <rte_compressdev_pmd.h>
11
12 #include "isal_compress_pmd_private.h"
13
14 #define RTE_COMP_ISAL_WINDOW_SIZE 15
15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */
16 #define RTE_COMP_ISAL_LEVEL_ONE 1
17 #define RTE_COMP_ISAL_LEVEL_TWO 2
18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
19 #define CHKSUM_SZ_CRC 8
20 #define CHKSUM_SZ_ADLER 4
21
22 #define STRINGIFY(s) #s
23 #define ISAL_TOSTRING(maj, min, patch) \
24         STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch)
25 #define ISAL_VERSION_STRING \
26         ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION)
27
28 int isal_logtype_driver;
29
30 /* Verify and set private xform parameters */
31 int
32 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
33                 const struct rte_comp_xform *xform)
34 {
35         if (xform == NULL)
36                 return -EINVAL;
37
38         /* Set compression private xform variables */
39         if (xform->type == RTE_COMP_COMPRESS) {
40                 /* Set private xform type - COMPRESS/DECOMPRESS */
41                 priv_xform->type = RTE_COMP_COMPRESS;
42
43                 /* Set private xform algorithm */
44                 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) {
45                         if (xform->compress.algo == RTE_COMP_ALGO_NULL) {
46                                 ISAL_PMD_LOG(ERR, "By-pass not supported\n");
47                                 return -ENOTSUP;
48                         }
49                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
50                         return -ENOTSUP;
51                 }
52                 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
53
54                 /* Set private xform window size, 32K supported */
55                 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
56                         priv_xform->compress.window_size =
57                                         RTE_COMP_ISAL_WINDOW_SIZE;
58                 else {
59                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
60                         return -ENOTSUP;
61                 }
62
63                 /* Set private xform huffman type */
64                 switch (xform->compress.deflate.huffman) {
65                 case(RTE_COMP_HUFFMAN_DEFAULT):
66                         priv_xform->compress.deflate.huffman =
67                                         RTE_COMP_HUFFMAN_DEFAULT;
68                         break;
69                 case(RTE_COMP_HUFFMAN_FIXED):
70                         priv_xform->compress.deflate.huffman =
71                                         RTE_COMP_HUFFMAN_FIXED;
72                         break;
73                 case(RTE_COMP_HUFFMAN_DYNAMIC):
74                         priv_xform->compress.deflate.huffman =
75                                         RTE_COMP_HUFFMAN_DYNAMIC;
76                         break;
77                 default:
78                         ISAL_PMD_LOG(ERR, "Huffman code not supported\n");
79                         return -ENOTSUP;
80                 }
81
82                 /* Set private xform checksum */
83                 switch (xform->compress.chksum) {
84                 /* Raw deflate by default */
85                 case(RTE_COMP_CHECKSUM_NONE):
86                         priv_xform->compress.chksum = IGZIP_DEFLATE;
87                         break;
88                 case(RTE_COMP_CHECKSUM_CRC32):
89                         priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
90                         break;
91                 case(RTE_COMP_CHECKSUM_ADLER32):
92                         priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
93                         break;
94                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
95                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
96                                         " supported\n");
97                         return -ENOTSUP;
98                 default:
99                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
100                         priv_xform->compress.chksum = IGZIP_DEFLATE;
101                         break;
102                 }
103
104                 /* Set private xform level.
105                  * Checking compliance with compressdev API, -1 <= level => 9
106                  */
107                 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT ||
108                                 xform->compress.level > RTE_COMP_LEVEL_MAX) {
109                         ISAL_PMD_LOG(ERR, "Compression level out of range\n");
110                         return -EINVAL;
111                 }
112                 /* Check for Compressdev API level 0, No compression
113                  * not supported in ISA-L
114                  */
115                 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) {
116                         ISAL_PMD_LOG(ERR, "No Compression not supported\n");
117                         return -ENOTSUP;
118                 }
119                 /* If using fixed huffman code, level must be 0 */
120                 else if (priv_xform->compress.deflate.huffman ==
121                                 RTE_COMP_HUFFMAN_FIXED) {
122                         ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a"
123                                         " fixed huffman code\n");
124                         priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO;
125                         priv_xform->level_buffer_size =
126                                         ISAL_DEF_LVL0_DEFAULT;
127                 } else {
128                         /* Mapping API levels to ISA-L levels 1,2 & 3 */
129                         switch (xform->compress.level) {
130                         case RTE_COMP_LEVEL_PMD_DEFAULT:
131                                 /* Default is 1 if not using fixed huffman */
132                                 priv_xform->compress.level =
133                                                 RTE_COMP_ISAL_LEVEL_ONE;
134                                 priv_xform->level_buffer_size =
135                                                 ISAL_DEF_LVL1_DEFAULT;
136                                 break;
137                         case RTE_COMP_LEVEL_MIN:
138                                 priv_xform->compress.level =
139                                                 RTE_COMP_ISAL_LEVEL_ONE;
140                                 priv_xform->level_buffer_size =
141                                                 ISAL_DEF_LVL1_DEFAULT;
142                                 break;
143                         case RTE_COMP_ISAL_LEVEL_TWO:
144                                 priv_xform->compress.level =
145                                                 RTE_COMP_ISAL_LEVEL_TWO;
146                                 priv_xform->level_buffer_size =
147                                                 ISAL_DEF_LVL2_DEFAULT;
148                                 break;
149                         /* Level 3 or higher requested */
150                         default:
151                                 /* Check for AVX512, to use ISA-L level 3 */
152                                 if (rte_cpu_get_flag_enabled(
153                                                 RTE_CPUFLAG_AVX512F)) {
154                                         priv_xform->compress.level =
155                                                 RTE_COMP_ISAL_LEVEL_THREE;
156                                         priv_xform->level_buffer_size =
157                                                 ISAL_DEF_LVL3_DEFAULT;
158                                 }
159                                 /* Check for AVX2, to use ISA-L level 3 */
160                                 else if (rte_cpu_get_flag_enabled(
161                                                 RTE_CPUFLAG_AVX2)) {
162                                         priv_xform->compress.level =
163                                                 RTE_COMP_ISAL_LEVEL_THREE;
164                                         priv_xform->level_buffer_size =
165                                                 ISAL_DEF_LVL3_DEFAULT;
166                                 } else {
167                                         ISAL_PMD_LOG(DEBUG, "Requested ISA-L level"
168                                                 " 3 or above; Level 3 optimized"
169                                                 " for AVX512 & AVX2 only."
170                                                 " level changed to 2.\n");
171                                         priv_xform->compress.level =
172                                                 RTE_COMP_ISAL_LEVEL_TWO;
173                                         priv_xform->level_buffer_size =
174                                                 ISAL_DEF_LVL2_DEFAULT;
175                                 }
176                         }
177                 }
178         }
179
180         /* Set decompression private xform variables */
181         else if (xform->type == RTE_COMP_DECOMPRESS) {
182
183                 /* Set private xform type - COMPRESS/DECOMPRESS */
184                 priv_xform->type = RTE_COMP_DECOMPRESS;
185
186                 /* Set private xform algorithm */
187                 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) {
188                         if (xform->decompress.algo == RTE_COMP_ALGO_NULL) {
189                                 ISAL_PMD_LOG(ERR, "By pass not supported\n");
190                                 return -ENOTSUP;
191                         }
192                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
193                         return -ENOTSUP;
194                 }
195                 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
196
197                 /* Set private xform checksum */
198                 switch (xform->decompress.chksum) {
199                 /* Raw deflate by default */
200                 case(RTE_COMP_CHECKSUM_NONE):
201                         priv_xform->decompress.chksum = ISAL_DEFLATE;
202                         break;
203                 case(RTE_COMP_CHECKSUM_CRC32):
204                         priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
205                         break;
206                 case(RTE_COMP_CHECKSUM_ADLER32):
207                         priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
208                         break;
209                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
210                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
211                                         " supported\n");
212                         return -ENOTSUP;
213                 default:
214                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
215                         priv_xform->decompress.chksum = ISAL_DEFLATE;
216                         break;
217                 }
218
219                 /* Set private xform window size, 32K supported */
220                 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
221                         priv_xform->decompress.window_size =
222                                         RTE_COMP_ISAL_WINDOW_SIZE;
223                 else {
224                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
225                         return -ENOTSUP;
226                 }
227         }
228         return 0;
229 }
230
231 /* Compression using chained mbufs for input/output data */
232 static int
233 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
234 {
235         int ret;
236         uint32_t remaining_offset;
237         uint32_t remaining_data = op->src.length;
238         struct rte_mbuf *src = op->m_src;
239         struct rte_mbuf *dst = op->m_dst;
240
241         /* check for source/destination offset passing multiple segments
242          * and point compression stream to input/output buffer.
243          */
244         remaining_offset = op->src.offset;
245         while (remaining_offset >= src->data_len) {
246                 remaining_offset -= src->data_len;
247                 src = src->next;
248         }
249         qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
250                         op->src.length);
251         qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
252                         remaining_offset);
253
254         remaining_offset = op->dst.offset;
255         while (remaining_offset >= dst->data_len) {
256                 remaining_offset -= dst->data_len;
257                 dst = dst->next;
258         }
259         qp->stream->avail_out = dst->data_len - remaining_offset;
260         qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
261                         remaining_offset);
262
263         if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
264                 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
265                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
266                 return -1;
267         }
268
269         while (qp->stream->internal_state.state != ZSTATE_END) {
270                 /* Last segment of data */
271                 if (remaining_data <= src->data_len)
272                         qp->stream->end_of_stream = 1;
273
274                 /* Execute compression operation */
275                 ret = isal_deflate(qp->stream);
276
277                 remaining_data = op->src.length - qp->stream->total_in;
278
279                 if (ret != COMP_OK) {
280                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
281                         op->status = RTE_COMP_OP_STATUS_ERROR;
282                         return ret;
283                 }
284
285                 if (qp->stream->avail_in == 0 &&
286                                 qp->stream->total_in != op->src.length) {
287                         if (src->next != NULL) {
288                                 src = src->next;
289                                 qp->stream->next_in =
290                                                 rte_pktmbuf_mtod(src, uint8_t *);
291                                 qp->stream->avail_in =
292                                         RTE_MIN(remaining_data, src->data_len);
293                         } else {
294                                 ISAL_PMD_LOG(ERR,
295                                 "Not enough input buffer segments\n");
296                                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
297                                 return -1;
298                         }
299                 }
300
301                 if (qp->stream->avail_out == 0 &&
302                                 qp->stream->internal_state.state != ZSTATE_END) {
303                         if (dst->next != NULL) {
304                                 dst = dst->next;
305                                 qp->stream->next_out =
306                                                 rte_pktmbuf_mtod(dst, uint8_t *);
307                                 qp->stream->avail_out = dst->data_len;
308                         } else {
309                                 ISAL_PMD_LOG(ERR,
310                                 "Not enough output buffer segments\n");
311                                 op->status =
312                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
313                                 return -1;
314                         }
315                 }
316         }
317
318         return 0;
319 }
320
321 /* Decompression using chained mbufs for input/output data */
322 static int
323 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
324 {
325         int ret;
326         uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
327         uint32_t remaining_data = op->src.length;
328         struct rte_mbuf *src = op->m_src;
329         struct rte_mbuf *dst = op->m_dst;
330
331         /* check for offset passing multiple segments
332          * and point decompression state to input/output buffer
333          */
334         src_remaining_offset = op->src.offset;
335         while (src_remaining_offset >= src->data_len) {
336                 src_remaining_offset -= src->data_len;
337                 src = src->next;
338         }
339         qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
340                         op->src.length);
341         qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
342                         src_remaining_offset);
343
344         dst_remaining_offset = op->dst.offset;
345         while (dst_remaining_offset >= dst->data_len) {
346                 dst_remaining_offset -= dst->data_len;
347                 dst = dst->next;
348         }
349         qp->state->avail_out = dst->data_len - dst_remaining_offset;
350         qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
351                         dst_remaining_offset);
352
353         while (qp->state->block_state != ISAL_BLOCK_FINISH) {
354
355                 ret = isal_inflate(qp->state);
356
357                 /* Check for first segment, offset needs to be accounted for */
358                 if (remaining_data == op->src.length) {
359                         consumed_data = src->data_len - src_remaining_offset;
360                 } else
361                         consumed_data = src->data_len;
362
363                 if (qp->state->avail_in == 0
364                                 && op->consumed != op->src.length) {
365                         op->consumed += consumed_data;
366                         remaining_data -= consumed_data;
367
368                         if (src->next != NULL) {
369                                 src = src->next;
370                                 qp->state->next_in =
371                                                 rte_pktmbuf_mtod(src, uint8_t *);
372                                 qp->state->avail_in =
373                                         RTE_MIN(remaining_data, src->data_len);
374                         }
375                 }
376
377                 if (ret == ISAL_OUT_OVERFLOW) {
378                         ISAL_PMD_LOG(ERR, "Decompression operation ran "
379                                 "out of space, but can be recovered.\n%d bytes "
380                                 "consumed\t%d bytes produced\n",
381                                 consumed_data, qp->state->total_out);
382                                 op->status =
383                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
384                         return ret;
385                 } else if (ret < 0) {
386                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
387                         op->status = RTE_COMP_OP_STATUS_ERROR;
388                         return ret;
389                 }
390
391                 if (qp->state->avail_out == 0 &&
392                                 qp->state->block_state != ISAL_BLOCK_FINISH) {
393                         if (dst->next != NULL) {
394                                 dst = dst->next;
395                                 qp->state->next_out =
396                                                 rte_pktmbuf_mtod(dst, uint8_t *);
397                                 qp->state->avail_out = dst->data_len;
398                         } else {
399                                 ISAL_PMD_LOG(ERR,
400                                 "Not enough output buffer segments\n");
401                                 op->status =
402                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
403                                 return -1;
404                         }
405                 }
406         }
407
408         return 0;
409 }
410
411 /* Stateless Compression Function */
412 static int
413 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
414                 struct isal_priv_xform *priv_xform)
415 {
416         int ret = 0;
417         op->status = RTE_COMP_OP_STATUS_SUCCESS;
418
419         /* Required due to init clearing level_buf */
420         uint8_t *temp_level_buf = qp->stream->level_buf;
421
422         /* Initialize compression stream */
423         isal_deflate_init(qp->stream);
424
425         qp->stream->level_buf = temp_level_buf;
426
427         /* Set Checksum flag */
428         qp->stream->gzip_flag = priv_xform->compress.chksum;
429
430         /* Stateless operation, input will be consumed in one go */
431         qp->stream->flush = NO_FLUSH;
432
433         /* set compression level & intermediate level buffer size */
434         qp->stream->level = priv_xform->compress.level;
435         qp->stream->level_buf_size = priv_xform->level_buffer_size;
436
437         /* Set op huffman code */
438         if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
439                 isal_deflate_set_hufftables(qp->stream, NULL,
440                                 IGZIP_HUFFTABLE_STATIC);
441         else if (priv_xform->compress.deflate.huffman ==
442                         RTE_COMP_HUFFMAN_DEFAULT)
443                 isal_deflate_set_hufftables(qp->stream, NULL,
444                         IGZIP_HUFFTABLE_DEFAULT);
445         /* Dynamically change the huffman code to suit the input data */
446         else if (priv_xform->compress.deflate.huffman ==
447                         RTE_COMP_HUFFMAN_DYNAMIC)
448                 isal_deflate_set_hufftables(qp->stream, NULL,
449                                 IGZIP_HUFFTABLE_DEFAULT);
450
451         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
452                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
453                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
454                 return -1;
455         }
456
457         if (op->dst.offset >= op->m_dst->pkt_len) {
458                 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
459                                 " for offset provided.\n");
460                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
461                 return -1;
462         }
463
464         /* Chained mbufs */
465         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
466                 ret = chained_mbuf_compression(op, qp);
467                 if (ret < 0)
468                         return ret;
469         } else {
470                 /* Linear buffer */
471                 qp->stream->end_of_stream = 1; /* All input consumed in one */
472                 /* Point compression stream to input buffer */
473                 qp->stream->avail_in = op->src.length;
474                 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
475                                 uint8_t *, op->src.offset);
476
477                 /*  Point compression stream to output buffer */
478                 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
479                 qp->stream->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
480                                 uint8_t *, op->dst.offset);
481
482                 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
483                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
484                                         " buffers\n");
485                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
486                         return -1;
487                 }
488
489                 /* Execute compression operation */
490                 ret =  isal_deflate_stateless(qp->stream);
491
492                 /* Check that output buffer did not run out of space */
493                 if (ret == STATELESS_OVERFLOW) {
494                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
495                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
496                         return ret;
497                 }
498
499                 /* Check that input buffer has been fully consumed */
500                 if (qp->stream->avail_in != (uint32_t)0) {
501                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
502                                         " entirely\n");
503                         op->status = RTE_COMP_OP_STATUS_ERROR;
504                         return -1;
505                 }
506
507                 if (ret != COMP_OK) {
508                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
509                         op->status = RTE_COMP_OP_STATUS_ERROR;
510                         return ret;
511                 }
512         }
513
514         op->consumed = qp->stream->total_in;
515         if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
516                 op->produced = qp->stream->total_out;
517         } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
518                 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
519                 op->output_chksum = qp->stream->internal_state.crc + 1;
520         } else {
521                 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
522                 op->output_chksum = qp->stream->internal_state.crc;
523         }
524
525         return ret;
526 }
527
528 /* Stateless Decompression Function */
529 static int
530 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
531                 struct isal_priv_xform *priv_xform)
532 {
533         int ret = 0;
534
535         op->status = RTE_COMP_OP_STATUS_SUCCESS;
536
537         /* Initialize decompression state */
538         isal_inflate_init(qp->state);
539
540         /* Set Checksum flag */
541         qp->state->crc_flag = priv_xform->decompress.chksum;
542
543         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
544                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
545                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
546                 return -1;
547         }
548
549         if (op->dst.offset >= op->m_dst->pkt_len) {
550                 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
551                                 "offset provided.\n");
552                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
553                 return -1;
554         }
555
556         /* Chained mbufs */
557         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
558                 ret = chained_mbuf_decompression(op, qp);
559                 if (ret !=  0)
560                         return ret;
561         } else {
562                 /* Linear buffer */
563                 /* Point decompression state to input buffer */
564                 qp->state->avail_in = op->src.length;
565                 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
566                                 uint8_t *, op->src.offset);
567
568                 /* Point decompression state to output buffer */
569                 qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
570                 qp->state->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
571                                 uint8_t *, op->dst.offset);
572
573                 if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
574                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
575                                         " buffers\n");
576                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
577                         return -1;
578                 }
579
580                 /* Execute decompression operation */
581                 ret = isal_inflate_stateless(qp->state);
582
583                 if (ret == ISAL_OUT_OVERFLOW) {
584                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
585                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
586                         return ret;
587                 }
588
589                 /* Check that input buffer has been fully consumed */
590                 if (qp->state->avail_in != (uint32_t)0) {
591                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
592                                         " entirely\n");
593                         op->status = RTE_COMP_OP_STATUS_ERROR;
594                         return -1;
595                 }
596
597                 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
598                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
599                         op->status = RTE_COMP_OP_STATUS_ERROR;
600                         return ret;
601                 }
602                 op->consumed = op->src.length - qp->state->avail_in;
603         }
604         op->produced = qp->state->total_out;
605         op->output_chksum = qp->state->crc;
606
607         return ret;
608 }
609
610 /* Process compression/decompression operation */
611 static int
612 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
613                 struct isal_priv_xform *priv_xform)
614 {
615         switch (priv_xform->type) {
616         case RTE_COMP_COMPRESS:
617                 process_isal_deflate(op, qp, priv_xform);
618                 break;
619         case RTE_COMP_DECOMPRESS:
620                 process_isal_inflate(op, qp, priv_xform);
621                 break;
622         default:
623                 ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
624                 return -ENOTSUP;
625         }
626         return 0;
627 }
628
629 /* Enqueue burst */
630 static uint16_t
631 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops,
632                         uint16_t nb_ops)
633 {
634         struct isal_comp_qp *qp = queue_pair;
635         uint16_t i;
636         int retval;
637         int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops);
638
639         for (i = 0; i < num_enq; i++) {
640                 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) {
641                         ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
642                         ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n");
643                         qp->qp_stats.enqueue_err_count++;
644                         continue;
645                 }
646                 retval = process_op(qp, ops[i], ops[i]->private_xform);
647                 if (unlikely(retval < 0) ||
648                                 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
649                         qp->qp_stats.enqueue_err_count++;
650                 }
651         }
652
653         retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops,
654                         num_enq, NULL);
655         qp->num_free_elements -= retval;
656         qp->qp_stats.enqueued_count += retval;
657
658         return retval;
659 }
660
661 /* Dequeue burst */
662 static uint16_t
663 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
664                 uint16_t nb_ops)
665 {
666         struct isal_comp_qp *qp = queue_pair;
667         uint16_t nb_dequeued;
668
669         nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops,
670                         nb_ops, NULL);
671         qp->num_free_elements += nb_dequeued;
672         qp->qp_stats.dequeued_count += nb_dequeued;
673
674         return nb_dequeued;
675 }
676
677 /* Create ISA-L compression device */
678 static int
679 compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
680                 struct rte_compressdev_pmd_init_params *init_params)
681 {
682         struct rte_compressdev *dev;
683
684         dev = rte_compressdev_pmd_create(name, &vdev->device,
685                         sizeof(struct isal_comp_private), init_params);
686         if (dev == NULL) {
687                 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev");
688                 return -EFAULT;
689         }
690
691         dev->dev_ops = isal_compress_pmd_ops;
692
693         /* register rx/tx burst functions for data path */
694         dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
695         dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
696
697         ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING);
698
699         return 0;
700 }
701
702 /** Remove compression device */
703 static int
704 compdev_isal_remove_dev(struct rte_vdev_device *vdev)
705 {
706         struct rte_compressdev *compdev;
707         const char *name;
708
709         name = rte_vdev_device_name(vdev);
710         if (name == NULL)
711                 return -EINVAL;
712
713         compdev = rte_compressdev_pmd_get_named_dev(name);
714         if (compdev == NULL)
715                 return -ENODEV;
716
717         return rte_compressdev_pmd_destroy(compdev);
718 }
719
720 /** Initialise ISA-L compression device */
721 static int
722 compdev_isal_probe(struct rte_vdev_device *dev)
723 {
724         struct rte_compressdev_pmd_init_params init_params = {
725                 "",
726                 rte_socket_id(),
727         };
728         const char *name, *args;
729         int retval;
730
731         name = rte_vdev_device_name(dev);
732         if (name == NULL)
733                 return -EINVAL;
734
735         args = rte_vdev_device_args(dev);
736
737         retval = rte_compressdev_pmd_parse_input_args(&init_params, args);
738         if (retval) {
739                 ISAL_PMD_LOG(ERR,
740                         "Failed to parse initialisation arguments[%s]\n", args);
741                 return -EINVAL;
742         }
743
744         return compdev_isal_create(name, dev, &init_params);
745 }
746
747 static struct rte_vdev_driver compdev_isal_pmd_drv = {
748         .probe = compdev_isal_probe,
749         .remove = compdev_isal_remove_dev,
750 };
751
752 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
753 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
754         "socket_id=<int>");
755
756 RTE_INIT(isal_init_log)
757 {
758         isal_logtype_driver = rte_log_register("pmd.compress.isal");
759         if (isal_logtype_driver >= 0)
760                 rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO);
761 }