drivers/compress/isal/isal_compress_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <isa-l.h>
   5
   6 #include <rte_bus_vdev.h>
   7 #include <rte_common.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mbuf.h>
  10 #include <rte_compressdev_pmd.h>
  11
  12 #include "isal_compress_pmd_private.h"
  13
  14 #define RTE_COMP_ISAL_WINDOW_SIZE 15
  15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */
  16 #define RTE_COMP_ISAL_LEVEL_ONE 1
  17 #define RTE_COMP_ISAL_LEVEL_TWO 2
  18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
  19 #define CHKSUM_SZ_CRC 8
  20 #define CHKSUM_SZ_ADLER 4
  21
  22 #define STRINGIFY(s) #s
  23 #define ISAL_TOSTRING(maj, min, patch) \
  24         STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch)
  25 #define ISAL_VERSION_STRING \
  26         ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION)
  27
  28 int isal_logtype_driver;
  29
  30 /* Verify and set private xform parameters */
  31 int
  32 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
  33                 const struct rte_comp_xform *xform)
  34 {
  35         if (xform == NULL)
  36                 return -EINVAL;
  37
  38         /* Set compression private xform variables */
  39         if (xform->type == RTE_COMP_COMPRESS) {
  40                 /* Set private xform type - COMPRESS/DECOMPRESS */
  41                 priv_xform->type = RTE_COMP_COMPRESS;
  42
  43                 /* Set private xform algorithm */
  44                 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) {
  45                         if (xform->compress.algo == RTE_COMP_ALGO_NULL) {
  46                                 ISAL_PMD_LOG(ERR, "By-pass not supported\n");
  47                                 return -ENOTSUP;
  48                         }
  49                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
  50                         return -ENOTSUP;
  51                 }
  52                 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
  53
  54                 /* Set private xform window size, 32K supported */
  55                 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
  56                         priv_xform->compress.window_size =
  57                                         RTE_COMP_ISAL_WINDOW_SIZE;
  58                 else {
  59                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
  60                         return -ENOTSUP;
  61                 }
  62
  63                 /* Set private xform huffman type */
  64                 switch (xform->compress.deflate.huffman) {
  65                 case(RTE_COMP_HUFFMAN_DEFAULT):
  66                         priv_xform->compress.deflate.huffman =
  67                                         RTE_COMP_HUFFMAN_DEFAULT;
  68                         break;
  69                 case(RTE_COMP_HUFFMAN_FIXED):
  70                         priv_xform->compress.deflate.huffman =
  71                                         RTE_COMP_HUFFMAN_FIXED;
  72                         break;
  73                 case(RTE_COMP_HUFFMAN_DYNAMIC):
  74                         priv_xform->compress.deflate.huffman =
  75                                         RTE_COMP_HUFFMAN_DYNAMIC;
  76                         break;
  77                 default:
  78                         ISAL_PMD_LOG(ERR, "Huffman code not supported\n");
  79                         return -ENOTSUP;
  80                 }
  81
  82                 /* Set private xform checksum */
  83                 switch (xform->compress.chksum) {
  84                 /* Raw deflate by default */
  85                 case(RTE_COMP_CHECKSUM_NONE):
  86                         priv_xform->compress.chksum = IGZIP_DEFLATE;
  87                         break;
  88                 case(RTE_COMP_CHECKSUM_CRC32):
  89                         priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
  90                         break;
  91                 case(RTE_COMP_CHECKSUM_ADLER32):
  92                         priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
  93                         break;
  94                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
  95                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
  96                                         " supported\n");
  97                         return -ENOTSUP;
  98                 default:
  99                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
 100                         priv_xform->compress.chksum = IGZIP_DEFLATE;
 101                         break;
 102                 }
 103
 104                 /* Set private xform level.
 105                  * Checking compliance with compressdev API, -1 <= level => 9
 106                  */
 107                 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT ||
 108                                 xform->compress.level > RTE_COMP_LEVEL_MAX) {
 109                         ISAL_PMD_LOG(ERR, "Compression level out of range\n");
 110                         return -EINVAL;
 111                 }
 112                 /* Check for Compressdev API level 0, No compression
 113                  * not supported in ISA-L
 114                  */
 115                 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) {
 116                         ISAL_PMD_LOG(ERR, "No Compression not supported\n");
 117                         return -ENOTSUP;
 118                 }
 119                 /* If using fixed huffman code, level must be 0 */
 120                 else if (priv_xform->compress.deflate.huffman ==
 121                                 RTE_COMP_HUFFMAN_FIXED) {
 122                         ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a"
 123                                         " fixed huffman code\n");
 124                         priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO;
 125                         priv_xform->level_buffer_size =
 126                                         ISAL_DEF_LVL0_DEFAULT;
 127                 } else {
 128                         /* Mapping API levels to ISA-L levels 1,2 & 3 */
 129                         switch (xform->compress.level) {
 130                         case RTE_COMP_LEVEL_PMD_DEFAULT:
 131                                 /* Default is 1 if not using fixed huffman */
 132                                 priv_xform->compress.level =
 133                                                 RTE_COMP_ISAL_LEVEL_ONE;
 134                                 priv_xform->level_buffer_size =
 135                                                 ISAL_DEF_LVL1_DEFAULT;
 136                                 break;
 137                         case RTE_COMP_LEVEL_MIN:
 138                                 priv_xform->compress.level =
 139                                                 RTE_COMP_ISAL_LEVEL_ONE;
 140                                 priv_xform->level_buffer_size =
 141                                                 ISAL_DEF_LVL1_DEFAULT;
 142                                 break;
 143                         case RTE_COMP_ISAL_LEVEL_TWO:
 144                                 priv_xform->compress.level =
 145                                                 RTE_COMP_ISAL_LEVEL_TWO;
 146                                 priv_xform->level_buffer_size =
 147                                                 ISAL_DEF_LVL2_DEFAULT;
 148                                 break;
 149                         /* Level 3 or higher requested */
 150                         default:
 151                                 /* Check for AVX512, to use ISA-L level 3 */
 152                                 if (rte_cpu_get_flag_enabled(
 153                                                 RTE_CPUFLAG_AVX512F)) {
 154                                         priv_xform->compress.level =
 155                                                 RTE_COMP_ISAL_LEVEL_THREE;
 156                                         priv_xform->level_buffer_size =
 157                                                 ISAL_DEF_LVL3_DEFAULT;
 158                                 }
 159                                 /* Check for AVX2, to use ISA-L level 3 */
 160                                 else if (rte_cpu_get_flag_enabled(
 161                                                 RTE_CPUFLAG_AVX2)) {
 162                                         priv_xform->compress.level =
 163                                                 RTE_COMP_ISAL_LEVEL_THREE;
 164                                         priv_xform->level_buffer_size =
 165                                                 ISAL_DEF_LVL3_DEFAULT;
 166                                 } else {
 167                                         ISAL_PMD_LOG(DEBUG, "Requested ISA-L level"
 168                                                 " 3 or above; Level 3 optimized"
 169                                                 " for AVX512 & AVX2 only."
 170                                                 " level changed to 2.\n");
 171                                         priv_xform->compress.level =
 172                                                 RTE_COMP_ISAL_LEVEL_TWO;
 173                                         priv_xform->level_buffer_size =
 174                                                 ISAL_DEF_LVL2_DEFAULT;
 175                                 }
 176                         }
 177                 }
 178         }
 179
 180         /* Set decompression private xform variables */
 181         else if (xform->type == RTE_COMP_DECOMPRESS) {
 182
 183                 /* Set private xform type - COMPRESS/DECOMPRESS */
 184                 priv_xform->type = RTE_COMP_DECOMPRESS;
 185
 186                 /* Set private xform algorithm */
 187                 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) {
 188                         if (xform->decompress.algo == RTE_COMP_ALGO_NULL) {
 189                                 ISAL_PMD_LOG(ERR, "By pass not supported\n");
 190                                 return -ENOTSUP;
 191                         }
 192                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
 193                         return -ENOTSUP;
 194                 }
 195                 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
 196
 197                 /* Set private xform checksum */
 198                 switch (xform->decompress.chksum) {
 199                 /* Raw deflate by default */
 200                 case(RTE_COMP_CHECKSUM_NONE):
 201                         priv_xform->decompress.chksum = ISAL_DEFLATE;
 202                         break;
 203                 case(RTE_COMP_CHECKSUM_CRC32):
 204                         priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
 205                         break;
 206                 case(RTE_COMP_CHECKSUM_ADLER32):
 207                         priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
 208                         break;
 209                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
 210                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
 211                                         " supported\n");
 212                         return -ENOTSUP;
 213                 default:
 214                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
 215                         priv_xform->decompress.chksum = ISAL_DEFLATE;
 216                         break;
 217                 }
 218
 219                 /* Set private xform window size, 32K supported */
 220                 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
 221                         priv_xform->decompress.window_size =
 222                                         RTE_COMP_ISAL_WINDOW_SIZE;
 223                 else {
 224                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
 225                         return -ENOTSUP;
 226                 }
 227         }
 228         return 0;
 229 }
 230
 231 /* Compression using chained mbufs for input/output data */
 232 static int
 233 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
 234 {
 235         int ret;
 236         uint32_t remaining_offset;
 237         uint32_t remaining_data = op->src.length;
 238         struct rte_mbuf *src = op->m_src;
 239         struct rte_mbuf *dst = op->m_dst;
 240
 241         /* check for source/destination offset passing multiple segments
 242          * and point compression stream to input/output buffer.
 243          */
 244         remaining_offset = op->src.offset;
 245         while (remaining_offset >= src->data_len) {
 246                 remaining_offset -= src->data_len;
 247                 src = src->next;
 248         }
 249         qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
 250                         op->src.length);
 251         qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
 252                         remaining_offset);
 253
 254         remaining_offset = op->dst.offset;
 255         while (remaining_offset >= dst->data_len) {
 256                 remaining_offset -= dst->data_len;
 257                 dst = dst->next;
 258         }
 259         qp->stream->avail_out = dst->data_len - remaining_offset;
 260         qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
 261                         remaining_offset);
 262
 263         if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
 264                 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
 265                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 266                 return -1;
 267         }
 268
 269         while (qp->stream->internal_state.state != ZSTATE_END) {
 270                 /* Last segment of data */
 271                 if (remaining_data <= src->data_len)
 272                         qp->stream->end_of_stream = 1;
 273
 274                 /* Execute compression operation */
 275                 ret = isal_deflate(qp->stream);
 276
 277                 remaining_data = op->src.length - qp->stream->total_in;
 278
 279                 if (ret != COMP_OK) {
 280                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
 281                         op->status = RTE_COMP_OP_STATUS_ERROR;
 282                         return ret;
 283                 }
 284
 285                 if (qp->stream->avail_in == 0 &&
 286                                 qp->stream->total_in != op->src.length) {
 287                         if (src->next != NULL) {
 288                                 src = src->next;
 289                                 qp->stream->next_in =
 290                                                 rte_pktmbuf_mtod(src, uint8_t *);
 291                                 qp->stream->avail_in =
 292                                         RTE_MIN(remaining_data, src->data_len);
 293                         } else {
 294                                 ISAL_PMD_LOG(ERR,
 295                                 "Not enough input buffer segments\n");
 296                                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 297                                 return -1;
 298                         }
 299                 }
 300
 301                 if (qp->stream->avail_out == 0 &&
 302                                 qp->stream->internal_state.state != ZSTATE_END) {
 303                         if (dst->next != NULL) {
 304                                 dst = dst->next;
 305                                 qp->stream->next_out =
 306                                                 rte_pktmbuf_mtod(dst, uint8_t *);
 307                                 qp->stream->avail_out = dst->data_len;
 308                         } else {
 309                                 ISAL_PMD_LOG(ERR,
 310                                 "Not enough output buffer segments\n");
 311                                 op->status =
 312                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 313                                 return -1;
 314                         }
 315                 }
 316         }
 317
 318         return 0;
 319 }
 320
 321 /* Decompression using chained mbufs for input/output data */
 322 static int
 323 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
 324 {
 325         int ret;
 326         uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
 327         uint32_t remaining_data = op->src.length;
 328         struct rte_mbuf *src = op->m_src;
 329         struct rte_mbuf *dst = op->m_dst;
 330
 331         /* check for offset passing multiple segments
 332          * and point decompression state to input/output buffer
 333          */
 334         src_remaining_offset = op->src.offset;
 335         while (src_remaining_offset >= src->data_len) {
 336                 src_remaining_offset -= src->data_len;
 337                 src = src->next;
 338         }
 339         qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
 340                         op->src.length);
 341         qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
 342                         src_remaining_offset);
 343
 344         dst_remaining_offset = op->dst.offset;
 345         while (dst_remaining_offset >= dst->data_len) {
 346                 dst_remaining_offset -= dst->data_len;
 347                 dst = dst->next;
 348         }
 349         qp->state->avail_out = dst->data_len - dst_remaining_offset;
 350         qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
 351                         dst_remaining_offset);
 352
 353         while (qp->state->block_state != ISAL_BLOCK_FINISH) {
 354
 355                 ret = isal_inflate(qp->state);
 356
 357                 /* Check for first segment, offset needs to be accounted for */
 358                 if (remaining_data == op->src.length) {
 359                         consumed_data = src->data_len - src_remaining_offset;
 360                 } else
 361                         consumed_data = src->data_len;
 362
 363                 if (qp->state->avail_in == 0
 364                                 && op->consumed != op->src.length) {
 365                         op->consumed += consumed_data;
 366                         remaining_data -= consumed_data;
 367
 368                         if (src->next != NULL) {
 369                                 src = src->next;
 370                                 qp->state->next_in =
 371                                                 rte_pktmbuf_mtod(src, uint8_t *);
 372                                 qp->state->avail_in =
 373                                         RTE_MIN(remaining_data, src->data_len);
 374                         }
 375                 }
 376
 377                 if (ret == ISAL_OUT_OVERFLOW) {
 378                         ISAL_PMD_LOG(ERR, "Decompression operation ran "
 379                                 "out of space, but can be recovered.\n%d bytes "
 380                                 "consumed\t%d bytes produced\n",
 381                                 consumed_data, qp->state->total_out);
 382                                 op->status =
 383                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
 384                         return ret;
 385                 } else if (ret < 0) {
 386                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 387                         op->status = RTE_COMP_OP_STATUS_ERROR;
 388                         return ret;
 389                 }
 390
 391                 if (qp->state->avail_out == 0 &&
 392                                 qp->state->block_state != ISAL_BLOCK_FINISH) {
 393                         if (dst->next != NULL) {
 394                                 dst = dst->next;
 395                                 qp->state->next_out =
 396                                                 rte_pktmbuf_mtod(dst, uint8_t *);
 397                                 qp->state->avail_out = dst->data_len;
 398                         } else {
 399                                 ISAL_PMD_LOG(ERR,
 400                                 "Not enough output buffer segments\n");
 401                                 op->status =
 402                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 403                                 return -1;
 404                         }
 405                 }
 406         }
 407
 408         return 0;
 409 }
 410
 411 /* Stateless Compression Function */
 412 static int
 413 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 414                 struct isal_priv_xform *priv_xform)
 415 {
 416         int ret = 0;
 417         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 418
 419         /* Required due to init clearing level_buf */
 420         uint8_t *temp_level_buf = qp->stream->level_buf;
 421
 422         /* Initialize compression stream */
 423         isal_deflate_init(qp->stream);
 424
 425         qp->stream->level_buf = temp_level_buf;
 426
 427         /* Set Checksum flag */
 428         qp->stream->gzip_flag = priv_xform->compress.chksum;
 429
 430         /* Stateless operation, input will be consumed in one go */
 431         qp->stream->flush = NO_FLUSH;
 432
 433         /* set compression level & intermediate level buffer size */
 434         qp->stream->level = priv_xform->compress.level;
 435         qp->stream->level_buf_size = priv_xform->level_buffer_size;
 436
 437         /* Set op huffman code */
 438         if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
 439                 isal_deflate_set_hufftables(qp->stream, NULL,
 440                                 IGZIP_HUFFTABLE_STATIC);
 441         else if (priv_xform->compress.deflate.huffman ==
 442                         RTE_COMP_HUFFMAN_DEFAULT)
 443                 isal_deflate_set_hufftables(qp->stream, NULL,
 444                         IGZIP_HUFFTABLE_DEFAULT);
 445         /* Dynamically change the huffman code to suit the input data */
 446         else if (priv_xform->compress.deflate.huffman ==
 447                         RTE_COMP_HUFFMAN_DYNAMIC)
 448                 isal_deflate_set_hufftables(qp->stream, NULL,
 449                                 IGZIP_HUFFTABLE_DEFAULT);
 450
 451         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 452                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 453                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 454                 return -1;
 455         }
 456
 457         if (op->dst.offset >= op->m_dst->pkt_len) {
 458                 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
 459                                 " for offset provided.\n");
 460                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 461                 return -1;
 462         }
 463
 464         /* Chained mbufs */
 465         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
 466                 ret = chained_mbuf_compression(op, qp);
 467                 if (ret < 0)
 468                         return ret;
 469         } else {
 470                 /* Linear buffer */
 471                 qp->stream->end_of_stream = 1; /* All input consumed in one */
 472                 /* Point compression stream to input buffer */
 473                 qp->stream->avail_in = op->src.length;
 474                 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
 475                                 uint8_t *, op->src.offset);
 476
 477                 /*  Point compression stream to output buffer */
 478                 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
 479                 qp->stream->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
 480                                 uint8_t *, op->dst.offset);
 481
 482                 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
 483                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
 484                                         " buffers\n");
 485                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 486                         return -1;
 487                 }
 488
 489                 /* Execute compression operation */
 490                 ret =  isal_deflate_stateless(qp->stream);
 491
 492                 /* Check that output buffer did not run out of space */
 493                 if (ret == STATELESS_OVERFLOW) {
 494                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
 495                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 496                         return ret;
 497                 }
 498
 499                 /* Check that input buffer has been fully consumed */
 500                 if (qp->stream->avail_in != (uint32_t)0) {
 501                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
 502                                         " entirely\n");
 503                         op->status = RTE_COMP_OP_STATUS_ERROR;
 504                         return -1;
 505                 }
 506
 507                 if (ret != COMP_OK) {
 508                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
 509                         op->status = RTE_COMP_OP_STATUS_ERROR;
 510                         return ret;
 511                 }
 512         }
 513
 514         op->consumed = qp->stream->total_in;
 515         if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
 516                 op->produced = qp->stream->total_out;
 517         } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
 518                 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
 519                 op->output_chksum = qp->stream->internal_state.crc + 1;
 520         } else {
 521                 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
 522                 op->output_chksum = qp->stream->internal_state.crc;
 523         }
 524
 525         return ret;
 526 }
 527
 528 /* Stateless Decompression Function */
 529 static int
 530 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 531                 struct isal_priv_xform *priv_xform)
 532 {
 533         int ret = 0;
 534
 535         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 536
 537         /* Initialize decompression state */
 538         isal_inflate_init(qp->state);
 539
 540         /* Set Checksum flag */
 541         qp->state->crc_flag = priv_xform->decompress.chksum;
 542
 543         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 544                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 545                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 546                 return -1;
 547         }
 548
 549         if (op->dst.offset >= op->m_dst->pkt_len) {
 550                 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
 551                                 "offset provided.\n");
 552                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 553                 return -1;
 554         }
 555
 556         /* Chained mbufs */
 557         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
 558                 ret = chained_mbuf_decompression(op, qp);
 559                 if (ret !=  0)
 560                         return ret;
 561         } else {
 562                 /* Linear buffer */
 563                 /* Point decompression state to input buffer */
 564                 qp->state->avail_in = op->src.length;
 565                 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
 566                                 uint8_t *, op->src.offset);
 567
 568                 /* Point decompression state to output buffer */
 569                 qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
 570                 qp->state->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
 571                                 uint8_t *, op->dst.offset);
 572
 573                 if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
 574                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
 575                                         " buffers\n");
 576                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 577                         return -1;
 578                 }
 579
 580                 /* Execute decompression operation */
 581                 ret = isal_inflate_stateless(qp->state);
 582
 583                 if (ret == ISAL_OUT_OVERFLOW) {
 584                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
 585                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 586                         return ret;
 587                 }
 588
 589                 /* Check that input buffer has been fully consumed */
 590                 if (qp->state->avail_in != (uint32_t)0) {
 591                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
 592                                         " entirely\n");
 593                         op->status = RTE_COMP_OP_STATUS_ERROR;
 594                         return -1;
 595                 }
 596
 597                 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
 598                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 599                         op->status = RTE_COMP_OP_STATUS_ERROR;
 600                         return ret;
 601                 }
 602                 op->consumed = op->src.length - qp->state->avail_in;
 603         }
 604         op->produced = qp->state->total_out;
 605         op->output_chksum = qp->state->crc;
 606
 607         return ret;
 608 }
 609
 610 /* Process compression/decompression operation */
 611 static int
 612 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
 613                 struct isal_priv_xform *priv_xform)
 614 {
 615         switch (priv_xform->type) {
 616         case RTE_COMP_COMPRESS:
 617                 process_isal_deflate(op, qp, priv_xform);
 618                 break;
 619         case RTE_COMP_DECOMPRESS:
 620                 process_isal_inflate(op, qp, priv_xform);
 621                 break;
 622         default:
 623                 ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
 624                 return -ENOTSUP;
 625         }
 626         return 0;
 627 }
 628
 629 /* Enqueue burst */
 630 static uint16_t
 631 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops,
 632                         uint16_t nb_ops)
 633 {
 634         struct isal_comp_qp *qp = queue_pair;
 635         uint16_t i;
 636         int retval;
 637         int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops);
 638
 639         for (i = 0; i < num_enq; i++) {
 640                 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) {
 641                         ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 642                         ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n");
 643                         qp->qp_stats.enqueue_err_count++;
 644                         continue;
 645                 }
 646                 retval = process_op(qp, ops[i], ops[i]->private_xform);
 647                 if (unlikely(retval < 0) ||
 648                                 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 649                         qp->qp_stats.enqueue_err_count++;
 650                 }
 651         }
 652
 653         retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops,
 654                         num_enq, NULL);
 655         qp->num_free_elements -= retval;
 656         qp->qp_stats.enqueued_count += retval;
 657
 658         return retval;
 659 }
 660
 661 /* Dequeue burst */
 662 static uint16_t
 663 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
 664                 uint16_t nb_ops)
 665 {
 666         struct isal_comp_qp *qp = queue_pair;
 667         uint16_t nb_dequeued;
 668
 669         nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops,
 670                         nb_ops, NULL);
 671         qp->num_free_elements += nb_dequeued;
 672         qp->qp_stats.dequeued_count += nb_dequeued;
 673
 674         return nb_dequeued;
 675 }
 676
 677 /* Create ISA-L compression device */
 678 static int
 679 compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
 680                 struct rte_compressdev_pmd_init_params *init_params)
 681 {
 682         struct rte_compressdev *dev;
 683
 684         dev = rte_compressdev_pmd_create(name, &vdev->device,
 685                         sizeof(struct isal_comp_private), init_params);
 686         if (dev == NULL) {
 687                 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev");
 688                 return -EFAULT;
 689         }
 690
 691         dev->dev_ops = isal_compress_pmd_ops;
 692
 693         /* register rx/tx burst functions for data path */
 694         dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
 695         dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
 696
 697         ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING);
 698
 699         return 0;
 700 }
 701
 702 /** Remove compression device */
 703 static int
 704 compdev_isal_remove_dev(struct rte_vdev_device *vdev)
 705 {
 706         struct rte_compressdev *compdev;
 707         const char *name;
 708
 709         name = rte_vdev_device_name(vdev);
 710         if (name == NULL)
 711                 return -EINVAL;
 712
 713         compdev = rte_compressdev_pmd_get_named_dev(name);
 714         if (compdev == NULL)
 715                 return -ENODEV;
 716
 717         return rte_compressdev_pmd_destroy(compdev);
 718 }
 719
 720 /** Initialise ISA-L compression device */
 721 static int
 722 compdev_isal_probe(struct rte_vdev_device *dev)
 723 {
 724         struct rte_compressdev_pmd_init_params init_params = {
 725                 "",
 726                 rte_socket_id(),
 727         };
 728         const char *name, *args;
 729         int retval;
 730
 731         name = rte_vdev_device_name(dev);
 732         if (name == NULL)
 733                 return -EINVAL;
 734
 735         args = rte_vdev_device_args(dev);
 736
 737         retval = rte_compressdev_pmd_parse_input_args(&init_params, args);
 738         if (retval) {
 739                 ISAL_PMD_LOG(ERR,
 740                         "Failed to parse initialisation arguments[%s]\n", args);
 741                 return -EINVAL;
 742         }
 743
 744         return compdev_isal_create(name, dev, &init_params);
 745 }
 746
 747 static struct rte_vdev_driver compdev_isal_pmd_drv = {
 748         .probe = compdev_isal_probe,
 749         .remove = compdev_isal_remove_dev,
 750 };
 751
 752 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
 753 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
 754         "socket_id=<int>");
 755
 756 RTE_INIT(isal_init_log)
 757 {
 758         isal_logtype_driver = rte_log_register("pmd.compress.isal");
 759         if (isal_logtype_driver >= 0)
 760                 rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO);
 761 }