drivers/compress/isal/isal_compress_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <isa-l.h>
   5
   6 #include <rte_bus_vdev.h>
   7 #include <rte_common.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mbuf.h>
  10 #include <rte_compressdev_pmd.h>
  11
  12 #include "isal_compress_pmd_private.h"
  13
  14 #define RTE_COMP_ISAL_WINDOW_SIZE 15
  15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */
  16 #define RTE_COMP_ISAL_LEVEL_ONE 1
  17 #define RTE_COMP_ISAL_LEVEL_TWO 2
  18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
  19 #define CHKSUM_SZ_CRC 8
  20 #define CHKSUM_SZ_ADLER 4
  21
  22 int isal_logtype_driver;
  23
  24 /* Verify and set private xform parameters */
  25 int
  26 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
  27                 const struct rte_comp_xform *xform)
  28 {
  29         if (xform == NULL)
  30                 return -EINVAL;
  31
  32         /* Set compression private xform variables */
  33         if (xform->type == RTE_COMP_COMPRESS) {
  34                 /* Set private xform type - COMPRESS/DECOMPRESS */
  35                 priv_xform->type = RTE_COMP_COMPRESS;
  36
  37                 /* Set private xform algorithm */
  38                 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) {
  39                         if (xform->compress.algo == RTE_COMP_ALGO_NULL) {
  40                                 ISAL_PMD_LOG(ERR, "By-pass not supported\n");
  41                                 return -ENOTSUP;
  42                         }
  43                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
  44                         return -ENOTSUP;
  45                 }
  46                 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
  47
  48                 /* Set private xform window size, 32K supported */
  49                 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
  50                         priv_xform->compress.window_size =
  51                                         RTE_COMP_ISAL_WINDOW_SIZE;
  52                 else {
  53                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
  54                         return -ENOTSUP;
  55                 }
  56
  57                 /* Set private xform huffman type */
  58                 switch (xform->compress.deflate.huffman) {
  59                 case(RTE_COMP_HUFFMAN_DEFAULT):
  60                         priv_xform->compress.deflate.huffman =
  61                                         RTE_COMP_HUFFMAN_DEFAULT;
  62                         break;
  63                 case(RTE_COMP_HUFFMAN_FIXED):
  64                         priv_xform->compress.deflate.huffman =
  65                                         RTE_COMP_HUFFMAN_FIXED;
  66                         break;
  67                 case(RTE_COMP_HUFFMAN_DYNAMIC):
  68                         priv_xform->compress.deflate.huffman =
  69                                         RTE_COMP_HUFFMAN_DYNAMIC;
  70                         break;
  71                 default:
  72                         ISAL_PMD_LOG(ERR, "Huffman code not supported\n");
  73                         return -ENOTSUP;
  74                 }
  75
  76                 /* Set private xform checksum */
  77                 switch (xform->compress.chksum) {
  78                 /* Raw deflate by default */
  79                 case(RTE_COMP_CHECKSUM_NONE):
  80                         priv_xform->compress.chksum = IGZIP_DEFLATE;
  81                         break;
  82                 case(RTE_COMP_CHECKSUM_CRC32):
  83                         priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
  84                         break;
  85                 case(RTE_COMP_CHECKSUM_ADLER32):
  86                         priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
  87                         break;
  88                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
  89                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
  90                                         " supported\n");
  91                         return -ENOTSUP;
  92                 default:
  93                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
  94                         priv_xform->compress.chksum = IGZIP_DEFLATE;
  95                         break;
  96                 }
  97
  98                 /* Set private xform level.
  99                  * Checking compliance with compressdev API, -1 <= level => 9
 100                  */
 101                 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT ||
 102                                 xform->compress.level > RTE_COMP_LEVEL_MAX) {
 103                         ISAL_PMD_LOG(ERR, "Compression level out of range\n");
 104                         return -EINVAL;
 105                 }
 106                 /* Check for Compressdev API level 0, No compression
 107                  * not supported in ISA-L
 108                  */
 109                 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) {
 110                         ISAL_PMD_LOG(ERR, "No Compression not supported\n");
 111                         return -ENOTSUP;
 112                 }
 113                 /* If using fixed huffman code, level must be 0 */
 114                 else if (priv_xform->compress.deflate.huffman ==
 115                                 RTE_COMP_HUFFMAN_FIXED) {
 116                         ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a"
 117                                         " fixed huffman code\n");
 118                         priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO;
 119                         priv_xform->level_buffer_size =
 120                                         ISAL_DEF_LVL0_DEFAULT;
 121                 } else {
 122                         /* Mapping API levels to ISA-L levels 1,2 & 3 */
 123                         switch (xform->compress.level) {
 124                         case RTE_COMP_LEVEL_PMD_DEFAULT:
 125                                 /* Default is 1 if not using fixed huffman */
 126                                 priv_xform->compress.level =
 127                                                 RTE_COMP_ISAL_LEVEL_ONE;
 128                                 priv_xform->level_buffer_size =
 129                                                 ISAL_DEF_LVL1_DEFAULT;
 130                                 break;
 131                         case RTE_COMP_LEVEL_MIN:
 132                                 priv_xform->compress.level =
 133                                                 RTE_COMP_ISAL_LEVEL_ONE;
 134                                 priv_xform->level_buffer_size =
 135                                                 ISAL_DEF_LVL1_DEFAULT;
 136                                 break;
 137                         case RTE_COMP_ISAL_LEVEL_TWO:
 138                                 priv_xform->compress.level =
 139                                                 RTE_COMP_ISAL_LEVEL_TWO;
 140                                 priv_xform->level_buffer_size =
 141                                                 ISAL_DEF_LVL2_DEFAULT;
 142                                 break;
 143                         /* Level 3 or higher requested */
 144                         default:
 145                                 /* Check for AVX512, to use ISA-L level 3 */
 146                                 if (rte_cpu_get_flag_enabled(
 147                                                 RTE_CPUFLAG_AVX512F)) {
 148                                         priv_xform->compress.level =
 149                                                 RTE_COMP_ISAL_LEVEL_THREE;
 150                                         priv_xform->level_buffer_size =
 151                                                 ISAL_DEF_LVL3_DEFAULT;
 152                                 }
 153                                 /* Check for AVX2, to use ISA-L level 3 */
 154                                 else if (rte_cpu_get_flag_enabled(
 155                                                 RTE_CPUFLAG_AVX2)) {
 156                                         priv_xform->compress.level =
 157                                                 RTE_COMP_ISAL_LEVEL_THREE;
 158                                         priv_xform->level_buffer_size =
 159                                                 ISAL_DEF_LVL3_DEFAULT;
 160                                 } else {
 161                                         ISAL_PMD_LOG(DEBUG, "Requested ISA-L level"
 162                                                 " 3 or above; Level 3 optimized"
 163                                                 " for AVX512 & AVX2 only."
 164                                                 " level changed to 2.\n");
 165                                         priv_xform->compress.level =
 166                                                 RTE_COMP_ISAL_LEVEL_TWO;
 167                                         priv_xform->level_buffer_size =
 168                                                 ISAL_DEF_LVL2_DEFAULT;
 169                                 }
 170                         }
 171                 }
 172         }
 173
 174         /* Set decompression private xform variables */
 175         else if (xform->type == RTE_COMP_DECOMPRESS) {
 176
 177                 /* Set private xform type - COMPRESS/DECOMPRESS */
 178                 priv_xform->type = RTE_COMP_DECOMPRESS;
 179
 180                 /* Set private xform algorithm */
 181                 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) {
 182                         if (xform->decompress.algo == RTE_COMP_ALGO_NULL) {
 183                                 ISAL_PMD_LOG(ERR, "By pass not supported\n");
 184                                 return -ENOTSUP;
 185                         }
 186                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
 187                         return -ENOTSUP;
 188                 }
 189                 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
 190
 191                 /* Set private xform checksum */
 192                 switch (xform->decompress.chksum) {
 193                 /* Raw deflate by default */
 194                 case(RTE_COMP_CHECKSUM_NONE):
 195                         priv_xform->decompress.chksum = ISAL_DEFLATE;
 196                         break;
 197                 case(RTE_COMP_CHECKSUM_CRC32):
 198                         priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
 199                         break;
 200                 case(RTE_COMP_CHECKSUM_ADLER32):
 201                         priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
 202                         break;
 203                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
 204                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
 205                                         " supported\n");
 206                         return -ENOTSUP;
 207                 default:
 208                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
 209                         priv_xform->decompress.chksum = ISAL_DEFLATE;
 210                         break;
 211                 }
 212
 213                 /* Set private xform window size, 32K supported */
 214                 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
 215                         priv_xform->decompress.window_size =
 216                                         RTE_COMP_ISAL_WINDOW_SIZE;
 217                 else {
 218                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
 219                         return -ENOTSUP;
 220                 }
 221         }
 222         return 0;
 223 }
 224
 225 /* Compression using chained mbufs for input/output data */
 226 static int
 227 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
 228 {
 229         int ret;
 230         uint32_t remaining_offset;
 231         uint32_t remaining_data = op->src.length;
 232         struct rte_mbuf *src = op->m_src;
 233         struct rte_mbuf *dst = op->m_dst;
 234
 235         /* check for source/destination offset passing multiple segments
 236          * and point compression stream to input/output buffer.
 237          */
 238         remaining_offset = op->src.offset;
 239         while (remaining_offset >= src->data_len) {
 240                 remaining_offset -= src->data_len;
 241                 src = src->next;
 242         }
 243         qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
 244                         op->src.length);
 245         qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
 246                         remaining_offset);
 247
 248         remaining_offset = op->dst.offset;
 249         while (remaining_offset >= dst->data_len) {
 250                 remaining_offset -= dst->data_len;
 251                 dst = dst->next;
 252         }
 253         qp->stream->avail_out = dst->data_len - remaining_offset;
 254         qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
 255                         remaining_offset);
 256
 257         if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
 258                 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
 259                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 260                 return -1;
 261         }
 262
 263         while (qp->stream->internal_state.state != ZSTATE_END) {
 264                 /* Last segment of data */
 265                 if (remaining_data <= src->data_len)
 266                         qp->stream->end_of_stream = 1;
 267
 268                 /* Execute compression operation */
 269                 ret = isal_deflate(qp->stream);
 270
 271                 remaining_data = op->src.length - qp->stream->total_in;
 272
 273                 if (ret != COMP_OK) {
 274                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
 275                         op->status = RTE_COMP_OP_STATUS_ERROR;
 276                         return ret;
 277                 }
 278
 279                 if (qp->stream->avail_in == 0 &&
 280                                 qp->stream->total_in != op->src.length) {
 281                         if (src->next != NULL) {
 282                                 src = src->next;
 283                                 qp->stream->next_in =
 284                                                 rte_pktmbuf_mtod(src, uint8_t *);
 285                                 qp->stream->avail_in =
 286                                         RTE_MIN(remaining_data, src->data_len);
 287                         } else {
 288                                 ISAL_PMD_LOG(ERR,
 289                                 "Not enough input buffer segments\n");
 290                                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 291                                 return -1;
 292                         }
 293                 }
 294
 295                 if (qp->stream->avail_out == 0 &&
 296                                 qp->stream->internal_state.state != ZSTATE_END) {
 297                         if (dst->next != NULL) {
 298                                 dst = dst->next;
 299                                 qp->stream->next_out =
 300                                                 rte_pktmbuf_mtod(dst, uint8_t *);
 301                                 qp->stream->avail_out = dst->data_len;
 302                         } else {
 303                                 ISAL_PMD_LOG(ERR,
 304                                 "Not enough output buffer segments\n");
 305                                 op->status =
 306                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 307                                 return -1;
 308                         }
 309                 }
 310         }
 311
 312         return 0;
 313 }
 314
 315 /* Decompression using chained mbufs for input/output data */
 316 static int
 317 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
 318 {
 319         int ret;
 320         uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
 321         uint32_t remaining_data = op->src.length;
 322         struct rte_mbuf *src = op->m_src;
 323         struct rte_mbuf *dst = op->m_dst;
 324
 325         /* check for offset passing multiple segments
 326          * and point decompression state to input/output buffer
 327          */
 328         src_remaining_offset = op->src.offset;
 329         while (src_remaining_offset >= src->data_len) {
 330                 src_remaining_offset -= src->data_len;
 331                 src = src->next;
 332         }
 333         qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
 334                         op->src.length);
 335         qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
 336                         src_remaining_offset);
 337
 338         dst_remaining_offset = op->dst.offset;
 339         while (dst_remaining_offset >= dst->data_len) {
 340                 dst_remaining_offset -= dst->data_len;
 341                 dst = dst->next;
 342         }
 343         qp->state->avail_out = dst->data_len - dst_remaining_offset;
 344         qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
 345                         dst_remaining_offset);
 346
 347         while (qp->state->block_state != ISAL_BLOCK_FINISH) {
 348
 349                 ret = isal_inflate(qp->state);
 350
 351                 if (ret != ISAL_DECOMP_OK) {
 352                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 353                         op->status = RTE_COMP_OP_STATUS_ERROR;
 354                         return ret;
 355                 }
 356
 357                 /* Check for first segment, offset needs to be accounted for */
 358                 if (remaining_data == op->src.length) {
 359                         consumed_data = src->data_len - src_remaining_offset;
 360                 } else
 361                         consumed_data = src->data_len;
 362
 363                 if (qp->state->avail_in == 0
 364                                 && op->consumed != op->src.length) {
 365                         op->consumed += consumed_data;
 366                         remaining_data -= consumed_data;
 367
 368                         if (src->next != NULL) {
 369                                 src = src->next;
 370                                 qp->state->next_in =
 371                                                 rte_pktmbuf_mtod(src, uint8_t *);
 372                                 qp->state->avail_in =
 373                                         RTE_MIN(remaining_data, src->data_len);
 374                         }
 375                 }
 376
 377                 if (qp->state->avail_out == 0 &&
 378                                 qp->state->block_state != ISAL_BLOCK_FINISH) {
 379                         if (dst->next != NULL) {
 380                                 dst = dst->next;
 381                                 qp->state->next_out =
 382                                                 rte_pktmbuf_mtod(dst, uint8_t *);
 383                                 qp->state->avail_out = dst->data_len;
 384                         } else {
 385                                 ISAL_PMD_LOG(ERR,
 386                                 "Not enough output buffer segments\n");
 387                                 op->status =
 388                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 389                                 return -1;
 390                         }
 391                 }
 392         }
 393
 394         return 0;
 395 }
 396
 397 /* Stateless Compression Function */
 398 static int
 399 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 400                 struct isal_priv_xform *priv_xform)
 401 {
 402         int ret = 0;
 403         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 404
 405         /* Required due to init clearing level_buf */
 406         uint8_t *temp_level_buf = qp->stream->level_buf;
 407
 408         /* Initialize compression stream */
 409         isal_deflate_stateless_init(qp->stream);
 410
 411         qp->stream->level_buf = temp_level_buf;
 412
 413         /* Set Checksum flag */
 414         qp->stream->gzip_flag = priv_xform->compress.chksum;
 415
 416         /* Stateless operation, input will be consumed in one go */
 417         qp->stream->flush = NO_FLUSH;
 418
 419         /* set compression level & intermediate level buffer size */
 420         qp->stream->level = priv_xform->compress.level;
 421         qp->stream->level_buf_size = priv_xform->level_buffer_size;
 422
 423         /* Set op huffman code */
 424         if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
 425                 isal_deflate_set_hufftables(qp->stream, NULL,
 426                                 IGZIP_HUFFTABLE_STATIC);
 427         else if (priv_xform->compress.deflate.huffman ==
 428                         RTE_COMP_HUFFMAN_DEFAULT)
 429                 isal_deflate_set_hufftables(qp->stream, NULL,
 430                         IGZIP_HUFFTABLE_DEFAULT);
 431         /* Dynamically change the huffman code to suit the input data */
 432         else if (priv_xform->compress.deflate.huffman ==
 433                         RTE_COMP_HUFFMAN_DYNAMIC)
 434                 isal_deflate_set_hufftables(qp->stream, NULL,
 435                                 IGZIP_HUFFTABLE_DEFAULT);
 436
 437         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 438                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 439                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 440                 return -1;
 441         }
 442
 443         if (op->dst.offset >= op->m_dst->pkt_len) {
 444                 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
 445                                 " for offset provided.\n");
 446                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 447                 return -1;
 448         }
 449
 450         /* Chained mbufs */
 451         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
 452                 ret = chained_mbuf_compression(op, qp);
 453                 if (ret < 0)
 454                         return ret;
 455         } else {
 456                 /* Linear buffer */
 457                 qp->stream->end_of_stream = 1; /* All input consumed in one */
 458                 /* Point compression stream to input buffer */
 459                 qp->stream->avail_in = op->src.length;
 460                 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
 461                                 uint8_t *, op->src.offset);
 462
 463                 /*  Point compression stream to output buffer */
 464                 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
 465                 qp->stream->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
 466                                 uint8_t *, op->dst.offset);
 467
 468                 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
 469                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
 470                                         " buffers\n");
 471                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 472                         return -1;
 473                 }
 474
 475                 /* Execute compression operation */
 476                 ret =  isal_deflate_stateless(qp->stream);
 477
 478                 /* Check that output buffer did not run out of space */
 479                 if (ret == STATELESS_OVERFLOW) {
 480                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
 481                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 482                         return ret;
 483                 }
 484
 485                 /* Check that input buffer has been fully consumed */
 486                 if (qp->stream->avail_in != (uint32_t)0) {
 487                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
 488                                         " entirely\n");
 489                         op->status = RTE_COMP_OP_STATUS_ERROR;
 490                         return -1;
 491                 }
 492
 493                 if (ret != COMP_OK) {
 494                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
 495                         op->status = RTE_COMP_OP_STATUS_ERROR;
 496                         return ret;
 497                 }
 498         }
 499
 500         op->consumed = qp->stream->total_in;
 501         if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
 502                 op->produced = qp->stream->total_out;
 503         } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
 504                 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
 505                 op->output_chksum = qp->stream->internal_state.crc + 1;
 506         } else {
 507                 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
 508                 op->output_chksum = qp->stream->internal_state.crc;
 509         }
 510
 511         isal_deflate_reset(qp->stream);
 512
 513         return ret;
 514 }
 515
 516 /* Stateless Decompression Function */
 517 static int
 518 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 519                 struct isal_priv_xform *priv_xform)
 520 {
 521         int ret = 0;
 522
 523         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 524
 525         /* Initialize decompression state */
 526         isal_inflate_init(qp->state);
 527
 528         /* Set Checksum flag */
 529         qp->state->crc_flag = priv_xform->decompress.chksum;
 530
 531         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 532                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 533                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 534                 return -1;
 535         }
 536
 537         if (op->dst.offset >= op->m_dst->pkt_len) {
 538                 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
 539                                 "offset provided.\n");
 540                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 541                 return -1;
 542         }
 543
 544         /* Chained mbufs */
 545         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
 546                 ret = chained_mbuf_decompression(op, qp);
 547                 if (ret !=  0)
 548                         return ret;
 549         } else {
 550                 /* Linear buffer */
 551                 /* Point decompression state to input buffer */
 552                 qp->state->avail_in = op->src.length;
 553                 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
 554                                 uint8_t *, op->src.offset);
 555
 556                 /* Point decompression state to output buffer */
 557                 qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
 558                 qp->state->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
 559                                 uint8_t *, op->dst.offset);
 560
 561                 if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
 562                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
 563                                         " buffers\n");
 564                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 565                         return -1;
 566                 }
 567
 568                 /* Execute decompression operation */
 569                 ret = isal_inflate_stateless(qp->state);
 570
 571                 if (ret == ISAL_OUT_OVERFLOW) {
 572                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
 573                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 574                         return ret;
 575                 }
 576
 577                 /* Check that input buffer has been fully consumed */
 578                 if (qp->state->avail_in != (uint32_t)0) {
 579                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
 580                                         " entirely\n");
 581                         op->status = RTE_COMP_OP_STATUS_ERROR;
 582                         return -1;
 583                 }
 584
 585                 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
 586                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 587                         op->status = RTE_COMP_OP_STATUS_ERROR;
 588                         return ret;
 589                 }
 590                 op->consumed = op->src.length - qp->state->avail_in;
 591         }
 592         op->produced = qp->state->total_out;
 593         op->output_chksum = qp->state->crc;
 594
 595         isal_inflate_reset(qp->state);
 596
 597         return ret;
 598 }
 599
 600 /* Process compression/decompression operation */
 601 static int
 602 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
 603                 struct isal_priv_xform *priv_xform)
 604 {
 605         switch (priv_xform->type) {
 606         case RTE_COMP_COMPRESS:
 607                 process_isal_deflate(op, qp, priv_xform);
 608                 break;
 609         case RTE_COMP_DECOMPRESS:
 610                 process_isal_inflate(op, qp, priv_xform);
 611                 break;
 612         default:
 613                 ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
 614                 return -ENOTSUP;
 615         }
 616         return 0;
 617 }
 618
 619 /* Enqueue burst */
 620 static uint16_t
 621 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops,
 622                         uint16_t nb_ops)
 623 {
 624         struct isal_comp_qp *qp = queue_pair;
 625         uint16_t i;
 626         int retval;
 627         int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops);
 628
 629         for (i = 0; i < num_enq; i++) {
 630                 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) {
 631                         ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 632                         ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n");
 633                         qp->qp_stats.enqueue_err_count++;
 634                         continue;
 635                 }
 636                 retval = process_op(qp, ops[i], ops[i]->private_xform);
 637                 if (unlikely(retval < 0) ||
 638                                 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 639                         qp->qp_stats.enqueue_err_count++;
 640                 }
 641         }
 642
 643         retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops,
 644                         num_enq, NULL);
 645         qp->num_free_elements -= retval;
 646         qp->qp_stats.enqueued_count += retval;
 647
 648         return retval;
 649 }
 650
 651 /* Dequeue burst */
 652 static uint16_t
 653 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
 654                 uint16_t nb_ops)
 655 {
 656         struct isal_comp_qp *qp = queue_pair;
 657         uint16_t nb_dequeued;
 658
 659         nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops,
 660                         nb_ops, NULL);
 661         qp->num_free_elements += nb_dequeued;
 662         qp->qp_stats.dequeued_count += nb_dequeued;
 663
 664         return nb_dequeued;
 665 }
 666
 667 /* Create ISA-L compression device */
 668 static int
 669 compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
 670                 struct rte_compressdev_pmd_init_params *init_params)
 671 {
 672         struct rte_compressdev *dev;
 673
 674         dev = rte_compressdev_pmd_create(name, &vdev->device,
 675                         sizeof(struct isal_comp_private), init_params);
 676         if (dev == NULL) {
 677                 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev");
 678                 return -EFAULT;
 679         }
 680
 681         dev->dev_ops = isal_compress_pmd_ops;
 682
 683         /* register rx/tx burst functions for data path */
 684         dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
 685         dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
 686
 687         return 0;
 688 }
 689
 690 /** Remove compression device */
 691 static int
 692 compdev_isal_remove_dev(struct rte_vdev_device *vdev)
 693 {
 694         struct rte_compressdev *compdev;
 695         const char *name;
 696
 697         name = rte_vdev_device_name(vdev);
 698         if (name == NULL)
 699                 return -EINVAL;
 700
 701         compdev = rte_compressdev_pmd_get_named_dev(name);
 702         if (compdev == NULL)
 703                 return -ENODEV;
 704
 705         return rte_compressdev_pmd_destroy(compdev);
 706 }
 707
 708 /** Initialise ISA-L compression device */
 709 static int
 710 compdev_isal_probe(struct rte_vdev_device *dev)
 711 {
 712         struct rte_compressdev_pmd_init_params init_params = {
 713                 "",
 714                 rte_socket_id(),
 715         };
 716         const char *name, *args;
 717         int retval;
 718
 719         name = rte_vdev_device_name(dev);
 720         if (name == NULL)
 721                 return -EINVAL;
 722
 723         args = rte_vdev_device_args(dev);
 724
 725         retval = rte_compressdev_pmd_parse_input_args(&init_params, args);
 726         if (retval) {
 727                 ISAL_PMD_LOG(ERR,
 728                         "Failed to parse initialisation arguments[%s]\n", args);
 729                 return -EINVAL;
 730         }
 731
 732         return compdev_isal_create(name, dev, &init_params);
 733 }
 734
 735 static struct rte_vdev_driver compdev_isal_pmd_drv = {
 736         .probe = compdev_isal_probe,
 737         .remove = compdev_isal_remove_dev,
 738 };
 739
 740 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
 741 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
 742         "socket_id=<int>");
 743
 744 RTE_INIT(isal_init_log)
 745 {
 746         isal_logtype_driver = rte_log_register("pmd.compress.isal");
 747         if (isal_logtype_driver >= 0)
 748                 rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO);
 749 }