drivers/compress/isal/isal_compress_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <isa-l.h>
   5
   6 #include <rte_bus_vdev.h>
   7 #include <rte_common.h>
   8 #include <rte_cpuflags.h>
   9 #include <rte_malloc.h>
  10 #include <rte_mbuf.h>
  11 #include <rte_compressdev_pmd.h>
  12
  13 #include "isal_compress_pmd_private.h"
  14
  15 #define RTE_COMP_ISAL_WINDOW_SIZE 15
  16 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */
  17 #define RTE_COMP_ISAL_LEVEL_ONE 1
  18 #define RTE_COMP_ISAL_LEVEL_TWO 2
  19 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
  20 #define CHKSUM_SZ_CRC 8
  21 #define CHKSUM_SZ_ADLER 4
  22
  23 #define STRINGIFY(s) #s
  24 #define ISAL_TOSTRING(maj, min, patch) \
  25         STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch)
  26 #define ISAL_VERSION_STRING \
  27         ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION)
  28
  29 /* Verify and set private xform parameters */
  30 int
  31 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
  32                 const struct rte_comp_xform *xform)
  33 {
  34         if (xform == NULL)
  35                 return -EINVAL;
  36
  37         /* Set compression private xform variables */
  38         if (xform->type == RTE_COMP_COMPRESS) {
  39                 /* Set private xform type - COMPRESS/DECOMPRESS */
  40                 priv_xform->type = RTE_COMP_COMPRESS;
  41
  42                 /* Set private xform algorithm */
  43                 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) {
  44                         if (xform->compress.algo == RTE_COMP_ALGO_NULL) {
  45                                 ISAL_PMD_LOG(ERR, "By-pass not supported\n");
  46                                 return -ENOTSUP;
  47                         }
  48                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
  49                         return -ENOTSUP;
  50                 }
  51                 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
  52
  53                 /* Set private xform window size, 32K supported */
  54                 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
  55                         priv_xform->compress.window_size =
  56                                         RTE_COMP_ISAL_WINDOW_SIZE;
  57                 else {
  58                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
  59                         return -ENOTSUP;
  60                 }
  61
  62                 /* Set private xform huffman type */
  63                 switch (xform->compress.deflate.huffman) {
  64                 case(RTE_COMP_HUFFMAN_DEFAULT):
  65                         priv_xform->compress.deflate.huffman =
  66                                         RTE_COMP_HUFFMAN_DEFAULT;
  67                         break;
  68                 case(RTE_COMP_HUFFMAN_FIXED):
  69                         priv_xform->compress.deflate.huffman =
  70                                         RTE_COMP_HUFFMAN_FIXED;
  71                         break;
  72                 case(RTE_COMP_HUFFMAN_DYNAMIC):
  73                         priv_xform->compress.deflate.huffman =
  74                                         RTE_COMP_HUFFMAN_DYNAMIC;
  75                         break;
  76                 default:
  77                         ISAL_PMD_LOG(ERR, "Huffman code not supported\n");
  78                         return -ENOTSUP;
  79                 }
  80
  81                 /* Set private xform checksum */
  82                 switch (xform->compress.chksum) {
  83                 /* Raw deflate by default */
  84                 case(RTE_COMP_CHECKSUM_NONE):
  85                         priv_xform->compress.chksum = IGZIP_DEFLATE;
  86                         break;
  87                 case(RTE_COMP_CHECKSUM_CRC32):
  88                         priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
  89                         break;
  90                 case(RTE_COMP_CHECKSUM_ADLER32):
  91                         priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
  92                         break;
  93                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
  94                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
  95                                         " supported\n");
  96                         return -ENOTSUP;
  97                 default:
  98                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
  99                         priv_xform->compress.chksum = IGZIP_DEFLATE;
 100                         break;
 101                 }
 102
 103                 /* Set private xform level.
 104                  * Checking compliance with compressdev API, -1 <= level => 9
 105                  */
 106                 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT ||
 107                                 xform->compress.level > RTE_COMP_LEVEL_MAX) {
 108                         ISAL_PMD_LOG(ERR, "Compression level out of range\n");
 109                         return -EINVAL;
 110                 }
 111                 /* Check for Compressdev API level 0, No compression
 112                  * not supported in ISA-L
 113                  */
 114                 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) {
 115                         ISAL_PMD_LOG(ERR, "No Compression not supported\n");
 116                         return -ENOTSUP;
 117                 }
 118                 /* If using fixed huffman code, level must be 0 */
 119                 else if (priv_xform->compress.deflate.huffman ==
 120                                 RTE_COMP_HUFFMAN_FIXED) {
 121                         ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a"
 122                                         " fixed huffman code\n");
 123                         priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO;
 124                         priv_xform->level_buffer_size =
 125                                         ISAL_DEF_LVL0_DEFAULT;
 126                 } else {
 127                         /* Mapping API levels to ISA-L levels 1,2 & 3 */
 128                         switch (xform->compress.level) {
 129                         case RTE_COMP_LEVEL_PMD_DEFAULT:
 130                                 /* Default is 1 if not using fixed huffman */
 131                                 priv_xform->compress.level =
 132                                                 RTE_COMP_ISAL_LEVEL_ONE;
 133                                 priv_xform->level_buffer_size =
 134                                                 ISAL_DEF_LVL1_DEFAULT;
 135                                 break;
 136                         case RTE_COMP_LEVEL_MIN:
 137                                 priv_xform->compress.level =
 138                                                 RTE_COMP_ISAL_LEVEL_ONE;
 139                                 priv_xform->level_buffer_size =
 140                                                 ISAL_DEF_LVL1_DEFAULT;
 141                                 break;
 142                         case RTE_COMP_ISAL_LEVEL_TWO:
 143                                 priv_xform->compress.level =
 144                                                 RTE_COMP_ISAL_LEVEL_TWO;
 145                                 priv_xform->level_buffer_size =
 146                                                 ISAL_DEF_LVL2_DEFAULT;
 147                                 break;
 148                         /* Level 3 or higher requested */
 149                         default:
 150 #ifdef RTE_ARCH_X86
 151                                 /* Check for AVX512, to use ISA-L level 3 */
 152                                 if (rte_cpu_get_flag_enabled(
 153                                                 RTE_CPUFLAG_AVX512F)) {
 154                                         priv_xform->compress.level =
 155                                                 RTE_COMP_ISAL_LEVEL_THREE;
 156                                         priv_xform->level_buffer_size =
 157                                                 ISAL_DEF_LVL3_DEFAULT;
 158                                 }
 159                                 /* Check for AVX2, to use ISA-L level 3 */
 160                                 else if (rte_cpu_get_flag_enabled(
 161                                                 RTE_CPUFLAG_AVX2)) {
 162                                         priv_xform->compress.level =
 163                                                 RTE_COMP_ISAL_LEVEL_THREE;
 164                                         priv_xform->level_buffer_size =
 165                                                 ISAL_DEF_LVL3_DEFAULT;
 166                                 } else
 167 #endif
 168                                 {
 169                                         ISAL_PMD_LOG(DEBUG, "Requested ISA-L level"
 170                                                 " 3 or above; Level 3 optimized"
 171                                                 " for AVX512 & AVX2 only."
 172                                                 " level changed to 2.\n");
 173                                         priv_xform->compress.level =
 174                                                 RTE_COMP_ISAL_LEVEL_TWO;
 175                                         priv_xform->level_buffer_size =
 176                                                 ISAL_DEF_LVL2_DEFAULT;
 177                                 }
 178                         }
 179                 }
 180         }
 181
 182         /* Set decompression private xform variables */
 183         else if (xform->type == RTE_COMP_DECOMPRESS) {
 184
 185                 /* Set private xform type - COMPRESS/DECOMPRESS */
 186                 priv_xform->type = RTE_COMP_DECOMPRESS;
 187
 188                 /* Set private xform algorithm */
 189                 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) {
 190                         if (xform->decompress.algo == RTE_COMP_ALGO_NULL) {
 191                                 ISAL_PMD_LOG(ERR, "By pass not supported\n");
 192                                 return -ENOTSUP;
 193                         }
 194                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
 195                         return -ENOTSUP;
 196                 }
 197                 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
 198
 199                 /* Set private xform checksum */
 200                 switch (xform->decompress.chksum) {
 201                 /* Raw deflate by default */
 202                 case(RTE_COMP_CHECKSUM_NONE):
 203                         priv_xform->decompress.chksum = ISAL_DEFLATE;
 204                         break;
 205                 case(RTE_COMP_CHECKSUM_CRC32):
 206                         priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
 207                         break;
 208                 case(RTE_COMP_CHECKSUM_ADLER32):
 209                         priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
 210                         break;
 211                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
 212                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
 213                                         " supported\n");
 214                         return -ENOTSUP;
 215                 default:
 216                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
 217                         priv_xform->decompress.chksum = ISAL_DEFLATE;
 218                         break;
 219                 }
 220
 221                 /* Set private xform window size, 32K supported */
 222                 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
 223                         priv_xform->decompress.window_size =
 224                                         RTE_COMP_ISAL_WINDOW_SIZE;
 225                 else {
 226                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
 227                         return -ENOTSUP;
 228                 }
 229         }
 230         return 0;
 231 }
 232
 233 /* Compression using chained mbufs for input/output data */
 234 static int
 235 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
 236 {
 237         int ret;
 238         uint32_t remaining_offset;
 239         uint32_t remaining_data = op->src.length;
 240         struct rte_mbuf *src = op->m_src;
 241         struct rte_mbuf *dst = op->m_dst;
 242
 243         /* check for source/destination offset passing multiple segments
 244          * and point compression stream to input/output buffer.
 245          */
 246         remaining_offset = op->src.offset;
 247         while (remaining_offset >= src->data_len) {
 248                 remaining_offset -= src->data_len;
 249                 src = src->next;
 250         }
 251         qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
 252                         op->src.length);
 253         qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
 254                         remaining_offset);
 255
 256         remaining_offset = op->dst.offset;
 257         while (remaining_offset >= dst->data_len) {
 258                 remaining_offset -= dst->data_len;
 259                 dst = dst->next;
 260         }
 261         qp->stream->avail_out = dst->data_len - remaining_offset;
 262         qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
 263                         remaining_offset);
 264
 265         if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
 266                 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
 267                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 268                 return -1;
 269         }
 270
 271         while (qp->stream->internal_state.state != ZSTATE_END) {
 272                 /* Last segment of data */
 273                 if (remaining_data <= src->data_len)
 274                         qp->stream->end_of_stream = 1;
 275
 276                 /* Execute compression operation */
 277                 ret = isal_deflate(qp->stream);
 278
 279                 remaining_data = op->src.length - qp->stream->total_in;
 280
 281                 if (ret != COMP_OK) {
 282                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
 283                         op->status = RTE_COMP_OP_STATUS_ERROR;
 284                         return ret;
 285                 }
 286
 287                 if (qp->stream->avail_in == 0 &&
 288                                 qp->stream->total_in != op->src.length) {
 289                         if (src->next != NULL) {
 290                                 src = src->next;
 291                                 qp->stream->next_in =
 292                                                 rte_pktmbuf_mtod(src, uint8_t *);
 293                                 qp->stream->avail_in =
 294                                         RTE_MIN(remaining_data, src->data_len);
 295                         } else {
 296                                 ISAL_PMD_LOG(ERR,
 297                                 "Not enough input buffer segments\n");
 298                                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 299                                 return -1;
 300                         }
 301                 }
 302
 303                 if (qp->stream->avail_out == 0 &&
 304                                 qp->stream->internal_state.state != ZSTATE_END) {
 305                         if (dst->next != NULL) {
 306                                 dst = dst->next;
 307                                 qp->stream->next_out =
 308                                                 rte_pktmbuf_mtod(dst, uint8_t *);
 309                                 qp->stream->avail_out = dst->data_len;
 310                         } else {
 311                                 ISAL_PMD_LOG(ERR,
 312                                 "Not enough output buffer segments\n");
 313                                 op->status =
 314                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 315                                 return -1;
 316                         }
 317                 }
 318         }
 319
 320         return 0;
 321 }
 322
 323 /* Decompression using chained mbufs for input/output data */
 324 static int
 325 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
 326 {
 327         int ret;
 328         uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
 329         uint32_t remaining_data = op->src.length;
 330         struct rte_mbuf *src = op->m_src;
 331         struct rte_mbuf *dst = op->m_dst;
 332
 333         /* check for offset passing multiple segments
 334          * and point decompression state to input/output buffer
 335          */
 336         src_remaining_offset = op->src.offset;
 337         while (src_remaining_offset >= src->data_len) {
 338                 src_remaining_offset -= src->data_len;
 339                 src = src->next;
 340         }
 341         qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
 342                         op->src.length);
 343         qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
 344                         src_remaining_offset);
 345
 346         dst_remaining_offset = op->dst.offset;
 347         while (dst_remaining_offset >= dst->data_len) {
 348                 dst_remaining_offset -= dst->data_len;
 349                 dst = dst->next;
 350         }
 351         qp->state->avail_out = dst->data_len - dst_remaining_offset;
 352         qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
 353                         dst_remaining_offset);
 354
 355         while (qp->state->block_state != ISAL_BLOCK_FINISH) {
 356
 357                 ret = isal_inflate(qp->state);
 358
 359                 /* Check for first segment, offset needs to be accounted for */
 360                 if (remaining_data == op->src.length) {
 361                         consumed_data = src->data_len - src_remaining_offset;
 362                 } else
 363                         consumed_data = src->data_len;
 364
 365                 if (qp->state->avail_in == 0
 366                                 && op->consumed != op->src.length) {
 367                         op->consumed += consumed_data;
 368                         remaining_data -= consumed_data;
 369
 370                         if (src->next != NULL) {
 371                                 src = src->next;
 372                                 qp->state->next_in =
 373                                                 rte_pktmbuf_mtod(src, uint8_t *);
 374                                 qp->state->avail_in =
 375                                         RTE_MIN(remaining_data, src->data_len);
 376                         }
 377                 }
 378
 379                 if (ret == ISAL_OUT_OVERFLOW) {
 380                         ISAL_PMD_LOG(ERR, "Decompression operation ran "
 381                                 "out of space, but can be recovered.\n%d bytes "
 382                                 "consumed\t%d bytes produced\n",
 383                                 consumed_data, qp->state->total_out);
 384                                 op->status =
 385                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
 386                         return ret;
 387                 } else if (ret < 0) {
 388                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 389                         op->status = RTE_COMP_OP_STATUS_ERROR;
 390                         return ret;
 391                 }
 392
 393                 if (qp->state->avail_out == 0 &&
 394                                 qp->state->block_state != ISAL_BLOCK_FINISH) {
 395                         if (dst->next != NULL) {
 396                                 dst = dst->next;
 397                                 qp->state->next_out =
 398                                                 rte_pktmbuf_mtod(dst, uint8_t *);
 399                                 qp->state->avail_out = dst->data_len;
 400                         } else {
 401                                 ISAL_PMD_LOG(ERR,
 402                                 "Not enough output buffer segments\n");
 403                                 op->status =
 404                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 405                                 return -1;
 406                         }
 407                 }
 408         }
 409
 410         return 0;
 411 }
 412
 413 /* Stateless Compression Function */
 414 static int
 415 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 416                 struct isal_priv_xform *priv_xform)
 417 {
 418         int ret = 0;
 419         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 420
 421         /* Required due to init clearing level_buf */
 422         uint8_t *temp_level_buf = qp->stream->level_buf;
 423
 424         /* Initialize compression stream */
 425         isal_deflate_init(qp->stream);
 426
 427         qp->stream->level_buf = temp_level_buf;
 428
 429         /* Set Checksum flag */
 430         qp->stream->gzip_flag = priv_xform->compress.chksum;
 431
 432         /* Stateless operation, input will be consumed in one go */
 433         qp->stream->flush = NO_FLUSH;
 434
 435         /* set compression level & intermediate level buffer size */
 436         qp->stream->level = priv_xform->compress.level;
 437         qp->stream->level_buf_size = priv_xform->level_buffer_size;
 438
 439         /* Set op huffman code */
 440         if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
 441                 isal_deflate_set_hufftables(qp->stream, NULL,
 442                                 IGZIP_HUFFTABLE_STATIC);
 443         else if (priv_xform->compress.deflate.huffman ==
 444                         RTE_COMP_HUFFMAN_DEFAULT)
 445                 isal_deflate_set_hufftables(qp->stream, NULL,
 446                         IGZIP_HUFFTABLE_DEFAULT);
 447         /* Dynamically change the huffman code to suit the input data */
 448         else if (priv_xform->compress.deflate.huffman ==
 449                         RTE_COMP_HUFFMAN_DYNAMIC)
 450                 isal_deflate_set_hufftables(qp->stream, NULL,
 451                                 IGZIP_HUFFTABLE_DEFAULT);
 452
 453         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 454                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 455                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 456                 return -1;
 457         }
 458
 459         if (op->dst.offset >= op->m_dst->pkt_len) {
 460                 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
 461                                 " for offset provided.\n");
 462                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 463                 return -1;
 464         }
 465
 466         /* Chained mbufs */
 467         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
 468                 ret = chained_mbuf_compression(op, qp);
 469                 if (ret < 0)
 470                         return ret;
 471         } else {
 472                 /* Linear buffer */
 473                 qp->stream->end_of_stream = 1; /* All input consumed in one */
 474                 /* Point compression stream to input buffer */
 475                 qp->stream->avail_in = op->src.length;
 476                 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
 477                                 uint8_t *, op->src.offset);
 478
 479                 /*  Point compression stream to output buffer */
 480                 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
 481                 qp->stream->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
 482                                 uint8_t *, op->dst.offset);
 483
 484                 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
 485                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
 486                                         " buffers\n");
 487                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 488                         return -1;
 489                 }
 490
 491                 /* Execute compression operation */
 492                 ret =  isal_deflate_stateless(qp->stream);
 493
 494                 /* Check that output buffer did not run out of space */
 495                 if (ret == STATELESS_OVERFLOW) {
 496                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
 497                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 498                         return ret;
 499                 }
 500
 501                 /* Check that input buffer has been fully consumed */
 502                 if (qp->stream->avail_in != (uint32_t)0) {
 503                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
 504                                         " entirely\n");
 505                         op->status = RTE_COMP_OP_STATUS_ERROR;
 506                         return -1;
 507                 }
 508
 509                 if (ret != COMP_OK) {
 510                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
 511                         op->status = RTE_COMP_OP_STATUS_ERROR;
 512                         return ret;
 513                 }
 514         }
 515
 516         op->consumed = qp->stream->total_in;
 517         if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
 518                 op->produced = qp->stream->total_out;
 519         } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
 520                 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
 521                 op->output_chksum = qp->stream->internal_state.crc + 1;
 522         } else {
 523                 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
 524                 op->output_chksum = qp->stream->internal_state.crc;
 525         }
 526
 527         return ret;
 528 }
 529
 530 /* Stateless Decompression Function */
 531 static int
 532 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 533                 struct isal_priv_xform *priv_xform)
 534 {
 535         int ret = 0;
 536
 537         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 538
 539         /* Initialize decompression state */
 540         isal_inflate_init(qp->state);
 541
 542         /* Set Checksum flag */
 543         qp->state->crc_flag = priv_xform->decompress.chksum;
 544
 545         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 546                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 547                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 548                 return -1;
 549         }
 550
 551         if (op->dst.offset >= op->m_dst->pkt_len) {
 552                 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
 553                                 "offset provided.\n");
 554                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 555                 return -1;
 556         }
 557
 558         /* Chained mbufs */
 559         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
 560                 ret = chained_mbuf_decompression(op, qp);
 561                 if (ret !=  0)
 562                         return ret;
 563         } else {
 564                 /* Linear buffer */
 565                 /* Point decompression state to input buffer */
 566                 qp->state->avail_in = op->src.length;
 567                 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
 568                                 uint8_t *, op->src.offset);
 569
 570                 /* Point decompression state to output buffer */
 571                 qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
 572                 qp->state->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
 573                                 uint8_t *, op->dst.offset);
 574
 575                 if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
 576                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
 577                                         " buffers\n");
 578                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 579                         return -1;
 580                 }
 581
 582                 /* Execute decompression operation */
 583                 ret = isal_inflate_stateless(qp->state);
 584
 585                 if (ret == ISAL_OUT_OVERFLOW) {
 586                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
 587                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 588                         return ret;
 589                 }
 590
 591                 /* Check that input buffer has been fully consumed */
 592                 if (qp->state->avail_in != (uint32_t)0) {
 593                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
 594                                         " entirely\n");
 595                         op->status = RTE_COMP_OP_STATUS_ERROR;
 596                         return -1;
 597                 }
 598
 599                 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
 600                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 601                         op->status = RTE_COMP_OP_STATUS_ERROR;
 602                         return ret;
 603                 }
 604                 op->consumed = op->src.length - qp->state->avail_in;
 605         }
 606         op->produced = qp->state->total_out;
 607         op->output_chksum = qp->state->crc;
 608
 609         return ret;
 610 }
 611
 612 /* Process compression/decompression operation */
 613 static int
 614 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
 615                 struct isal_priv_xform *priv_xform)
 616 {
 617         switch (priv_xform->type) {
 618         case RTE_COMP_COMPRESS:
 619                 process_isal_deflate(op, qp, priv_xform);
 620                 break;
 621         case RTE_COMP_DECOMPRESS:
 622                 process_isal_inflate(op, qp, priv_xform);
 623                 break;
 624         default:
 625                 ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
 626                 return -ENOTSUP;
 627         }
 628         return 0;
 629 }
 630
 631 /* Enqueue burst */
 632 static uint16_t
 633 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops,
 634                         uint16_t nb_ops)
 635 {
 636         struct isal_comp_qp *qp = queue_pair;
 637         uint16_t i;
 638         int retval;
 639         int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops);
 640
 641         for (i = 0; i < num_enq; i++) {
 642                 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) {
 643                         ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 644                         ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n");
 645                         qp->qp_stats.enqueue_err_count++;
 646                         continue;
 647                 }
 648                 retval = process_op(qp, ops[i], ops[i]->private_xform);
 649                 if (unlikely(retval < 0) ||
 650                                 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 651                         qp->qp_stats.enqueue_err_count++;
 652                 }
 653         }
 654
 655         retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops,
 656                         num_enq, NULL);
 657         qp->num_free_elements -= retval;
 658         qp->qp_stats.enqueued_count += retval;
 659
 660         return retval;
 661 }
 662
 663 /* Dequeue burst */
 664 static uint16_t
 665 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
 666                 uint16_t nb_ops)
 667 {
 668         struct isal_comp_qp *qp = queue_pair;
 669         uint16_t nb_dequeued;
 670
 671         nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops,
 672                         nb_ops, NULL);
 673         qp->num_free_elements += nb_dequeued;
 674         qp->qp_stats.dequeued_count += nb_dequeued;
 675
 676         return nb_dequeued;
 677 }
 678
 679 /* Create ISA-L compression device */
 680 static int
 681 compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
 682                 struct rte_compressdev_pmd_init_params *init_params)
 683 {
 684         struct rte_compressdev *dev;
 685
 686         dev = rte_compressdev_pmd_create(name, &vdev->device,
 687                         sizeof(struct isal_comp_private), init_params);
 688         if (dev == NULL) {
 689                 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev");
 690                 return -EFAULT;
 691         }
 692
 693         dev->dev_ops = isal_compress_pmd_ops;
 694
 695         /* register rx/tx burst functions for data path */
 696         dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
 697         dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
 698
 699         ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING);
 700
 701         return 0;
 702 }
 703
 704 /** Remove compression device */
 705 static int
 706 compdev_isal_remove_dev(struct rte_vdev_device *vdev)
 707 {
 708         struct rte_compressdev *compdev;
 709         const char *name;
 710
 711         name = rte_vdev_device_name(vdev);
 712         if (name == NULL)
 713                 return -EINVAL;
 714
 715         compdev = rte_compressdev_pmd_get_named_dev(name);
 716         if (compdev == NULL)
 717                 return -ENODEV;
 718
 719         return rte_compressdev_pmd_destroy(compdev);
 720 }
 721
 722 /** Initialise ISA-L compression device */
 723 static int
 724 compdev_isal_probe(struct rte_vdev_device *dev)
 725 {
 726         struct rte_compressdev_pmd_init_params init_params = {
 727                 "",
 728                 rte_socket_id(),
 729         };
 730         const char *name, *args;
 731         int retval;
 732
 733         name = rte_vdev_device_name(dev);
 734         if (name == NULL)
 735                 return -EINVAL;
 736
 737         args = rte_vdev_device_args(dev);
 738
 739         retval = rte_compressdev_pmd_parse_input_args(&init_params, args);
 740         if (retval) {
 741                 ISAL_PMD_LOG(ERR,
 742                         "Failed to parse initialisation arguments[%s]\n", args);
 743                 return -EINVAL;
 744         }
 745
 746         return compdev_isal_create(name, dev, &init_params);
 747 }
 748
 749 static struct rte_vdev_driver compdev_isal_pmd_drv = {
 750         .probe = compdev_isal_probe,
 751         .remove = compdev_isal_remove_dev,
 752 };
 753
 754 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
 755 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
 756         "socket_id=<int>");
 757 RTE_LOG_REGISTER_DEFAULT(isal_logtype_driver, INFO);