drivers/compress/isal/isal_compress_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <isa-l.h>
   5
   6 #include <rte_bus_vdev.h>
   7 #include <rte_common.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mbuf.h>
  10 #include <rte_compressdev_pmd.h>
  11
  12 #include "isal_compress_pmd_private.h"
  13
  14 #define RTE_COMP_ISAL_WINDOW_SIZE 15
  15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */
  16 #define RTE_COMP_ISAL_LEVEL_ONE 1
  17 #define RTE_COMP_ISAL_LEVEL_TWO 2
  18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
  19 #define CHKSUM_SZ_CRC 8
  20 #define CHKSUM_SZ_ADLER 4
  21
  22 #define STRINGIFY(s) #s
  23 #define ISAL_TOSTRING(maj, min, patch) \
  24         STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch)
  25 #define ISAL_VERSION_STRING \
  26         ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION)
  27
  28 /* Verify and set private xform parameters */
  29 int
  30 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
  31                 const struct rte_comp_xform *xform)
  32 {
  33         if (xform == NULL)
  34                 return -EINVAL;
  35
  36         /* Set compression private xform variables */
  37         if (xform->type == RTE_COMP_COMPRESS) {
  38                 /* Set private xform type - COMPRESS/DECOMPRESS */
  39                 priv_xform->type = RTE_COMP_COMPRESS;
  40
  41                 /* Set private xform algorithm */
  42                 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) {
  43                         if (xform->compress.algo == RTE_COMP_ALGO_NULL) {
  44                                 ISAL_PMD_LOG(ERR, "By-pass not supported\n");
  45                                 return -ENOTSUP;
  46                         }
  47                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
  48                         return -ENOTSUP;
  49                 }
  50                 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
  51
  52                 /* Set private xform window size, 32K supported */
  53                 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
  54                         priv_xform->compress.window_size =
  55                                         RTE_COMP_ISAL_WINDOW_SIZE;
  56                 else {
  57                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
  58                         return -ENOTSUP;
  59                 }
  60
  61                 /* Set private xform huffman type */
  62                 switch (xform->compress.deflate.huffman) {
  63                 case(RTE_COMP_HUFFMAN_DEFAULT):
  64                         priv_xform->compress.deflate.huffman =
  65                                         RTE_COMP_HUFFMAN_DEFAULT;
  66                         break;
  67                 case(RTE_COMP_HUFFMAN_FIXED):
  68                         priv_xform->compress.deflate.huffman =
  69                                         RTE_COMP_HUFFMAN_FIXED;
  70                         break;
  71                 case(RTE_COMP_HUFFMAN_DYNAMIC):
  72                         priv_xform->compress.deflate.huffman =
  73                                         RTE_COMP_HUFFMAN_DYNAMIC;
  74                         break;
  75                 default:
  76                         ISAL_PMD_LOG(ERR, "Huffman code not supported\n");
  77                         return -ENOTSUP;
  78                 }
  79
  80                 /* Set private xform checksum */
  81                 switch (xform->compress.chksum) {
  82                 /* Raw deflate by default */
  83                 case(RTE_COMP_CHECKSUM_NONE):
  84                         priv_xform->compress.chksum = IGZIP_DEFLATE;
  85                         break;
  86                 case(RTE_COMP_CHECKSUM_CRC32):
  87                         priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
  88                         break;
  89                 case(RTE_COMP_CHECKSUM_ADLER32):
  90                         priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
  91                         break;
  92                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
  93                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
  94                                         " supported\n");
  95                         return -ENOTSUP;
  96                 default:
  97                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
  98                         priv_xform->compress.chksum = IGZIP_DEFLATE;
  99                         break;
 100                 }
 101
 102                 /* Set private xform level.
 103                  * Checking compliance with compressdev API, -1 <= level => 9
 104                  */
 105                 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT ||
 106                                 xform->compress.level > RTE_COMP_LEVEL_MAX) {
 107                         ISAL_PMD_LOG(ERR, "Compression level out of range\n");
 108                         return -EINVAL;
 109                 }
 110                 /* Check for Compressdev API level 0, No compression
 111                  * not supported in ISA-L
 112                  */
 113                 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) {
 114                         ISAL_PMD_LOG(ERR, "No Compression not supported\n");
 115                         return -ENOTSUP;
 116                 }
 117                 /* If using fixed huffman code, level must be 0 */
 118                 else if (priv_xform->compress.deflate.huffman ==
 119                                 RTE_COMP_HUFFMAN_FIXED) {
 120                         ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a"
 121                                         " fixed huffman code\n");
 122                         priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO;
 123                         priv_xform->level_buffer_size =
 124                                         ISAL_DEF_LVL0_DEFAULT;
 125                 } else {
 126                         /* Mapping API levels to ISA-L levels 1,2 & 3 */
 127                         switch (xform->compress.level) {
 128                         case RTE_COMP_LEVEL_PMD_DEFAULT:
 129                                 /* Default is 1 if not using fixed huffman */
 130                                 priv_xform->compress.level =
 131                                                 RTE_COMP_ISAL_LEVEL_ONE;
 132                                 priv_xform->level_buffer_size =
 133                                                 ISAL_DEF_LVL1_DEFAULT;
 134                                 break;
 135                         case RTE_COMP_LEVEL_MIN:
 136                                 priv_xform->compress.level =
 137                                                 RTE_COMP_ISAL_LEVEL_ONE;
 138                                 priv_xform->level_buffer_size =
 139                                                 ISAL_DEF_LVL1_DEFAULT;
 140                                 break;
 141                         case RTE_COMP_ISAL_LEVEL_TWO:
 142                                 priv_xform->compress.level =
 143                                                 RTE_COMP_ISAL_LEVEL_TWO;
 144                                 priv_xform->level_buffer_size =
 145                                                 ISAL_DEF_LVL2_DEFAULT;
 146                                 break;
 147                         /* Level 3 or higher requested */
 148                         default:
 149                                 /* Check for AVX512, to use ISA-L level 3 */
 150                                 if (rte_cpu_get_flag_enabled(
 151                                                 RTE_CPUFLAG_AVX512F)) {
 152                                         priv_xform->compress.level =
 153                                                 RTE_COMP_ISAL_LEVEL_THREE;
 154                                         priv_xform->level_buffer_size =
 155                                                 ISAL_DEF_LVL3_DEFAULT;
 156                                 }
 157                                 /* Check for AVX2, to use ISA-L level 3 */
 158                                 else if (rte_cpu_get_flag_enabled(
 159                                                 RTE_CPUFLAG_AVX2)) {
 160                                         priv_xform->compress.level =
 161                                                 RTE_COMP_ISAL_LEVEL_THREE;
 162                                         priv_xform->level_buffer_size =
 163                                                 ISAL_DEF_LVL3_DEFAULT;
 164                                 } else {
 165                                         ISAL_PMD_LOG(DEBUG, "Requested ISA-L level"
 166                                                 " 3 or above; Level 3 optimized"
 167                                                 " for AVX512 & AVX2 only."
 168                                                 " level changed to 2.\n");
 169                                         priv_xform->compress.level =
 170                                                 RTE_COMP_ISAL_LEVEL_TWO;
 171                                         priv_xform->level_buffer_size =
 172                                                 ISAL_DEF_LVL2_DEFAULT;
 173                                 }
 174                         }
 175                 }
 176         }
 177
 178         /* Set decompression private xform variables */
 179         else if (xform->type == RTE_COMP_DECOMPRESS) {
 180
 181                 /* Set private xform type - COMPRESS/DECOMPRESS */
 182                 priv_xform->type = RTE_COMP_DECOMPRESS;
 183
 184                 /* Set private xform algorithm */
 185                 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) {
 186                         if (xform->decompress.algo == RTE_COMP_ALGO_NULL) {
 187                                 ISAL_PMD_LOG(ERR, "By pass not supported\n");
 188                                 return -ENOTSUP;
 189                         }
 190                         ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
 191                         return -ENOTSUP;
 192                 }
 193                 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
 194
 195                 /* Set private xform checksum */
 196                 switch (xform->decompress.chksum) {
 197                 /* Raw deflate by default */
 198                 case(RTE_COMP_CHECKSUM_NONE):
 199                         priv_xform->decompress.chksum = ISAL_DEFLATE;
 200                         break;
 201                 case(RTE_COMP_CHECKSUM_CRC32):
 202                         priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
 203                         break;
 204                 case(RTE_COMP_CHECKSUM_ADLER32):
 205                         priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
 206                         break;
 207                 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
 208                         ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
 209                                         " supported\n");
 210                         return -ENOTSUP;
 211                 default:
 212                         ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
 213                         priv_xform->decompress.chksum = ISAL_DEFLATE;
 214                         break;
 215                 }
 216
 217                 /* Set private xform window size, 32K supported */
 218                 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
 219                         priv_xform->decompress.window_size =
 220                                         RTE_COMP_ISAL_WINDOW_SIZE;
 221                 else {
 222                         ISAL_PMD_LOG(ERR, "Window size not supported\n");
 223                         return -ENOTSUP;
 224                 }
 225         }
 226         return 0;
 227 }
 228
 229 /* Compression using chained mbufs for input/output data */
 230 static int
 231 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
 232 {
 233         int ret;
 234         uint32_t remaining_offset;
 235         uint32_t remaining_data = op->src.length;
 236         struct rte_mbuf *src = op->m_src;
 237         struct rte_mbuf *dst = op->m_dst;
 238
 239         /* check for source/destination offset passing multiple segments
 240          * and point compression stream to input/output buffer.
 241          */
 242         remaining_offset = op->src.offset;
 243         while (remaining_offset >= src->data_len) {
 244                 remaining_offset -= src->data_len;
 245                 src = src->next;
 246         }
 247         qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
 248                         op->src.length);
 249         qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
 250                         remaining_offset);
 251
 252         remaining_offset = op->dst.offset;
 253         while (remaining_offset >= dst->data_len) {
 254                 remaining_offset -= dst->data_len;
 255                 dst = dst->next;
 256         }
 257         qp->stream->avail_out = dst->data_len - remaining_offset;
 258         qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
 259                         remaining_offset);
 260
 261         if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
 262                 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
 263                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 264                 return -1;
 265         }
 266
 267         while (qp->stream->internal_state.state != ZSTATE_END) {
 268                 /* Last segment of data */
 269                 if (remaining_data <= src->data_len)
 270                         qp->stream->end_of_stream = 1;
 271
 272                 /* Execute compression operation */
 273                 ret = isal_deflate(qp->stream);
 274
 275                 remaining_data = op->src.length - qp->stream->total_in;
 276
 277                 if (ret != COMP_OK) {
 278                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
 279                         op->status = RTE_COMP_OP_STATUS_ERROR;
 280                         return ret;
 281                 }
 282
 283                 if (qp->stream->avail_in == 0 &&
 284                                 qp->stream->total_in != op->src.length) {
 285                         if (src->next != NULL) {
 286                                 src = src->next;
 287                                 qp->stream->next_in =
 288                                                 rte_pktmbuf_mtod(src, uint8_t *);
 289                                 qp->stream->avail_in =
 290                                         RTE_MIN(remaining_data, src->data_len);
 291                         } else {
 292                                 ISAL_PMD_LOG(ERR,
 293                                 "Not enough input buffer segments\n");
 294                                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 295                                 return -1;
 296                         }
 297                 }
 298
 299                 if (qp->stream->avail_out == 0 &&
 300                                 qp->stream->internal_state.state != ZSTATE_END) {
 301                         if (dst->next != NULL) {
 302                                 dst = dst->next;
 303                                 qp->stream->next_out =
 304                                                 rte_pktmbuf_mtod(dst, uint8_t *);
 305                                 qp->stream->avail_out = dst->data_len;
 306                         } else {
 307                                 ISAL_PMD_LOG(ERR,
 308                                 "Not enough output buffer segments\n");
 309                                 op->status =
 310                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 311                                 return -1;
 312                         }
 313                 }
 314         }
 315
 316         return 0;
 317 }
 318
 319 /* Decompression using chained mbufs for input/output data */
 320 static int
 321 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
 322 {
 323         int ret;
 324         uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
 325         uint32_t remaining_data = op->src.length;
 326         struct rte_mbuf *src = op->m_src;
 327         struct rte_mbuf *dst = op->m_dst;
 328
 329         /* check for offset passing multiple segments
 330          * and point decompression state to input/output buffer
 331          */
 332         src_remaining_offset = op->src.offset;
 333         while (src_remaining_offset >= src->data_len) {
 334                 src_remaining_offset -= src->data_len;
 335                 src = src->next;
 336         }
 337         qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
 338                         op->src.length);
 339         qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
 340                         src_remaining_offset);
 341
 342         dst_remaining_offset = op->dst.offset;
 343         while (dst_remaining_offset >= dst->data_len) {
 344                 dst_remaining_offset -= dst->data_len;
 345                 dst = dst->next;
 346         }
 347         qp->state->avail_out = dst->data_len - dst_remaining_offset;
 348         qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
 349                         dst_remaining_offset);
 350
 351         while (qp->state->block_state != ISAL_BLOCK_FINISH) {
 352
 353                 ret = isal_inflate(qp->state);
 354
 355                 /* Check for first segment, offset needs to be accounted for */
 356                 if (remaining_data == op->src.length) {
 357                         consumed_data = src->data_len - src_remaining_offset;
 358                 } else
 359                         consumed_data = src->data_len;
 360
 361                 if (qp->state->avail_in == 0
 362                                 && op->consumed != op->src.length) {
 363                         op->consumed += consumed_data;
 364                         remaining_data -= consumed_data;
 365
 366                         if (src->next != NULL) {
 367                                 src = src->next;
 368                                 qp->state->next_in =
 369                                                 rte_pktmbuf_mtod(src, uint8_t *);
 370                                 qp->state->avail_in =
 371                                         RTE_MIN(remaining_data, src->data_len);
 372                         }
 373                 }
 374
 375                 if (ret == ISAL_OUT_OVERFLOW) {
 376                         ISAL_PMD_LOG(ERR, "Decompression operation ran "
 377                                 "out of space, but can be recovered.\n%d bytes "
 378                                 "consumed\t%d bytes produced\n",
 379                                 consumed_data, qp->state->total_out);
 380                                 op->status =
 381                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
 382                         return ret;
 383                 } else if (ret < 0) {
 384                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 385                         op->status = RTE_COMP_OP_STATUS_ERROR;
 386                         return ret;
 387                 }
 388
 389                 if (qp->state->avail_out == 0 &&
 390                                 qp->state->block_state != ISAL_BLOCK_FINISH) {
 391                         if (dst->next != NULL) {
 392                                 dst = dst->next;
 393                                 qp->state->next_out =
 394                                                 rte_pktmbuf_mtod(dst, uint8_t *);
 395                                 qp->state->avail_out = dst->data_len;
 396                         } else {
 397                                 ISAL_PMD_LOG(ERR,
 398                                 "Not enough output buffer segments\n");
 399                                 op->status =
 400                                 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 401                                 return -1;
 402                         }
 403                 }
 404         }
 405
 406         return 0;
 407 }
 408
 409 /* Stateless Compression Function */
 410 static int
 411 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 412                 struct isal_priv_xform *priv_xform)
 413 {
 414         int ret = 0;
 415         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 416
 417         /* Required due to init clearing level_buf */
 418         uint8_t *temp_level_buf = qp->stream->level_buf;
 419
 420         /* Initialize compression stream */
 421         isal_deflate_init(qp->stream);
 422
 423         qp->stream->level_buf = temp_level_buf;
 424
 425         /* Set Checksum flag */
 426         qp->stream->gzip_flag = priv_xform->compress.chksum;
 427
 428         /* Stateless operation, input will be consumed in one go */
 429         qp->stream->flush = NO_FLUSH;
 430
 431         /* set compression level & intermediate level buffer size */
 432         qp->stream->level = priv_xform->compress.level;
 433         qp->stream->level_buf_size = priv_xform->level_buffer_size;
 434
 435         /* Set op huffman code */
 436         if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
 437                 isal_deflate_set_hufftables(qp->stream, NULL,
 438                                 IGZIP_HUFFTABLE_STATIC);
 439         else if (priv_xform->compress.deflate.huffman ==
 440                         RTE_COMP_HUFFMAN_DEFAULT)
 441                 isal_deflate_set_hufftables(qp->stream, NULL,
 442                         IGZIP_HUFFTABLE_DEFAULT);
 443         /* Dynamically change the huffman code to suit the input data */
 444         else if (priv_xform->compress.deflate.huffman ==
 445                         RTE_COMP_HUFFMAN_DYNAMIC)
 446                 isal_deflate_set_hufftables(qp->stream, NULL,
 447                                 IGZIP_HUFFTABLE_DEFAULT);
 448
 449         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 450                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 451                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 452                 return -1;
 453         }
 454
 455         if (op->dst.offset >= op->m_dst->pkt_len) {
 456                 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
 457                                 " for offset provided.\n");
 458                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 459                 return -1;
 460         }
 461
 462         /* Chained mbufs */
 463         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
 464                 ret = chained_mbuf_compression(op, qp);
 465                 if (ret < 0)
 466                         return ret;
 467         } else {
 468                 /* Linear buffer */
 469                 qp->stream->end_of_stream = 1; /* All input consumed in one */
 470                 /* Point compression stream to input buffer */
 471                 qp->stream->avail_in = op->src.length;
 472                 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
 473                                 uint8_t *, op->src.offset);
 474
 475                 /*  Point compression stream to output buffer */
 476                 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
 477                 qp->stream->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
 478                                 uint8_t *, op->dst.offset);
 479
 480                 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
 481                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
 482                                         " buffers\n");
 483                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 484                         return -1;
 485                 }
 486
 487                 /* Execute compression operation */
 488                 ret =  isal_deflate_stateless(qp->stream);
 489
 490                 /* Check that output buffer did not run out of space */
 491                 if (ret == STATELESS_OVERFLOW) {
 492                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
 493                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 494                         return ret;
 495                 }
 496
 497                 /* Check that input buffer has been fully consumed */
 498                 if (qp->stream->avail_in != (uint32_t)0) {
 499                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
 500                                         " entirely\n");
 501                         op->status = RTE_COMP_OP_STATUS_ERROR;
 502                         return -1;
 503                 }
 504
 505                 if (ret != COMP_OK) {
 506                         ISAL_PMD_LOG(ERR, "Compression operation failed\n");
 507                         op->status = RTE_COMP_OP_STATUS_ERROR;
 508                         return ret;
 509                 }
 510         }
 511
 512         op->consumed = qp->stream->total_in;
 513         if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
 514                 op->produced = qp->stream->total_out;
 515         } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
 516                 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
 517                 op->output_chksum = qp->stream->internal_state.crc + 1;
 518         } else {
 519                 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
 520                 op->output_chksum = qp->stream->internal_state.crc;
 521         }
 522
 523         return ret;
 524 }
 525
 526 /* Stateless Decompression Function */
 527 static int
 528 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 529                 struct isal_priv_xform *priv_xform)
 530 {
 531         int ret = 0;
 532
 533         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 534
 535         /* Initialize decompression state */
 536         isal_inflate_init(qp->state);
 537
 538         /* Set Checksum flag */
 539         qp->state->crc_flag = priv_xform->decompress.chksum;
 540
 541         if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
 542                 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
 543                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 544                 return -1;
 545         }
 546
 547         if (op->dst.offset >= op->m_dst->pkt_len) {
 548                 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
 549                                 "offset provided.\n");
 550                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 551                 return -1;
 552         }
 553
 554         /* Chained mbufs */
 555         if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
 556                 ret = chained_mbuf_decompression(op, qp);
 557                 if (ret !=  0)
 558                         return ret;
 559         } else {
 560                 /* Linear buffer */
 561                 /* Point decompression state to input buffer */
 562                 qp->state->avail_in = op->src.length;
 563                 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
 564                                 uint8_t *, op->src.offset);
 565
 566                 /* Point decompression state to output buffer */
 567                 qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
 568                 qp->state->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
 569                                 uint8_t *, op->dst.offset);
 570
 571                 if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
 572                         ISAL_PMD_LOG(ERR, "Invalid source or destination"
 573                                         " buffers\n");
 574                         op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 575                         return -1;
 576                 }
 577
 578                 /* Execute decompression operation */
 579                 ret = isal_inflate_stateless(qp->state);
 580
 581                 if (ret == ISAL_OUT_OVERFLOW) {
 582                         ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
 583                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 584                         return ret;
 585                 }
 586
 587                 /* Check that input buffer has been fully consumed */
 588                 if (qp->state->avail_in != (uint32_t)0) {
 589                         ISAL_PMD_LOG(ERR, "Input buffer could not be read"
 590                                         " entirely\n");
 591                         op->status = RTE_COMP_OP_STATUS_ERROR;
 592                         return -1;
 593                 }
 594
 595                 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
 596                         ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
 597                         op->status = RTE_COMP_OP_STATUS_ERROR;
 598                         return ret;
 599                 }
 600                 op->consumed = op->src.length - qp->state->avail_in;
 601         }
 602         op->produced = qp->state->total_out;
 603         op->output_chksum = qp->state->crc;
 604
 605         return ret;
 606 }
 607
 608 /* Process compression/decompression operation */
 609 static int
 610 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
 611                 struct isal_priv_xform *priv_xform)
 612 {
 613         switch (priv_xform->type) {
 614         case RTE_COMP_COMPRESS:
 615                 process_isal_deflate(op, qp, priv_xform);
 616                 break;
 617         case RTE_COMP_DECOMPRESS:
 618                 process_isal_inflate(op, qp, priv_xform);
 619                 break;
 620         default:
 621                 ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
 622                 return -ENOTSUP;
 623         }
 624         return 0;
 625 }
 626
 627 /* Enqueue burst */
 628 static uint16_t
 629 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops,
 630                         uint16_t nb_ops)
 631 {
 632         struct isal_comp_qp *qp = queue_pair;
 633         uint16_t i;
 634         int retval;
 635         int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops);
 636
 637         for (i = 0; i < num_enq; i++) {
 638                 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) {
 639                         ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 640                         ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n");
 641                         qp->qp_stats.enqueue_err_count++;
 642                         continue;
 643                 }
 644                 retval = process_op(qp, ops[i], ops[i]->private_xform);
 645                 if (unlikely(retval < 0) ||
 646                                 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 647                         qp->qp_stats.enqueue_err_count++;
 648                 }
 649         }
 650
 651         retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops,
 652                         num_enq, NULL);
 653         qp->num_free_elements -= retval;
 654         qp->qp_stats.enqueued_count += retval;
 655
 656         return retval;
 657 }
 658
 659 /* Dequeue burst */
 660 static uint16_t
 661 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
 662                 uint16_t nb_ops)
 663 {
 664         struct isal_comp_qp *qp = queue_pair;
 665         uint16_t nb_dequeued;
 666
 667         nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops,
 668                         nb_ops, NULL);
 669         qp->num_free_elements += nb_dequeued;
 670         qp->qp_stats.dequeued_count += nb_dequeued;
 671
 672         return nb_dequeued;
 673 }
 674
 675 /* Create ISA-L compression device */
 676 static int
 677 compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
 678                 struct rte_compressdev_pmd_init_params *init_params)
 679 {
 680         struct rte_compressdev *dev;
 681
 682         dev = rte_compressdev_pmd_create(name, &vdev->device,
 683                         sizeof(struct isal_comp_private), init_params);
 684         if (dev == NULL) {
 685                 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev");
 686                 return -EFAULT;
 687         }
 688
 689         dev->dev_ops = isal_compress_pmd_ops;
 690
 691         /* register rx/tx burst functions for data path */
 692         dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
 693         dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
 694
 695         ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING);
 696
 697         return 0;
 698 }
 699
 700 /** Remove compression device */
 701 static int
 702 compdev_isal_remove_dev(struct rte_vdev_device *vdev)
 703 {
 704         struct rte_compressdev *compdev;
 705         const char *name;
 706
 707         name = rte_vdev_device_name(vdev);
 708         if (name == NULL)
 709                 return -EINVAL;
 710
 711         compdev = rte_compressdev_pmd_get_named_dev(name);
 712         if (compdev == NULL)
 713                 return -ENODEV;
 714
 715         return rte_compressdev_pmd_destroy(compdev);
 716 }
 717
 718 /** Initialise ISA-L compression device */
 719 static int
 720 compdev_isal_probe(struct rte_vdev_device *dev)
 721 {
 722         struct rte_compressdev_pmd_init_params init_params = {
 723                 "",
 724                 rte_socket_id(),
 725         };
 726         const char *name, *args;
 727         int retval;
 728
 729         name = rte_vdev_device_name(dev);
 730         if (name == NULL)
 731                 return -EINVAL;
 732
 733         args = rte_vdev_device_args(dev);
 734
 735         retval = rte_compressdev_pmd_parse_input_args(&init_params, args);
 736         if (retval) {
 737                 ISAL_PMD_LOG(ERR,
 738                         "Failed to parse initialisation arguments[%s]\n", args);
 739                 return -EINVAL;
 740         }
 741
 742         return compdev_isal_create(name, dev, &init_params);
 743 }
 744
 745 static struct rte_vdev_driver compdev_isal_pmd_drv = {
 746         .probe = compdev_isal_probe,
 747         .remove = compdev_isal_remove_dev,
 748 };
 749
 750 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
 751 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
 752         "socket_id=<int>");
 753 RTE_LOG_REGISTER(isal_logtype_driver, pmd.compress.isal, INFO);