drivers/compress/zlib/zlib_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Cavium Networks
   3  */
   4
   5 #include <rte_bus_vdev.h>
   6 #include <rte_common.h>
   7
   8 #include "zlib_pmd_private.h"
   9
  10 int zlib_logtype_driver;
  11
  12 /** Compute next mbuf in the list, assign data buffer and length,
  13  *  returns 0 if mbuf is NULL
  14  */
  15 #define COMPUTE_BUF(mbuf, data, len)            \
  16                 ((mbuf = mbuf->next) ?          \
  17                 (data = rte_pktmbuf_mtod(mbuf, uint8_t *)),     \
  18                 (len = rte_pktmbuf_data_len(mbuf)) : 0)
  19
  20 static void
  21 process_zlib_deflate(struct rte_comp_op *op, z_stream *strm)
  22 {
  23         int ret, flush, fin_flush;
  24         struct rte_mbuf *mbuf_src = op->m_src;
  25         struct rte_mbuf *mbuf_dst = op->m_dst;
  26
  27         switch (op->flush_flag) {
  28         case RTE_COMP_FLUSH_FULL:
  29         case RTE_COMP_FLUSH_FINAL:
  30                 fin_flush = Z_FINISH;
  31                 break;
  32         default:
  33                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
  34                 ZLIB_PMD_ERR("Invalid flush value\n");
  35                 return;
  36         }
  37
  38         if (unlikely(!strm)) {
  39                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
  40                 ZLIB_PMD_ERR("Invalid z_stream\n");
  41                 return;
  42         }
  43         /* Update z_stream with the inputs provided by application */
  44         strm->next_in = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
  45                         op->src.offset);
  46
  47         strm->avail_in = rte_pktmbuf_data_len(mbuf_src) - op->src.offset;
  48
  49         strm->next_out = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
  50                         op->dst.offset);
  51
  52         strm->avail_out = rte_pktmbuf_data_len(mbuf_dst) - op->dst.offset;
  53
  54         /* Set flush value to NO_FLUSH unless it is last mbuf */
  55         flush = Z_NO_FLUSH;
  56         /* Initialize status to SUCCESS */
  57         op->status = RTE_COMP_OP_STATUS_SUCCESS;
  58
  59         do {
  60                 /* Set flush value to Z_FINISH for last block */
  61                 if ((op->src.length - strm->total_in) <= strm->avail_in) {
  62                         strm->avail_in = (op->src.length - strm->total_in);
  63                         flush = fin_flush;
  64                 }
  65                 do {
  66                         ret = deflate(strm, flush);
  67                         if (unlikely(ret == Z_STREAM_ERROR)) {
  68                                 /* error return, do not process further */
  69                                 op->status =  RTE_COMP_OP_STATUS_ERROR;
  70                                 goto def_end;
  71                         }
  72                         /* Break if Z_STREAM_END is encountered */
  73                         if (ret == Z_STREAM_END)
  74                                 goto def_end;
  75
  76                 /* Keep looping until input mbuf is consumed.
  77                  * Exit if destination mbuf gets exhausted.
  78                  */
  79                 } while ((strm->avail_out == 0) &&
  80                         COMPUTE_BUF(mbuf_dst, strm->next_out, strm->avail_out));
  81
  82                 if (!strm->avail_out) {
  83                         /* there is no space for compressed output */
  84                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
  85                         break;
  86                 }
  87
  88         /* Update source buffer to next mbuf
  89          * Exit if input buffers are fully consumed
  90          */
  91         } while (COMPUTE_BUF(mbuf_src, strm->next_in, strm->avail_in));
  92
  93 def_end:
  94         /* Update op stats */
  95         switch (op->status) {
  96         case RTE_COMP_OP_STATUS_SUCCESS:
  97                 op->consumed += strm->total_in;
  98         /* Fall-through */
  99         case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED:
 100                 op->produced += strm->total_out;
 101                 break;
 102         default:
 103                 ZLIB_PMD_ERR("stats not updated for status:%d\n",
 104                                 op->status);
 105         }
 106
 107         deflateReset(strm);
 108 }
 109
 110 static void
 111 process_zlib_inflate(struct rte_comp_op *op, z_stream *strm)
 112 {
 113         int ret, flush;
 114         struct rte_mbuf *mbuf_src = op->m_src;
 115         struct rte_mbuf *mbuf_dst = op->m_dst;
 116
 117         if (unlikely(!strm)) {
 118                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 119                 ZLIB_PMD_ERR("Invalid z_stream\n");
 120                 return;
 121         }
 122         strm->next_in = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
 123                         op->src.offset);
 124
 125         strm->avail_in = rte_pktmbuf_data_len(mbuf_src) - op->src.offset;
 126
 127         strm->next_out = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
 128                         op->dst.offset);
 129
 130         strm->avail_out = rte_pktmbuf_data_len(mbuf_dst) - op->dst.offset;
 131
 132         /** Ignoring flush value provided from application for decompression */
 133         flush = Z_NO_FLUSH;
 134         /* initialize status to SUCCESS */
 135         op->status = RTE_COMP_OP_STATUS_SUCCESS;
 136
 137         do {
 138                 do {
 139                         ret = inflate(strm, flush);
 140
 141                         switch (ret) {
 142                         /* Fall-through */
 143                         case Z_NEED_DICT:
 144                                 ret = Z_DATA_ERROR;
 145                         /* Fall-through */
 146                         case Z_DATA_ERROR:
 147                         /* Fall-through */
 148                         case Z_MEM_ERROR:
 149                         /* Fall-through */
 150                         case Z_STREAM_ERROR:
 151                                 op->status = RTE_COMP_OP_STATUS_ERROR;
 152                         /* Fall-through */
 153                         case Z_STREAM_END:
 154                                 /* no further computation needed if
 155                                  * Z_STREAM_END is encountered
 156                                  */
 157                                 goto inf_end;
 158                         default:
 159                                 /* success */
 160                                 break;
 161
 162                         }
 163                 /* Keep looping until input mbuf is consumed.
 164                  * Exit if destination mbuf gets exhausted.
 165                  */
 166                 } while ((strm->avail_out == 0) &&
 167                         COMPUTE_BUF(mbuf_dst, strm->next_out, strm->avail_out));
 168
 169                 if (!strm->avail_out) {
 170                         /* there is no more space for decompressed output */
 171                         op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
 172                         break;
 173                 }
 174         /* Read next input buffer to be processed, exit if compressed
 175          * blocks are fully read
 176          */
 177         } while (COMPUTE_BUF(mbuf_src, strm->next_in, strm->avail_in));
 178
 179 inf_end:
 180         /* Update op stats */
 181         switch (op->status) {
 182         case RTE_COMP_OP_STATUS_SUCCESS:
 183                 op->consumed += strm->total_in;
 184         /* Fall-through */
 185         case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED:
 186                 op->produced += strm->total_out;
 187                 break;
 188         default:
 189                 ZLIB_PMD_ERR("stats not produced for status:%d\n",
 190                                 op->status);
 191         }
 192
 193         inflateReset(strm);
 194 }
 195
 196 /** Process comp operation for mbuf */
 197 static inline int
 198 process_zlib_op(struct zlib_qp *qp, struct rte_comp_op *op)
 199 {
 200         struct zlib_stream *stream;
 201         struct zlib_priv_xform *private_xform;
 202
 203         if ((op->op_type == RTE_COMP_OP_STATEFUL) ||
 204                         (op->src.offset > rte_pktmbuf_data_len(op->m_src)) ||
 205                         (op->dst.offset > rte_pktmbuf_data_len(op->m_dst))) {
 206                 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 207                 ZLIB_PMD_ERR("Invalid source or destination buffers or "
 208                              "invalid Operation requested\n");
 209         } else {
 210                 private_xform = (struct zlib_priv_xform *)op->private_xform;
 211                 stream = &private_xform->stream;
 212                 stream->comp(op, &stream->strm);
 213         }
 214         /* whatever is out of op, put it into completion queue with
 215          * its status
 216          */
 217         return rte_ring_enqueue(qp->processed_pkts, (void *)op);
 218 }
 219
 220 /** Parse comp xform and set private xform/Stream parameters */
 221 int
 222 zlib_set_stream_parameters(const struct rte_comp_xform *xform,
 223                 struct zlib_stream *stream)
 224 {
 225         int strategy, level, wbits;
 226         z_stream *strm = &stream->strm;
 227
 228         /* allocate deflate state */
 229         strm->zalloc = Z_NULL;
 230         strm->zfree = Z_NULL;
 231         strm->opaque = Z_NULL;
 232
 233         switch (xform->type) {
 234         case RTE_COMP_COMPRESS:
 235                 stream->comp = process_zlib_deflate;
 236                 stream->free = deflateEnd;
 237                 /** Compression window bits */
 238                 switch (xform->compress.algo) {
 239                 case RTE_COMP_ALGO_DEFLATE:
 240                         wbits = -(xform->compress.window_size);
 241                         break;
 242                 default:
 243                         ZLIB_PMD_ERR("Compression algorithm not supported\n");
 244                         return -1;
 245                 }
 246                 /** Compression Level */
 247                 switch (xform->compress.level) {
 248                 case RTE_COMP_LEVEL_PMD_DEFAULT:
 249                         level = Z_DEFAULT_COMPRESSION;
 250                         break;
 251                 case RTE_COMP_LEVEL_NONE:
 252                         level = Z_NO_COMPRESSION;
 253                         break;
 254                 case RTE_COMP_LEVEL_MIN:
 255                         level = Z_BEST_SPEED;
 256                         break;
 257                 case RTE_COMP_LEVEL_MAX:
 258                         level = Z_BEST_COMPRESSION;
 259                         break;
 260                 default:
 261                         level = xform->compress.level;
 262                         if (level < RTE_COMP_LEVEL_MIN ||
 263                                         level > RTE_COMP_LEVEL_MAX) {
 264                                 ZLIB_PMD_ERR("Compression level %d "
 265                                                 "not supported\n",
 266                                                 level);
 267                                 return -1;
 268                         }
 269                         break;
 270                 }
 271                 /** Compression strategy */
 272                 switch (xform->compress.deflate.huffman) {
 273                 case RTE_COMP_HUFFMAN_DEFAULT:
 274                         strategy = Z_DEFAULT_STRATEGY;
 275                         break;
 276                 case RTE_COMP_HUFFMAN_FIXED:
 277                         strategy = Z_FIXED;
 278                         break;
 279                 case RTE_COMP_HUFFMAN_DYNAMIC:
 280                         strategy = Z_DEFAULT_STRATEGY;
 281                         break;
 282                 default:
 283                         ZLIB_PMD_ERR("Compression strategy not supported\n");
 284                         return -1;
 285                 }
 286                 if (deflateInit2(strm, level,
 287                                         Z_DEFLATED, wbits,
 288                                         DEF_MEM_LEVEL, strategy) != Z_OK) {
 289                         ZLIB_PMD_ERR("Deflate init failed\n");
 290                         return -1;
 291                 }
 292                 break;
 293
 294         case RTE_COMP_DECOMPRESS:
 295                 stream->comp = process_zlib_inflate;
 296                 stream->free = inflateEnd;
 297                 /** window bits */
 298                 switch (xform->decompress.algo) {
 299                 case RTE_COMP_ALGO_DEFLATE:
 300                         wbits = -(xform->decompress.window_size);
 301                         break;
 302                 default:
 303                         ZLIB_PMD_ERR("Compression algorithm not supported\n");
 304                         return -1;
 305                 }
 306
 307                 if (inflateInit2(strm, wbits) != Z_OK) {
 308                         ZLIB_PMD_ERR("Inflate init failed\n");
 309                         return -1;
 310                 }
 311                 break;
 312         default:
 313                 return -1;
 314         }
 315         return 0;
 316 }
 317
 318 static uint16_t
 319 zlib_pmd_enqueue_burst(void *queue_pair,
 320                         struct rte_comp_op **ops, uint16_t nb_ops)
 321 {
 322         struct zlib_qp *qp = queue_pair;
 323         int ret;
 324         uint16_t i;
 325         uint16_t enqd = 0;
 326         for (i = 0; i < nb_ops; i++) {
 327                 ret = process_zlib_op(qp, ops[i]);
 328                 if (unlikely(ret < 0)) {
 329                         /* increment count if failed to push to completion
 330                          * queue
 331                          */
 332                         qp->qp_stats.enqueue_err_count++;
 333                 } else {
 334                         qp->qp_stats.enqueued_count++;
 335                         enqd++;
 336                 }
 337         }
 338         return enqd;
 339 }
 340
 341 static uint16_t
 342 zlib_pmd_dequeue_burst(void *queue_pair,
 343                         struct rte_comp_op **ops, uint16_t nb_ops)
 344 {
 345         struct zlib_qp *qp = queue_pair;
 346
 347         unsigned int nb_dequeued = 0;
 348
 349         nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts,
 350                         (void **)ops, nb_ops, NULL);
 351         qp->qp_stats.dequeued_count += nb_dequeued;
 352
 353         return nb_dequeued;
 354 }
 355
 356 static int
 357 zlib_create(const char *name,
 358                 struct rte_vdev_device *vdev,
 359                 struct rte_compressdev_pmd_init_params *init_params)
 360 {
 361         struct rte_compressdev *dev;
 362
 363         dev = rte_compressdev_pmd_create(name, &vdev->device,
 364                         sizeof(struct zlib_private), init_params);
 365         if (dev == NULL) {
 366                 ZLIB_PMD_ERR("driver %s: create failed", init_params->name);
 367                 return -ENODEV;
 368         }
 369
 370         dev->dev_ops = rte_zlib_pmd_ops;
 371
 372         /* register rx/tx burst functions for data path */
 373         dev->dequeue_burst = zlib_pmd_dequeue_burst;
 374         dev->enqueue_burst = zlib_pmd_enqueue_burst;
 375
 376         return 0;
 377 }
 378
 379 static int
 380 zlib_probe(struct rte_vdev_device *vdev)
 381 {
 382         struct rte_compressdev_pmd_init_params init_params = {
 383                 "",
 384                 rte_socket_id()
 385         };
 386         const char *name;
 387         const char *input_args;
 388         int retval;
 389
 390         name = rte_vdev_device_name(vdev);
 391
 392         if (name == NULL)
 393                 return -EINVAL;
 394
 395         input_args = rte_vdev_device_args(vdev);
 396
 397         retval = rte_compressdev_pmd_parse_input_args(&init_params, input_args);
 398         if (retval < 0) {
 399                 ZLIB_PMD_LOG(ERR,
 400                         "Failed to parse initialisation arguments[%s]\n",
 401                         input_args);
 402                 return -EINVAL;
 403         }
 404
 405         return zlib_create(name, vdev, &init_params);
 406 }
 407
 408 static int
 409 zlib_remove(struct rte_vdev_device *vdev)
 410 {
 411         struct rte_compressdev *compressdev;
 412         const char *name;
 413
 414         name = rte_vdev_device_name(vdev);
 415         if (name == NULL)
 416                 return -EINVAL;
 417
 418         compressdev = rte_compressdev_pmd_get_named_dev(name);
 419         if (compressdev == NULL)
 420                 return -ENODEV;
 421
 422         return rte_compressdev_pmd_destroy(compressdev);
 423 }
 424
 425 static struct rte_vdev_driver zlib_pmd_drv = {
 426         .probe = zlib_probe,
 427         .remove = zlib_remove
 428 };
 429
 430 RTE_PMD_REGISTER_VDEV(COMPRESSDEV_NAME_ZLIB_PMD, zlib_pmd_drv);
 431
 432 RTE_INIT(zlib_init_log)
 433 {
 434         zlib_logtype_driver = rte_log_register("pmd.compress.zlib");
 435         if (zlib_logtype_driver >= 0)
 436                 rte_log_set_level(zlib_logtype_driver, RTE_LOG_INFO);
 437 }