drivers/compress/isal/isal_compress_pmd_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4 #include <isa-l.h>
   5
   6 #include <rte_common.h>
   7 #include <rte_cpuflags.h>
   8 #include <rte_compressdev_pmd.h>
   9 #include <rte_malloc.h>
  10
  11 #include "isal_compress_pmd_private.h"
  12
  13 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
  14         {
  15                 .algo = RTE_COMP_ALGO_DEFLATE,
  16                 .comp_feature_flags =   RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
  17                                         RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
  18                                         RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
  19                                         RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
  20                                         RTE_COMP_FF_HUFFMAN_FIXED |
  21                                         RTE_COMP_FF_HUFFMAN_DYNAMIC |
  22                                         RTE_COMP_FF_CRC32_CHECKSUM |
  23                                         RTE_COMP_FF_ADLER32_CHECKSUM,
  24                 .window_size = {
  25                         .min = 15,
  26                         .max = 15,
  27                         .increment = 0
  28                 },
  29         },
  30         RTE_COMP_END_OF_CAPABILITIES_LIST()
  31 };
  32
  33 /** Configure device */
  34 static int
  35 isal_comp_pmd_config(struct rte_compressdev *dev,
  36                 struct rte_compressdev_config *config)
  37 {
  38         int ret = 0;
  39         unsigned int n;
  40         char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN];
  41         unsigned int elt_size = sizeof(struct isal_priv_xform);
  42         struct isal_comp_private *internals = dev->data->dev_private;
  43
  44         n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp",
  45                         dev->data->dev_id);
  46         if (n > sizeof(mp_name)) {
  47                 ISAL_PMD_LOG(ERR,
  48                         "Unable to create unique name for xform mempool");
  49                 return -ENOMEM;
  50         }
  51
  52         internals->priv_xform_mp = rte_mempool_lookup(mp_name);
  53
  54         if (internals->priv_xform_mp != NULL) {
  55                 if (((internals->priv_xform_mp)->elt_size != elt_size) ||
  56                                 ((internals->priv_xform_mp)->size <
  57                                         config->max_nb_priv_xforms)) {
  58
  59                         ISAL_PMD_LOG(ERR, "%s mempool already exists with different"
  60                                 " initialization parameters", mp_name);
  61                         internals->priv_xform_mp = NULL;
  62                         return -ENOMEM;
  63                 }
  64         } else { /* First time configuration */
  65                 internals->priv_xform_mp = rte_mempool_create(
  66                                 mp_name, /* mempool name */
  67                                 /* number of elements*/
  68                                 config->max_nb_priv_xforms,
  69                                 elt_size, /* element size*/
  70                                 0, /* Cache size*/
  71                                 0, /* private data size */
  72                                 NULL, /* obj initialization constructor */
  73                                 NULL, /* obj initialization constructor arg */
  74                                 NULL, /**< obj constructor*/
  75                                 NULL, /* obj constructor arg */
  76                                 config->socket_id, /* socket id */
  77                                 0); /* flags */
  78         }
  79
  80         if (internals->priv_xform_mp == NULL) {
  81                 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name);
  82                 return -ENOMEM;
  83         }
  84
  85         dev->data->dev_private = internals;
  86
  87         return ret;
  88 }
  89
  90 /** Start device */
  91 static int
  92 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev)
  93 {
  94         return 0;
  95 }
  96
  97 /** Stop device */
  98 static void
  99 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev)
 100 {
 101 }
 102
 103 /** Close device */
 104 static int
 105 isal_comp_pmd_close(struct rte_compressdev *dev)
 106 {
 107         /* Free private data */
 108         struct isal_comp_private *internals = dev->data->dev_private;
 109
 110         rte_mempool_free(internals->priv_xform_mp);
 111         return 0;
 112 }
 113
 114 /** Get device statistics */
 115 static void
 116 isal_comp_pmd_stats_get(struct rte_compressdev *dev,
 117                 struct rte_compressdev_stats *stats)
 118 {
 119         uint16_t qp_id;
 120
 121         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 122                 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 123
 124                 stats->enqueued_count += qp->qp_stats.enqueued_count;
 125                 stats->dequeued_count += qp->qp_stats.dequeued_count;
 126
 127                 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
 128                 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
 129         }
 130 }
 131
 132 /** Get device info */
 133 static void
 134 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused,
 135                 struct rte_compressdev_info *dev_info)
 136 {
 137         if (dev_info != NULL) {
 138                 dev_info->capabilities = isal_pmd_capabilities;
 139
 140                 /* Check CPU for supported vector instruction and set
 141                  * feature_flags
 142                  */
 143 #if defined(RTE_ARCH_X86)
 144                 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
 145                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX512;
 146                 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
 147                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX2;
 148                 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
 149                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX;
 150                 else
 151                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_SSE;
 152 #elif defined(RTE_ARCH_ARM)
 153                 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON))
 154                         dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_NEON;
 155 #endif
 156         }
 157 }
 158
 159 /** Reset device statistics */
 160 static void
 161 isal_comp_pmd_stats_reset(struct rte_compressdev *dev)
 162 {
 163         uint16_t qp_id;
 164
 165         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 166                 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 167                 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
 168         }
 169 }
 170
 171 /** Release queue pair */
 172 static int
 173 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
 174 {
 175         struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
 176
 177         if (qp == NULL)
 178                 return -EINVAL;
 179
 180         if (qp->stream)
 181                 rte_free(qp->stream->level_buf);
 182
 183         rte_free(qp->state);
 184         rte_ring_free(qp->processed_pkts);
 185         rte_free(qp->stream);
 186         rte_free(qp);
 187         dev->data->queue_pairs[qp_id] = NULL;
 188
 189         return 0;
 190 }
 191
 192 /** Create a ring to place process packets on */
 193 static struct rte_ring *
 194 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp,
 195                 unsigned int ring_size, int socket_id)
 196 {
 197         struct rte_ring *r;
 198
 199         r = rte_ring_lookup(qp->name);
 200         if (r) {
 201                 if (rte_ring_get_size(r) >= ring_size) {
 202                         ISAL_PMD_LOG(DEBUG,
 203                                 "Reusing existing ring %s for processed packets",
 204                                 qp->name);
 205                         return r;
 206                 }
 207
 208                         ISAL_PMD_LOG(ERR,
 209                                 "Unable to reuse existing ring %s"
 210                                 " for processed packets",
 211                          qp->name);
 212                 return NULL;
 213         }
 214
 215         return rte_ring_create(qp->name, ring_size, socket_id,
 216                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 217 }
 218
 219 /** set a unique name for the queue pair based on its name, dev_id and qp_id */
 220 static int
 221 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev,
 222 struct isal_comp_qp *qp)
 223 {
 224         unsigned int n = snprintf(qp->name, sizeof(qp->name),
 225                         "isal_comp_pmd_%u_qp_%u",
 226                         dev->data->dev_id, qp->id);
 227
 228         if (n >= sizeof(qp->name))
 229                 return -1;
 230
 231         return 0;
 232 }
 233
 234 /* Setup a queue pair */
 235 static int
 236 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 237                 uint32_t max_inflight_ops, int socket_id)
 238 {
 239         struct isal_comp_qp *qp = NULL;
 240         int retval;
 241
 242         /* Free memory prior to re-allocation if needed. */
 243         if (dev->data->queue_pairs[qp_id] != NULL)
 244                 isal_comp_pmd_qp_release(dev, qp_id);
 245
 246         /* Allocate the queue pair data structure. */
 247         qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp),
 248                                         RTE_CACHE_LINE_SIZE, socket_id);
 249         if (qp == NULL) {
 250                 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory");
 251                 return (-ENOMEM);
 252         }
 253
 254         /* Initialize memory for compression stream structure */
 255         qp->stream = rte_zmalloc_socket("Isa-l compression stream ",
 256                         sizeof(struct isal_zstream),  RTE_CACHE_LINE_SIZE,
 257                         socket_id);
 258         if (qp->stream == NULL) {
 259                 ISAL_PMD_LOG(ERR, "Failed to allocate compression stream memory");
 260                 goto qp_setup_cleanup;
 261         }
 262         /* Initialize memory for compression level buffer */
 263         qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf",
 264                         ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE,
 265                         socket_id);
 266         if (qp->stream->level_buf == NULL) {
 267                 ISAL_PMD_LOG(ERR, "Failed to allocate compression level_buf memory");
 268                 goto qp_setup_cleanup;
 269         }
 270
 271         /* Initialize memory for decompression state structure */
 272         qp->state = rte_zmalloc_socket("Isa-l decompression state",
 273                         sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE,
 274                         socket_id);
 275         if (qp->state == NULL) {
 276                 ISAL_PMD_LOG(ERR, "Failed to allocate decompression state memory");
 277                 goto qp_setup_cleanup;
 278         }
 279
 280         qp->id = qp_id;
 281         dev->data->queue_pairs[qp_id] = qp;
 282
 283         retval = isal_comp_pmd_qp_set_unique_name(dev, qp);
 284         if (retval) {
 285                 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
 286                                 "compression device");
 287                 goto qp_setup_cleanup;
 288         }
 289
 290         qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp,
 291                         max_inflight_ops, socket_id);
 292         if (qp->processed_pkts == NULL) {
 293                 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
 294                                 "compression device");
 295                 goto qp_setup_cleanup;
 296         }
 297
 298         qp->num_free_elements = rte_ring_free_count(qp->processed_pkts);
 299
 300         memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
 301         return 0;
 302
 303 qp_setup_cleanup:
 304         if (qp->stream)
 305                 rte_free(qp->stream->level_buf);
 306         rte_free(qp->stream);
 307         rte_free(qp->state);
 308         rte_free(qp);
 309
 310         return -1;
 311 }
 312
 313 /** Set private xform data*/
 314 static int
 315 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev,
 316                         const struct rte_comp_xform *xform, void **priv_xform)
 317 {
 318         int ret;
 319         struct isal_comp_private *internals = dev->data->dev_private;
 320
 321         if (xform == NULL) {
 322                 ISAL_PMD_LOG(ERR, "Invalid Xform struct");
 323                 return -EINVAL;
 324         }
 325
 326         if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) {
 327                 ISAL_PMD_LOG(ERR,
 328                         "Couldn't get object from private xform mempool");
 329                 return -ENOMEM;
 330         }
 331
 332         ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform);
 333         if (ret != 0) {
 334                 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters");
 335
 336                 /* Return private xform to mempool */
 337                 rte_mempool_put(internals->priv_xform_mp, priv_xform);
 338                 return ret;
 339         }
 340         return 0;
 341 }
 342
 343 /** Clear memory of the private xform so it doesn't leave key material behind */
 344 static int
 345 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform)
 346 {
 347         struct isal_comp_private *internals = dev->data->dev_private;
 348
 349         /* Zero out the whole structure */
 350         if (priv_xform) {
 351                 memset(priv_xform, 0, sizeof(struct isal_priv_xform));
 352                 rte_mempool_put(internals->priv_xform_mp, priv_xform);
 353         }
 354         return 0;
 355 }
 356
 357 struct rte_compressdev_ops isal_pmd_ops = {
 358                 .dev_configure          = isal_comp_pmd_config,
 359                 .dev_start              = isal_comp_pmd_start,
 360                 .dev_stop               = isal_comp_pmd_stop,
 361                 .dev_close              = isal_comp_pmd_close,
 362
 363                 .stats_get              = isal_comp_pmd_stats_get,
 364                 .stats_reset            = isal_comp_pmd_stats_reset,
 365
 366                 .dev_infos_get          = isal_comp_pmd_info_get,
 367
 368                 .queue_pair_setup       = isal_comp_pmd_qp_setup,
 369                 .queue_pair_release     = isal_comp_pmd_qp_release,
 370
 371                 .private_xform_create   = isal_comp_pmd_priv_xform_create,
 372                 .private_xform_free     = isal_comp_pmd_priv_xform_free,
 373 };
 374
 375 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops;