drivers/regex/mlx5/mlx5_regex_fastpath.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <unistd.h>
   6 #include <sys/mman.h>
   7
   8 #include <rte_malloc.h>
   9 #include <rte_log.h>
  10 #include <rte_errno.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_pci.h>
  13 #include <rte_regexdev_driver.h>
  14 #include <rte_mbuf.h>
  15
  16 #include <infiniband/mlx5dv.h>
  17 #include <mlx5_glue.h>
  18 #include <mlx5_common.h>
  19 #include <mlx5_prm.h>
  20 #include <strings.h>
  21
  22 #include "mlx5_regex_utils.h"
  23 #include "mlx5_rxp.h"
  24 #include "mlx5_regex.h"
  25
  26 #define MLX5_REGEX_METADATA_SIZE 64
  27 #define MLX5_REGEX_MAX_INPUT (1 << 14)
  28 #define MLX5_REGEX_MAX_OUTPUT (1 << 11)
  29
  30 #define MLX5_REGEX_WQE_METADATA_OFFSET 16
  31 #define MLX5_REGEX_WQE_GATHER_OFFSET 32
  32 #define MLX5_REGEX_WQE_SCATTER_OFFSET 48
  33
  34 static inline uint32_t
  35 sq_size_get(struct mlx5_regex_sq *sq)
  36 {
  37         return (1U << sq->log_nb_desc);
  38 }
  39
  40 struct mlx5_regex_job {
  41         uint64_t user_id;
  42         uint8_t *input;
  43         volatile uint8_t *output;
  44         volatile uint8_t *metadata;
  45 } __rte_cached_aligned;
  46
  47 static inline void
  48 set_data_seg(struct mlx5_wqe_data_seg *seg,
  49              uint32_t length, uint32_t lkey,
  50              uintptr_t address)
  51 {
  52         seg->byte_count = rte_cpu_to_be_32(length);
  53         seg->lkey = rte_cpu_to_be_32(lkey);
  54         seg->addr = rte_cpu_to_be_64(address);
  55 }
  56
  57 static inline void
  58 set_metadata_seg(struct mlx5_wqe_metadata_seg *seg,
  59                  uint32_t mmo_control_31_0, uint32_t lkey,
  60                  uintptr_t address)
  61 {
  62         seg->mmo_control_31_0 = htobe32(mmo_control_31_0);
  63         seg->lkey = rte_cpu_to_be_32(lkey);
  64         seg->addr = rte_cpu_to_be_64(address);
  65 }
  66
  67 static void
  68 setup_sqs(struct mlx5_regex_qp *queue)
  69 {
  70         size_t sqid, entry;
  71         uint32_t job_id;
  72         for (sqid = 0; sqid < queue->nb_obj; sqid++) {
  73                 struct mlx5_regex_sq *sq = &queue->sqs[sqid];
  74                 uint8_t *wqe = (uint8_t *)sq->wqe;
  75                 for (entry = 0 ; entry < sq_size_get(sq); entry++) {
  76                         job_id = sqid * sq_size_get(sq) + entry;
  77                         struct mlx5_regex_job *job = &queue->jobs[job_id];
  78
  79                         set_metadata_seg((struct mlx5_wqe_metadata_seg *)
  80                                          (wqe + MLX5_REGEX_WQE_METADATA_OFFSET),
  81                                          0, queue->metadata->lkey,
  82                                          (uintptr_t)job->metadata);
  83                         set_data_seg((struct mlx5_wqe_data_seg *)
  84                                      (wqe + MLX5_REGEX_WQE_GATHER_OFFSET),
  85                                      0, queue->inputs->lkey,
  86                                      (uintptr_t)job->input);
  87                         set_data_seg((struct mlx5_wqe_data_seg *)
  88                                      (wqe + MLX5_REGEX_WQE_SCATTER_OFFSET),
  89                                      MLX5_REGEX_MAX_OUTPUT,
  90                                      queue->outputs->lkey,
  91                                      (uintptr_t)job->output);
  92                         wqe += 64;
  93                 }
  94                 queue->free_sqs |= 1 << sqid;
  95         }
  96 }
  97
  98 static int
  99 setup_buffers(struct mlx5_regex_qp *qp, struct ibv_pd *pd)
 100 {
 101         uint32_t i;
 102         int err;
 103
 104         void *ptr = rte_calloc(__func__, qp->nb_desc,
 105                                MLX5_REGEX_METADATA_SIZE,
 106                                MLX5_REGEX_METADATA_SIZE);
 107         if (!ptr)
 108                 return -ENOMEM;
 109
 110         qp->metadata = mlx5_glue->reg_mr(pd, ptr,
 111                                          MLX5_REGEX_METADATA_SIZE*qp->nb_desc,
 112                                          IBV_ACCESS_LOCAL_WRITE);
 113         if (!qp->metadata) {
 114                 rte_free(ptr);
 115                 return -EINVAL;
 116         }
 117         ptr = rte_calloc(__func__, qp->nb_desc,
 118                          MLX5_REGEX_MAX_INPUT,
 119                          MLX5_REGEX_MAX_INPUT);
 120
 121         if (!ptr) {
 122                 err = -ENOMEM;
 123                 goto err_input;
 124         }
 125         qp->inputs = mlx5_glue->reg_mr(pd, ptr,
 126                                        MLX5_REGEX_MAX_INPUT*qp->nb_desc,
 127                                        IBV_ACCESS_LOCAL_WRITE);
 128         if (!qp->inputs) {
 129                 rte_free(ptr);
 130                 err = -EINVAL;
 131                 goto err_input;
 132         }
 133
 134         ptr = rte_calloc(__func__, qp->nb_desc,
 135                          MLX5_REGEX_MAX_OUTPUT,
 136                          MLX5_REGEX_MAX_OUTPUT);
 137         if (!ptr) {
 138                 err = -ENOMEM;
 139                 goto err_output;
 140         }
 141         qp->outputs = mlx5_glue->reg_mr(pd, ptr,
 142                                         MLX5_REGEX_MAX_OUTPUT * qp->nb_desc,
 143                                         IBV_ACCESS_LOCAL_WRITE);
 144         if (!qp->outputs) {
 145                 rte_free(ptr);
 146                 err = -EINVAL;
 147                 goto err_output;
 148         }
 149
 150         /* distribute buffers to jobs */
 151         for (i = 0; i < qp->nb_desc; i++) {
 152                 qp->jobs[i].input =
 153                         (uint8_t *)qp->inputs->addr +
 154                         (i % qp->nb_desc) * MLX5_REGEX_MAX_INPUT;
 155                 qp->jobs[i].output =
 156                         (uint8_t *)qp->outputs->addr +
 157                         (i % qp->nb_desc) * MLX5_REGEX_MAX_OUTPUT;
 158                 qp->jobs[i].metadata =
 159                         (uint8_t *)qp->metadata->addr +
 160                         (i % qp->nb_desc) * MLX5_REGEX_METADATA_SIZE;
 161         }
 162         return 0;
 163
 164 err_output:
 165         ptr = qp->inputs->addr;
 166         rte_free(ptr);
 167         mlx5_glue->dereg_mr(qp->inputs);
 168 err_input:
 169         ptr = qp->metadata->addr;
 170         rte_free(ptr);
 171         mlx5_glue->dereg_mr(qp->metadata);
 172         return err;
 173 }
 174
 175 int
 176 mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id)
 177 {
 178         struct mlx5_regex_qp *qp = &priv->qps[qp_id];
 179         int err;
 180
 181         qp->jobs = rte_calloc(__func__, qp->nb_desc, sizeof(*qp->jobs),
 182                               sizeof(*qp->jobs));
 183         if (!qp->jobs)
 184                 return -ENOMEM;
 185         err = setup_buffers(qp, priv->pd);
 186         if (err)
 187                 return err;
 188         setup_sqs(qp);
 189         return 0;
 190 }