drivers/regex/mlx5/mlx5_regex.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_log.h>
   7 #include <rte_errno.h>
   8 #include <rte_pci.h>
   9 #include <rte_regexdev.h>
  10 #include <rte_regexdev_core.h>
  11 #include <rte_regexdev_driver.h>
  12 #include <rte_bus_pci.h>
  13
  14 #include <mlx5_common.h>
  15 #include <mlx5_common_mr.h>
  16 #include <mlx5_glue.h>
  17 #include <mlx5_devx_cmds.h>
  18 #include <mlx5_prm.h>
  19
  20 #include "mlx5_regex.h"
  21 #include "mlx5_regex_utils.h"
  22 #include "mlx5_rxp_csrs.h"
  23
  24 #define MLX5_REGEX_DRIVER_NAME regex_mlx5
  25
  26 int mlx5_regex_logtype;
  27
  28 TAILQ_HEAD(regex_mem_event, mlx5_regex_priv) mlx5_mem_event_list =
  29                                 TAILQ_HEAD_INITIALIZER(mlx5_mem_event_list);
  30 static pthread_mutex_t mem_event_list_lock = PTHREAD_MUTEX_INITIALIZER;
  31
  32 const struct rte_regexdev_ops mlx5_regexdev_ops = {
  33         .dev_info_get = mlx5_regex_info_get,
  34         .dev_configure = mlx5_regex_configure,
  35         .dev_db_import = mlx5_regex_rules_db_import,
  36         .dev_qp_setup = mlx5_regex_qp_setup,
  37         .dev_start = mlx5_regex_start,
  38         .dev_stop = mlx5_regex_stop,
  39         .dev_close = mlx5_regex_close,
  40 };
  41
  42 int
  43 mlx5_regex_start(struct rte_regexdev *dev __rte_unused)
  44 {
  45         return 0;
  46 }
  47
  48 int
  49 mlx5_regex_stop(struct rte_regexdev *dev __rte_unused)
  50 {
  51         return 0;
  52 }
  53
  54 int
  55 mlx5_regex_close(struct rte_regexdev *dev __rte_unused)
  56 {
  57         return 0;
  58 }
  59
  60 static int
  61 mlx5_regex_engines_status(struct ibv_context *ctx, int num_engines)
  62 {
  63         uint32_t fpga_ident = 0;
  64         int err;
  65         int i;
  66
  67         for (i = 0; i < num_engines; i++) {
  68                 err = mlx5_devx_regex_register_read(ctx, i,
  69                                                     MLX5_RXP_CSR_IDENTIFIER,
  70                                                     &fpga_ident);
  71                 fpga_ident = (fpga_ident & (0x0000FFFF));
  72                 if (err || fpga_ident != MLX5_RXP_IDENTIFIER) {
  73                         DRV_LOG(ERR, "Failed setup RXP %d err %d database "
  74                                 "memory 0x%x", i, err, fpga_ident);
  75                         if (!err)
  76                                 err = EINVAL;
  77                         return err;
  78                 }
  79         }
  80         return 0;
  81 }
  82
  83 static void
  84 mlx5_regex_get_name(char *name, struct rte_device *dev)
  85 {
  86         sprintf(name, "mlx5_regex_%s", dev->name);
  87 }
  88
  89 /**
  90  * Callback for memory event.
  91  *
  92  * @param event_type
  93  *   Memory event type.
  94  * @param addr
  95  *   Address of memory.
  96  * @param len
  97  *   Size of memory.
  98  */
  99 static void
 100 mlx5_regex_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
 101                            size_t len, void *arg __rte_unused)
 102 {
 103         struct mlx5_regex_priv *priv;
 104
 105         /* Must be called from the primary process. */
 106         MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
 107         switch (event_type) {
 108         case RTE_MEM_EVENT_FREE:
 109                 pthread_mutex_lock(&mem_event_list_lock);
 110                 /* Iterate all the existing mlx5 devices. */
 111                 TAILQ_FOREACH(priv, &mlx5_mem_event_list, mem_event_cb)
 112                         mlx5_free_mr_by_addr(&priv->mr_scache,
 113                                              priv->ctx->device->name,
 114                                              addr, len);
 115                 pthread_mutex_unlock(&mem_event_list_lock);
 116                 break;
 117         case RTE_MEM_EVENT_ALLOC:
 118         default:
 119                 break;
 120         }
 121 }
 122
 123 static int
 124 mlx5_regex_dev_probe(struct rte_device *rte_dev)
 125 {
 126         struct ibv_device *ibv;
 127         struct mlx5_regex_priv *priv = NULL;
 128         struct ibv_context *ctx = NULL;
 129         struct mlx5_hca_attr attr;
 130         char name[RTE_REGEXDEV_NAME_MAX_LEN];
 131         int ret;
 132         uint32_t val;
 133
 134         ibv = mlx5_os_get_ibv_dev(rte_dev);
 135         if (ibv == NULL)
 136                 return -rte_errno;
 137         DRV_LOG(INFO, "Probe device \"%s\".", ibv->name);
 138         ctx = mlx5_glue->dv_open_device(ibv);
 139         if (!ctx) {
 140                 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
 141                 rte_errno = ENODEV;
 142                 return -rte_errno;
 143         }
 144         ret = mlx5_devx_cmd_query_hca_attr(ctx, &attr);
 145         if (ret) {
 146                 DRV_LOG(ERR, "Unable to read HCA capabilities.");
 147                 rte_errno = ENOTSUP;
 148                 goto dev_error;
 149         } else if (!attr.regex || attr.regexp_num_of_engines == 0) {
 150                 DRV_LOG(ERR, "Not enough capabilities to support RegEx, maybe "
 151                         "old FW/OFED version?");
 152                 rte_errno = ENOTSUP;
 153                 goto dev_error;
 154         }
 155         if (mlx5_regex_engines_status(ctx, 2)) {
 156                 DRV_LOG(ERR, "RegEx engine error.");
 157                 rte_errno = ENOMEM;
 158                 goto dev_error;
 159         }
 160         priv = rte_zmalloc("mlx5 regex device private", sizeof(*priv),
 161                            RTE_CACHE_LINE_SIZE);
 162         if (!priv) {
 163                 DRV_LOG(ERR, "Failed to allocate private memory.");
 164                 rte_errno = ENOMEM;
 165                 goto dev_error;
 166         }
 167         priv->sq_ts_format = attr.sq_ts_format;
 168         priv->ctx = ctx;
 169         priv->nb_engines = 2; /* attr.regexp_num_of_engines */
 170         ret = mlx5_devx_regex_register_read(priv->ctx, 0,
 171                                             MLX5_RXP_CSR_IDENTIFIER, &val);
 172         if (ret) {
 173                 DRV_LOG(ERR, "CSR read failed!");
 174                 goto dev_error;
 175         }
 176         if (val == MLX5_RXP_BF2_IDENTIFIER)
 177                 priv->is_bf2 = 1;
 178         /* Default RXP programming mode to Shared. */
 179         priv->prog_mode = MLX5_RXP_SHARED_PROG_MODE;
 180         mlx5_regex_get_name(name, rte_dev);
 181         priv->regexdev = rte_regexdev_register(name);
 182         if (priv->regexdev == NULL) {
 183                 DRV_LOG(ERR, "Failed to register RegEx device.");
 184                 rte_errno = rte_errno ? rte_errno : EINVAL;
 185                 goto error;
 186         }
 187         /*
 188          * This PMD always claims the write memory barrier on UAR
 189          * registers writings, it is safe to allocate UAR with any
 190          * memory mapping type.
 191          */
 192         priv->uar = mlx5_devx_alloc_uar(ctx, -1);
 193         if (!priv->uar) {
 194                 DRV_LOG(ERR, "can't allocate uar.");
 195                 rte_errno = ENOMEM;
 196                 goto error;
 197         }
 198         priv->pd = mlx5_glue->alloc_pd(ctx);
 199         if (!priv->pd) {
 200                 DRV_LOG(ERR, "can't allocate pd.");
 201                 rte_errno = ENOMEM;
 202                 goto error;
 203         }
 204         priv->regexdev->dev_ops = &mlx5_regexdev_ops;
 205         priv->regexdev->enqueue = mlx5_regexdev_enqueue;
 206 #ifdef HAVE_MLX5_UMR_IMKEY
 207         if (!attr.umr_indirect_mkey_disabled &&
 208             !attr.umr_modify_entity_size_disabled)
 209                 priv->has_umr = 1;
 210         if (priv->has_umr)
 211                 priv->regexdev->enqueue = mlx5_regexdev_enqueue_gga;
 212 #endif
 213         priv->regexdev->dequeue = mlx5_regexdev_dequeue;
 214         priv->regexdev->device = rte_dev;
 215         priv->regexdev->data->dev_private = priv;
 216         priv->regexdev->state = RTE_REGEXDEV_READY;
 217         priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 218         priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 219         ret = mlx5_mr_btree_init(&priv->mr_scache.cache,
 220                                  MLX5_MR_BTREE_CACHE_N * 2,
 221                                  rte_socket_id());
 222         if (ret) {
 223                 DRV_LOG(ERR, "MR init tree failed.");
 224             rte_errno = ENOMEM;
 225                 goto error;
 226         }
 227         /* Register callback function for global shared MR cache management. */
 228         if (TAILQ_EMPTY(&mlx5_mem_event_list))
 229                 rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
 230                                                 mlx5_regex_mr_mem_event_cb,
 231                                                 NULL);
 232         /* Add device to memory callback list. */
 233         pthread_mutex_lock(&mem_event_list_lock);
 234         TAILQ_INSERT_TAIL(&mlx5_mem_event_list, priv, mem_event_cb);
 235         pthread_mutex_unlock(&mem_event_list_lock);
 236         DRV_LOG(INFO, "RegEx GGA is %s.",
 237                 priv->has_umr ? "supported" : "unsupported");
 238         return 0;
 239
 240 error:
 241         if (priv->pd)
 242                 mlx5_glue->dealloc_pd(priv->pd);
 243         if (priv->uar)
 244                 mlx5_glue->devx_free_uar(priv->uar);
 245         if (priv->regexdev)
 246                 rte_regexdev_unregister(priv->regexdev);
 247 dev_error:
 248         if (ctx)
 249                 mlx5_glue->close_device(ctx);
 250         if (priv)
 251                 rte_free(priv);
 252         return -rte_errno;
 253 }
 254
 255 static int
 256 mlx5_regex_dev_remove(struct rte_device *rte_dev)
 257 {
 258         char name[RTE_REGEXDEV_NAME_MAX_LEN];
 259         struct rte_regexdev *dev;
 260         struct mlx5_regex_priv *priv = NULL;
 261
 262         mlx5_regex_get_name(name, rte_dev);
 263         dev = rte_regexdev_get_device_by_name(name);
 264         if (!dev)
 265                 return 0;
 266         priv = dev->data->dev_private;
 267         if (priv) {
 268                 /* Remove from memory callback device list. */
 269                 pthread_mutex_lock(&mem_event_list_lock);
 270                 TAILQ_REMOVE(&mlx5_mem_event_list, priv, mem_event_cb);
 271                 pthread_mutex_unlock(&mem_event_list_lock);
 272                 if (TAILQ_EMPTY(&mlx5_mem_event_list))
 273                         rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
 274                                                           NULL);
 275                 if (priv->mr_scache.cache.table)
 276                         mlx5_mr_release_cache(&priv->mr_scache);
 277                 if (priv->pd)
 278                         mlx5_glue->dealloc_pd(priv->pd);
 279                 if (priv->uar)
 280                         mlx5_glue->devx_free_uar(priv->uar);
 281                 if (priv->regexdev)
 282                         rte_regexdev_unregister(priv->regexdev);
 283                 if (priv->ctx)
 284                         mlx5_glue->close_device(priv->ctx);
 285                 rte_free(priv);
 286         }
 287         return 0;
 288 }
 289
 290 static const struct rte_pci_id mlx5_regex_pci_id_map[] = {
 291         {
 292                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 293                                 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
 294         },
 295         {
 296                 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
 297                                 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF)
 298         },
 299         {
 300                 .vendor_id = 0
 301         }
 302 };
 303
 304 static struct mlx5_class_driver mlx5_regex_driver = {
 305         .drv_class = MLX5_CLASS_REGEX,
 306         .name = RTE_STR(MLX5_REGEX_DRIVER_NAME),
 307         .id_table = mlx5_regex_pci_id_map,
 308         .probe = mlx5_regex_dev_probe,
 309         .remove = mlx5_regex_dev_remove,
 310 };
 311
 312 RTE_INIT(rte_mlx5_regex_init)
 313 {
 314         mlx5_common_init();
 315         if (mlx5_glue)
 316                 mlx5_class_driver_register(&mlx5_regex_driver);
 317 }
 318
 319 RTE_LOG_REGISTER_DEFAULT(mlx5_regex_logtype, NOTICE)
 320 RTE_PMD_EXPORT_NAME(MLX5_REGEX_DRIVER_NAME, __COUNTER__);
 321 RTE_PMD_REGISTER_PCI_TABLE(MLX5_REGEX_DRIVER_NAME, mlx5_regex_pci_id_map);
 322 RTE_PMD_REGISTER_KMOD_DEP(MLX5_REGEX_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");