drivers/regex/mlx5/mlx5_rxp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_log.h>
   6 #include <rte_errno.h>
   7 #include <rte_malloc.h>
   8 #include <rte_regexdev.h>
   9 #include <rte_regexdev_core.h>
  10 #include <rte_regexdev_driver.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_devx_cmds.h>
  14 #include <mlx5_prm.h>
  15 #include <mlx5_common_os.h>
  16
  17 #include "mlx5_regex.h"
  18 #include "mlx5_regex_utils.h"
  19 #include "mlx5_rxp_csrs.h"
  20 #include "mlx5_rxp.h"
  21
  22 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES
  23 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH
  24 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX
  25 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS
  26
  27 /* Private Declarations */
  28 static int
  29 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
  30                        uint32_t address, uint32_t expected_value,
  31                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id);
  32 static int
  33 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use);
  34 static int
  35 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id);
  36 static int
  37 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id);
  38 static int
  39 program_rxp_rules(struct mlx5_regex_priv *priv,
  40                   struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id);
  41 static int
  42 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id);
  43 static int
  44 write_private_rules(struct mlx5_regex_priv *priv,
  45                     struct mlx5_rxp_ctl_rules_pgm *rules,
  46                     uint8_t id);
  47 static int
  48 write_shared_rules(struct mlx5_regex_priv *priv,
  49                    struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
  50                    uint8_t db_to_program);
  51 static int
  52 rxp_db_setup(struct mlx5_regex_priv *priv);
  53 static void
  54 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id);
  55 static int
  56 rxp_write_rules_via_cp(struct ibv_context *ctx,
  57                        struct mlx5_rxp_rof_entry *rules,
  58                        int count, uint8_t id);
  59 static int
  60 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules,
  61                 int count, uint8_t id);
  62 static int
  63 rxp_start_engine(struct ibv_context *ctx, uint8_t id);
  64 static int
  65 rxp_stop_engine(struct ibv_context *ctx, uint8_t id);
  66
  67 static void __rte_unused
  68 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused)
  69 {
  70         uint32_t reg, i;
  71
  72         /* Main CSRs*/
  73         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  74                 if (mlx5_devx_regex_register_read(ctx, id,
  75                                                   (MLX5_RXP_CSR_WIDTH * i) +
  76                                                   MLX5_RXP_CSR_BASE_ADDRESS,
  77                                                   &reg)) {
  78                         DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id);
  79                         return;
  80                 }
  81                 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x",
  82                         id, i, reg);
  83         }
  84         /* RTRU CSRs*/
  85         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  86                 if (mlx5_devx_regex_register_read(ctx, id,
  87                                                   (MLX5_RXP_CSR_WIDTH * i) +
  88                                                  MLX5_RXP_RTRU_CSR_BASE_ADDRESS,
  89                                                   &reg)) {
  90                         DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id);
  91                         return;
  92                 }
  93                 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x",
  94                         id, i, reg);
  95         }
  96         /* STAT CSRs */
  97         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  98                 if (mlx5_devx_regex_register_read(ctx, id,
  99                                                   (MLX5_RXP_CSR_WIDTH * i) +
 100                                                 MLX5_RXP_STATS_CSR_BASE_ADDRESS,
 101                                                   &reg)) {
 102                         DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id);
 103                         return;
 104                 }
 105                 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x",
 106                         id, i, reg);
 107         }
 108 }
 109
 110 int
 111 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused,
 112                     struct rte_regexdev_info *info)
 113 {
 114         info->max_matches = MLX5_REGEX_MAX_MATCHES;
 115         info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE;
 116         info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP;
 117         info->max_groups = MLX5_REGEX_MAX_GROUPS;
 118         info->max_queue_pairs = 1;
 119         info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F;
 120         info->rule_flags = 0;
 121         return 0;
 122 }
 123
 124 /**
 125  * Actual writing of RXP instructions to RXP via CSRs.
 126  */
 127 static int
 128 rxp_write_rules_via_cp(struct ibv_context *ctx,
 129                        struct mlx5_rxp_rof_entry *rules,
 130                        int count, uint8_t id)
 131 {
 132         int i, ret = 0;
 133         uint32_t tmp;
 134
 135         for (i = 0; i < count; i++) {
 136                 tmp = (uint32_t)rules[i].value;
 137                 ret |= mlx5_devx_regex_register_write(ctx, id,
 138                                                       MLX5_RXP_RTRU_CSR_DATA_0,
 139                                                       tmp);
 140                 tmp = (uint32_t)(rules[i].value >> 32);
 141                 ret |= mlx5_devx_regex_register_write(ctx, id,
 142                                                       MLX5_RXP_RTRU_CSR_DATA_0 +
 143                                                       MLX5_RXP_CSR_WIDTH, tmp);
 144                 tmp = rules[i].addr;
 145                 ret |= mlx5_devx_regex_register_write(ctx, id,
 146                                                       MLX5_RXP_RTRU_CSR_ADDR,
 147                                                       tmp);
 148                 if (ret) {
 149                         DRV_LOG(ERR, "Failed to copy instructions to RXP.");
 150                         return -1;
 151                 }
 152         }
 153         DRV_LOG(DEBUG, "Written %d instructions", count);
 154         return 0;
 155 }
 156
 157 static int
 158 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules,
 159                 int count, uint8_t id)
 160 {
 161         uint32_t val, fifo_depth;
 162         int ret;
 163
 164         ret = rxp_write_rules_via_cp(ctx, rules, count, id);
 165         if (ret < 0) {
 166                 DRV_LOG(ERR, "Failed to write rules via CSRs.");
 167                 return -1;
 168         }
 169         ret = mlx5_devx_regex_register_read(ctx, id,
 170                                             MLX5_RXP_RTRU_CSR_CAPABILITY,
 171                                             &fifo_depth);
 172         if (ret) {
 173                 DRV_LOG(ERR, "CSR read failed!");
 174                 return -1;
 175         }
 176         ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_FIFO_STAT,
 177                                      count, ~0,
 178                                      MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 179         if (ret < 0) {
 180                 DRV_LOG(ERR, "Rules not rx by RXP: credit: %d, depth: %d", val,
 181                         fifo_depth);
 182                 return ret;
 183         }
 184         DRV_LOG(DEBUG, "RTRU FIFO depth: 0x%x", fifo_depth);
 185         DRV_LOG(DEBUG, "Rules flush took %d cycles.", ret);
 186         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 187                                             &val);
 188         if (ret) {
 189                 DRV_LOG(ERR, "CSR read failed!");
 190                 return -1;
 191         }
 192         val |= MLX5_RXP_RTRU_CSR_CTRL_GO;
 193         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 194                                              val);
 195         ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_STATUS,
 196                                      MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 197                                      MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 198                                      MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 199         if (ret < 0) {
 200                 DRV_LOG(ERR, "Rules update timeout: 0x%08X", val);
 201                 return ret;
 202         }
 203         DRV_LOG(DEBUG, "Rules update took %d cycles", ret);
 204         if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 205                                           &val)) {
 206                 DRV_LOG(ERR, "CSR read failed!");
 207                 return -1;
 208         }
 209         val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO);
 210         if (mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 211                                            val)) {
 212                 DRV_LOG(ERR, "CSR write write failed!");
 213                 return -1;
 214         }
 215
 216         DRV_LOG(DEBUG, "RXP Flush rules finished.");
 217         return 0;
 218 }
 219
 220 static int
 221 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
 222                        uint32_t address, uint32_t expected_value,
 223                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id)
 224 {
 225         unsigned int i;
 226         int ret = 0;
 227
 228         ret = -EBUSY;
 229         for (i = 0; i < timeout_ms; i++) {
 230                 if (mlx5_devx_regex_register_read(ctx, id, address, value))
 231                         return -1;
 232                 if ((*value & expected_mask) == expected_value) {
 233                         ret = 0;
 234                         break;
 235                 }
 236                 rte_delay_us(1000);
 237         }
 238         return ret;
 239 }
 240
 241 static int
 242 rxp_start_engine(struct ibv_context *ctx, uint8_t id)
 243 {
 244         uint32_t ctrl;
 245         int ret;
 246
 247         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 248         if (ret)
 249                 return ret;
 250         ctrl |= MLX5_RXP_CSR_CTRL_GO;
 251         ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C;
 252         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 253         return ret;
 254 }
 255
 256 static int
 257 rxp_stop_engine(struct ibv_context *ctx, uint8_t id)
 258 {
 259         uint32_t ctrl;
 260         int ret;
 261
 262         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 263         if (ret)
 264                 return ret;
 265         ctrl &= ~MLX5_RXP_CSR_CTRL_GO;
 266         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 267         return ret;
 268 }
 269
 270 static int
 271 rxp_init_rtru(struct ibv_context *ctx, uint8_t id, uint32_t init_bits)
 272 {
 273         uint32_t ctrl_value;
 274         uint32_t poll_value;
 275         uint32_t expected_value;
 276         uint32_t expected_mask;
 277         int ret = 0;
 278
 279         /* Read the rtru ctrl CSR. */
 280         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 281                                             &ctrl_value);
 282         if (ret)
 283                 return -1;
 284         /* Clear any previous init modes. */
 285         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK);
 286         if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) {
 287                 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 288                 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 289                                                ctrl_value);
 290         }
 291         /* Set the init_mode bits in the rtru ctrl CSR. */
 292         ctrl_value |= init_bits;
 293         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 294                                        ctrl_value);
 295         /* Need to sleep for a short period after pulsing the rtru init bit. */
 296         rte_delay_us(20000);
 297         /* Poll the rtru status CSR until all the init done bits are set. */
 298         DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init");
 299         /* Set the init bit in the rtru ctrl CSR. */
 300         ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT;
 301         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 302                                        ctrl_value);
 303         /* Clear the init bit in the rtru ctrl CSR */
 304         ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT;
 305         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 306                                        ctrl_value);
 307         /* Check that the following bits are set in the RTRU_CSR. */
 308         if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) {
 309                 /* Must be incremental mode */
 310                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
 311                         MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 312         } else {
 313                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE |
 314                         MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
 315                         MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 316         }
 317         expected_mask = expected_value;
 318         ret = rxp_poll_csr_for_value(ctx, &poll_value,
 319                                      MLX5_RXP_RTRU_CSR_STATUS,
 320                                      expected_value, expected_mask,
 321                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 322         if (ret)
 323                 return ret;
 324         DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value);
 325         /* Clear the init bit in the rtru ctrl CSR */
 326         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 327         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 328                                        ctrl_value);
 329         return 0;
 330 }
 331
 332 static int
 333 rxp_parse_rof(const char *buf, uint32_t len,
 334               struct mlx5_rxp_ctl_rules_pgm **rules)
 335 {
 336         static const char del[] = "\n\r";
 337         char *line;
 338         char *tmp;
 339         char *cur_pos;
 340         uint32_t lines = 0;
 341         uint32_t entries;
 342         struct mlx5_rxp_rof_entry *curentry;
 343
 344         tmp = rte_malloc("", len, 0);
 345         if (!tmp)
 346                 return -ENOMEM;
 347         memcpy(tmp, buf, len);
 348         line = strtok(tmp, del);
 349         while (line) {
 350                 if (line[0] != '#' && line[0] != '\0')
 351                         lines++;
 352                 line = strtok(NULL, del);
 353         }
 354         *rules = rte_malloc("", lines * sizeof(*curentry) + sizeof(**rules), 0);
 355         if (!(*rules)) {
 356                 rte_free(tmp);
 357                 return -ENOMEM;
 358         }
 359         memset(*rules, 0, lines * sizeof(curentry) + sizeof(**rules));
 360         curentry = (*rules)->rules;
 361         (*rules)->hdr.cmd = MLX5_RXP_CTL_RULES_PGM;
 362         entries = 0;
 363         memcpy(tmp, buf, len);
 364         line = strtok(tmp, del);
 365         while (line) {
 366                 if (line[0] == '#' || line[0] == '\0') {
 367                         line = strtok(NULL, del);
 368                         continue;
 369                 }
 370                 curentry->type = strtoul(line, &cur_pos, 10);
 371                 if (cur_pos == line || cur_pos[0] != ',')
 372                         goto parse_error;
 373                 cur_pos++;
 374                 curentry->addr = strtoul(cur_pos, &cur_pos, 16);
 375                 if (cur_pos[0] != ',')
 376                         goto parse_error;
 377                 cur_pos++;
 378                 curentry->value = strtoull(cur_pos, &cur_pos, 16);
 379                 if (cur_pos[0] != '\0' && cur_pos[0] != '\n')
 380                         goto parse_error;
 381                 curentry++;
 382                 entries++;
 383                 if (entries > lines)
 384                         goto parse_error;
 385                 line = strtok(NULL, del);
 386         }
 387         (*rules)->count = entries;
 388         (*rules)->hdr.len = entries * sizeof(*curentry) + sizeof(**rules);
 389         rte_free(tmp);
 390         return 0;
 391 parse_error:
 392         rte_free(tmp);
 393         if (*rules)
 394                 rte_free(*rules);
 395         return -EINVAL;
 396 }
 397
 398 static int
 399 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use)
 400 {
 401         int ret;
 402         uint32_t umem_id;
 403
 404         ret = mlx5_devx_regex_database_stop(priv->ctx, id);
 405         if (ret < 0) {
 406                 DRV_LOG(ERR, "stop engine failed!");
 407                 return ret;
 408         }
 409         umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem);
 410         ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0);
 411         if (ret < 0) {
 412                 DRV_LOG(ERR, "program db failed!");
 413                 return ret;
 414         }
 415         return 0;
 416 }
 417
 418 static int
 419 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id)
 420 {
 421         mlx5_devx_regex_database_resume(priv->ctx, id);
 422         return 0;
 423 }
 424
 425 /*
 426  * Assign db memory for RXP programming.
 427  */
 428 static int
 429 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id)
 430 {
 431         unsigned int i;
 432         uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED;
 433         uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED;
 434
 435         /* Check which database rxp_eng is currently located if any? */
 436         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 437              i++) {
 438                 if (priv->db[i].db_assigned_to_eng_num == id) {
 439                         eng_assigned = i;
 440                         break;
 441                 }
 442         }
 443         /*
 444          * If private mode then, we can keep the same db ptr as RXP will be
 445          * programming EM itself if necessary, however need to see if
 446          * programmed yet.
 447          */
 448         if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) &&
 449             (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED))
 450                 return eng_assigned;
 451         /* Check for inactive db memory to use. */
 452         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 453              i++) {
 454                 if (priv->db[i].active == true)
 455                         continue; /* Already in use, so skip db. */
 456                 /* Set this db to active now as free to use. */
 457                 priv->db[i].active = true;
 458                 /* Now unassign last db index in use by RXP Eng. */
 459                 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) {
 460                         priv->db[eng_assigned].active = false;
 461                         priv->db[eng_assigned].db_assigned_to_eng_num =
 462                                 MLX5_RXP_DB_NOT_ASSIGNED;
 463
 464                         /* Set all DB memory to 0's before setting up DB. */
 465                         memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 466                 }
 467                 /* Now reassign new db index with RXP Engine. */
 468                 priv->db[i].db_assigned_to_eng_num = id;
 469                 db_free = i;
 470                 break;
 471         }
 472         if (db_free == MLX5_RXP_DB_NOT_ASSIGNED)
 473                 return -1;
 474         return db_free;
 475 }
 476
 477 /*
 478  * Program RXP instruction db to RXP engine/s.
 479  */
 480 static int
 481 program_rxp_rules(struct mlx5_regex_priv *priv,
 482                   struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id)
 483 {
 484         int ret, db_free;
 485         uint32_t rule_cnt;
 486
 487         rule_cnt = rules->count;
 488         db_free = mlnx_update_database(priv, id);
 489         if (db_free < 0) {
 490                 DRV_LOG(ERR, "Failed to setup db memory!");
 491                 return db_free;
 492         }
 493         if (priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) {
 494                 /* Register early to ensure RXP writes to EM use valid addr. */
 495                 ret = mlnx_set_database(priv, id, db_free);
 496                 if (ret < 0) {
 497                         DRV_LOG(ERR, "Failed to register db memory!");
 498                         return ret;
 499                 }
 500         }
 501         ret = write_private_rules(priv, rules, id);
 502         if (ret < 0) {
 503                 DRV_LOG(ERR, "Failed to write rules!");
 504                 return ret;
 505         }
 506         if (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) {
 507                 /* Write external rules directly to EM. */
 508                 rules->count = rule_cnt;
 509                /* Now write external instructions to EM. */
 510                 ret = write_shared_rules(priv, rules, rules->hdr.len, db_free);
 511                 if (ret < 0) {
 512                         DRV_LOG(ERR, "Failed to write EM rules!");
 513                         return ret;
 514                 }
 515                 ret = mlnx_set_database(priv, id, db_free);
 516                 if (ret < 0) {
 517                         DRV_LOG(ERR, "Failed to register db memory!");
 518                         return ret;
 519                 }
 520         }
 521         ret = mlnx_resume_database(priv, id);
 522         if (ret < 0) {
 523                 DRV_LOG(ERR, "Failed to resume engine!");
 524                 return ret;
 525         }
 526         DRV_LOG(DEBUG, "Programmed RXP Engine %d\n", id);
 527         rules->count = rule_cnt;
 528         return 0;
 529 }
 530
 531 static int
 532 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id)
 533 {
 534         uint32_t ctrl;
 535         uint32_t reg;
 536         struct ibv_context *ctx = priv->ctx;
 537         int ret;
 538
 539         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 540         if (ret)
 541                 return ret;
 542         if (ctrl & MLX5_RXP_CSR_CTRL_INIT) {
 543                 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 544                 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 545                                                      ctrl);
 546                 if (ret)
 547                         return ret;
 548         }
 549         ctrl |= MLX5_RXP_CSR_CTRL_INIT;
 550         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 551         if (ret)
 552                 return ret;
 553         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 554         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 555         rte_delay_us(20000);
 556         ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS,
 557                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 558                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 559                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 560         if (ret)
 561                 return ret;
 562         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 563         if (ret)
 564                 return ret;
 565         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 566         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 567                                              ctrl);
 568         if (ret)
 569                 return ret;
 570         ret = rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2);
 571         if (ret)
 572                 return ret;
 573         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5,
 574                                             &reg);
 575         if (ret)
 576                 return ret;
 577         DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16,
 578                 reg & 0xffff);
 579         if ((reg >> 16) >= priv->nb_max_matches)
 580                 ret = mlx5_devx_regex_register_write(ctx, id,
 581                                                      MLX5_RXP_CSR_MAX_MATCH,
 582                                                      priv->nb_max_matches);
 583         else
 584                 ret = mlx5_devx_regex_register_write(ctx, id,
 585                                                      MLX5_RXP_CSR_MAX_MATCH,
 586                                                      (reg >> 16));
 587         ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX,
 588                                          (reg & 0xFFFF));
 589         ret |= mlx5_devx_regex_register_write(ctx, id,
 590                                               MLX5_RXP_CSR_MAX_LATENCY, 0);
 591         ret |= mlx5_devx_regex_register_write(ctx, id,
 592                                               MLX5_RXP_CSR_MAX_PRI_THREAD, 0);
 593         return ret;
 594 }
 595
 596 static int
 597 write_private_rules(struct mlx5_regex_priv *priv,
 598                     struct mlx5_rxp_ctl_rules_pgm *rules,
 599                     uint8_t id)
 600 {
 601         unsigned int pending;
 602         uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries;
 603         int ret = 1;
 604
 605         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 606                 return -EINVAL;
 607         if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM ||
 608                                    rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR)
 609                 return -EINVAL;
 610         /* For a non-incremental rules program, re-init the RXP. */
 611         if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) {
 612                 ret = rxp_init_eng(priv, id);
 613                 if (ret < 0)
 614                         return ret;
 615         } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) {
 616                 /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */
 617                 ret = rxp_init_rtru(priv->ctx, id,
 618                                     MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
 619                 if (ret < 0)
 620                         return ret;
 621         }
 622         if (rules->count == 0)
 623                 return -EINVAL;
 624         /* Confirm the RXP is initialised. */
 625         if (mlx5_devx_regex_register_read(priv->ctx, id,
 626                                             MLX5_RXP_CSR_STATUS, &val)) {
 627                 DRV_LOG(ERR, "Failed to read from RXP!");
 628                 return -ENODEV;
 629         }
 630         if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) {
 631                 DRV_LOG(ERR, "RXP not initialised...");
 632                 return -EBUSY;
 633         }
 634         /* Get the RTRU maximum number of entries allowed. */
 635         if (mlx5_devx_regex_register_read(priv->ctx, id,
 636                         MLX5_RXP_RTRU_CSR_CAPABILITY, &rtru_max_num_entries)) {
 637                 DRV_LOG(ERR, "Failed to read RTRU capability!");
 638                 return -ENODEV;
 639         }
 640         rtru_max_num_entries = (rtru_max_num_entries & 0x00FF);
 641         rule_cnt = 0;
 642         pending = 0;
 643         while (rules->count > 0) {
 644                 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_INST) ||
 645                     (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_IM) ||
 646                     (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM)) {
 647                         if ((rules->rules[rule_cnt].type ==
 648                              MLX5_RXP_ROF_ENTRY_EM) &&
 649                             (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
 650                                 /* Skip EM rules programming. */
 651                                 if (pending > 0) {
 652                                         /* Flush any rules that are pending. */
 653                                         rule_offset = (rule_cnt - pending);
 654                                         ret = rxp_flush_rules(priv->ctx,
 655                                                 &rules->rules[rule_offset],
 656                                                 pending, id);
 657                                         if (ret < 0) {
 658                                                 DRV_LOG(ERR, "Flushing rules.");
 659                                                 return -ENODEV;
 660                                         }
 661                                         pending = 0;
 662                                 }
 663                                 rule_cnt++;
 664                         } else {
 665                                 pending++;
 666                                 rule_cnt++;
 667                                 /*
 668                                  * If parsing the last rule, or if reached the
 669                                  * maximum number of rules for this batch, then
 670                                  * flush the rules batch to the RXP.
 671                                  */
 672                                 if ((rules->count == 1) ||
 673                                     (pending == rtru_max_num_entries)) {
 674                                         rule_offset = (rule_cnt - pending);
 675                                         ret = rxp_flush_rules(priv->ctx,
 676                                                 &rules->rules[rule_offset],
 677                                                 pending, id);
 678                                         if (ret < 0) {
 679                                                 DRV_LOG(ERR, "Flushing rules.");
 680                                                 return -ENODEV;
 681                                         }
 682                                         pending = 0;
 683                                 }
 684                         }
 685                 } else if ((rules->rules[rule_cnt].type ==
 686                                 MLX5_RXP_ROF_ENTRY_EQ) ||
 687                          (rules->rules[rule_cnt].type ==
 688                                 MLX5_RXP_ROF_ENTRY_GTE) ||
 689                          (rules->rules[rule_cnt].type ==
 690                                 MLX5_RXP_ROF_ENTRY_LTE) ||
 691                          (rules->rules[rule_cnt].type ==
 692                                 MLX5_RXP_ROF_ENTRY_CHECKSUM) ||
 693                          (rules->rules[rule_cnt].type ==
 694                                 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM)) {
 695                         if (pending) {
 696                                 /* Flush rules before checking reg values. */
 697                                 rule_offset = (rule_cnt - pending);
 698                                 ret = rxp_flush_rules(priv->ctx,
 699                                         &rules->rules[rule_offset],
 700                                         pending, id);
 701                                 if (ret < 0) {
 702                                         DRV_LOG(ERR, "Failed to flush rules.");
 703                                         return -ENODEV;
 704                                 }
 705                         }
 706                         block = (rules->rules[rule_cnt].addr >> 16) & 0xFFFF;
 707                         if (block == 0)
 708                                 reg = MLX5_RXP_CSR_BASE_ADDRESS;
 709                         else if (block == 1)
 710                                 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS;
 711                         else {
 712                                 DRV_LOG(ERR, "Invalid ROF register 0x%08X!",
 713                                         rules->rules[rule_cnt].addr);
 714                                 return -EINVAL;
 715                         }
 716                         reg += (rules->rules[rule_cnt].addr & 0xFFFF) *
 717                                 MLX5_RXP_CSR_WIDTH;
 718                         ret = mlx5_devx_regex_register_read(priv->ctx, id,
 719                                                             reg, &val);
 720                         if (ret) {
 721                                 DRV_LOG(ERR, "RXP CSR read failed!");
 722                                 return ret;
 723                         }
 724                         if ((priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) &&
 725                             ((rules->rules[rule_cnt].type ==
 726                             MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM) &&
 727                             (val != rules->rules[rule_cnt].value))) {
 728                                 DRV_LOG(ERR, "Unexpected value for register:");
 729                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 730                                         rules->rules[rule_cnt].addr, val);
 731                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 732                                         rules->rules[rule_cnt].value);
 733                                         return -EINVAL;
 734                         } else if ((priv->prog_mode ==
 735                                  MLX5_RXP_PRIVATE_PROG_MODE) &&
 736                                  (rules->rules[rule_cnt].type ==
 737                                  MLX5_RXP_ROF_ENTRY_CHECKSUM) &&
 738                                  (val != rules->rules[rule_cnt].value)) {
 739                                 DRV_LOG(ERR, "Unexpected value for register:");
 740                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 741                                         rules->rules[rule_cnt].addr, val);
 742                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 743                                         rules->rules[rule_cnt].value);
 744                                 return -EINVAL;
 745                         } else if ((rules->rules[rule_cnt].type ==
 746                                         MLX5_RXP_ROF_ENTRY_EQ) &&
 747                                   (val != rules->rules[rule_cnt].value)) {
 748                                 DRV_LOG(ERR, "Unexpected value for register:");
 749                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 750                                         rules->rules[rule_cnt].addr, val);
 751                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 752                                         rules->rules[rule_cnt].value);
 753                                         return -EINVAL;
 754                         } else if ((rules->rules[rule_cnt].type ==
 755                                         MLX5_RXP_ROF_ENTRY_GTE) &&
 756                                  (val < rules->rules[rule_cnt].value)) {
 757                                 DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
 758                                         rules->rules[rule_cnt].addr);
 759                                 DRV_LOG(ERR, "got %X, expected >= %" PRIx64 ".",
 760                                         val, rules->rules[rule_cnt].value);
 761                                 return -EINVAL;
 762                         } else if ((rules->rules[rule_cnt].type ==
 763                                         MLX5_RXP_ROF_ENTRY_LTE) &&
 764                                  (val > rules->rules[rule_cnt].value)) {
 765                                 DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
 766                                         rules->rules[rule_cnt].addr);
 767                                 DRV_LOG(ERR, "got %08X expected <= %" PRIx64,
 768                                         val, rules->rules[rule_cnt].value);
 769                                 return -EINVAL;
 770                         }
 771                         rule_cnt++;
 772                         pending = 0;
 773                 } else {
 774                         DRV_LOG(ERR, "Error: Invalid rule type %d!",
 775                                 rules->rules[rule_cnt].type);
 776                         return -EINVAL;
 777                 }
 778                 rules->count--;
 779         }
 780         return ret;
 781 }
 782
 783 /*
 784  * Shared memory programming mode, here all external db instructions are written
 785  * to EM via the host.
 786  */
 787 static int
 788 write_shared_rules(struct mlx5_regex_priv *priv,
 789                    struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
 790                    uint8_t db_to_program)
 791 {
 792         uint32_t rule_cnt, rof_rule_addr;
 793         uint64_t tmp_write_swap[4];
 794
 795         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 796                 return -EINVAL;
 797         if ((rules->count == 0) || (count == 0))
 798                 return -EINVAL;
 799         rule_cnt = 0;
 800         /*
 801          * Note the following section of code carries out a 32byte swap of
 802          * instruction to coincide with HW 32byte swap. This may need removed
 803          * in new variants of this programming function!
 804          */
 805         while (rule_cnt < rules->count) {
 806                 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM) &&
 807                     (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
 808                         /*
 809                          * Note there are always blocks of 8 instructions for
 810                          * 7's written sequentially. However there is no
 811                          * guarantee that all blocks are sequential!
 812                          */
 813                         if (count >= (rule_cnt + MLX5_RXP_INST_BLOCK_SIZE)) {
 814                                 /*
 815                                  * Ensure memory write not exceeding boundary
 816                                  * Check essential to ensure 0x10000 offset
 817                                  * accounted for!
 818                                  */
 819                                 if ((uint8_t *)((uint8_t *)
 820                                     priv->db[db_to_program].ptr +
 821                                     ((rules->rules[rule_cnt + 7].addr <<
 822                                     MLX5_RXP_INST_OFFSET))) >=
 823                                     ((uint8_t *)((uint8_t *)
 824                                     priv->db[db_to_program].ptr +
 825                                     MLX5_MAX_DB_SIZE))) {
 826                                         DRV_LOG(ERR, "DB exceeded memory!");
 827                                         return -ENODEV;
 828                                 }
 829                                 /*
 830                                  * Rule address Offset to align with RXP
 831                                  * external instruction offset.
 832                                  */
 833                                 rof_rule_addr = (rules->rules[rule_cnt].addr <<
 834                                                  MLX5_RXP_INST_OFFSET);
 835                                 /* 32 byte instruction swap (sw work around)! */
 836                                 tmp_write_swap[0] = le64toh(
 837                                         rules->rules[(rule_cnt + 4)].value);
 838                                 tmp_write_swap[1] = le64toh(
 839                                         rules->rules[(rule_cnt + 5)].value);
 840                                 tmp_write_swap[2] = le64toh(
 841                                         rules->rules[(rule_cnt + 6)].value);
 842                                 tmp_write_swap[3] = le64toh(
 843                                         rules->rules[(rule_cnt + 7)].value);
 844                                 /* Write only 4 of the 8 instructions. */
 845                                 memcpy((uint8_t *)((uint8_t *)
 846                                        priv->db[db_to_program].ptr +
 847                                        rof_rule_addr), &tmp_write_swap,
 848                                        (sizeof(uint64_t) * 4));
 849                                 /* Write 1st 4 rules of block after last 4. */
 850                                 rof_rule_addr = (rules->rules[
 851                                                  (rule_cnt + 4)].addr <<
 852                                                  MLX5_RXP_INST_OFFSET);
 853                                 tmp_write_swap[0] = le64toh(
 854                                         rules->rules[(rule_cnt + 0)].value);
 855                                 tmp_write_swap[1] = le64toh(
 856                                         rules->rules[(rule_cnt + 1)].value);
 857                                 tmp_write_swap[2] = le64toh(
 858                                         rules->rules[(rule_cnt + 2)].value);
 859                                 tmp_write_swap[3] = le64toh(
 860                                         rules->rules[(rule_cnt + 3)].value);
 861                                 memcpy((uint8_t *)((uint8_t *)
 862                                        priv->db[db_to_program].ptr +
 863                                        rof_rule_addr), &tmp_write_swap,
 864                                        (sizeof(uint64_t) * 4));
 865                         } else
 866                                 return -1;
 867                         /* Fast forward as already handled block of 8. */
 868                         rule_cnt += MLX5_RXP_INST_BLOCK_SIZE;
 869                 } else
 870                         rule_cnt++; /* Must be something other than EM rule. */
 871         }
 872         return 0;
 873 }
 874
 875 static int
 876 rxp_db_setup(struct mlx5_regex_priv *priv)
 877 {
 878         int ret;
 879         uint8_t i;
 880
 881         /* Setup database memories for both RXP engines + reprogram memory. */
 882         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 883                 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 0);
 884                 if (!priv->db[i].ptr) {
 885                         DRV_LOG(ERR, "Failed to alloc db memory!");
 886                         ret = ENODEV;
 887                         goto tidyup_error;
 888                 }
 889                 /* Register the memory. */
 890                 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx,
 891                                                         priv->db[i].ptr,
 892                                                         MLX5_MAX_DB_SIZE, 7);
 893                 if (!priv->db[i].umem.umem) {
 894                         DRV_LOG(ERR, "Failed to register memory!");
 895                         ret = ENODEV;
 896                         goto tidyup_error;
 897                 }
 898                 /* Ensure set all DB memory to 0's before setting up DB. */
 899                 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 900                 /* No data currently in database. */
 901                 priv->db[i].len = 0;
 902                 priv->db[i].active = false;
 903                 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED;
 904         }
 905         return 0;
 906 tidyup_error:
 907         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 908                 if (priv->db[i].ptr)
 909                         rte_free(priv->db[i].ptr);
 910                 if (priv->db[i].umem.umem)
 911                         mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem);
 912         }
 913         return -ret;
 914 }
 915
 916 int
 917 mlx5_regex_rules_db_import(struct rte_regexdev *dev,
 918                      const char *rule_db, uint32_t rule_db_len)
 919 {
 920         struct mlx5_regex_priv *priv = dev->data->dev_private;
 921         struct mlx5_rxp_ctl_rules_pgm *rules = NULL;
 922         uint8_t id;
 923         int ret;
 924
 925         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
 926                 DRV_LOG(ERR, "RXP programming mode not set!");
 927                 return -1;
 928         }
 929         if (rule_db == NULL) {
 930                 DRV_LOG(ERR, "Database empty!");
 931                 return -ENODEV;
 932         }
 933         if (rule_db_len == 0)
 934                 return -EINVAL;
 935         ret = rxp_parse_rof(rule_db, rule_db_len, &rules);
 936         if (ret) {
 937                 DRV_LOG(ERR, "Can't parse ROF file.");
 938                 return ret;
 939         }
 940         /* Need to ensure RXP not busy before stop! */
 941         for (id = 0; id < priv->nb_engines; id++) {
 942                 ret = rxp_stop_engine(priv->ctx, id);
 943                 if (ret) {
 944                         DRV_LOG(ERR, "Can't stop engine.");
 945                         ret = -ENODEV;
 946                         goto tidyup_error;
 947                 }
 948                 ret = program_rxp_rules(priv, rules, id);
 949                 if (ret < 0) {
 950                         DRV_LOG(ERR, "Failed to program rxp rules.");
 951                         ret = -ENODEV;
 952                         goto tidyup_error;
 953                 }
 954                 ret = rxp_start_engine(priv->ctx, id);
 955                 if (ret) {
 956                         DRV_LOG(ERR, "Can't start engine.");
 957                         ret = -ENODEV;
 958                         goto tidyup_error;
 959                 }
 960         }
 961         rte_free(rules);
 962         return 0;
 963 tidyup_error:
 964         rte_free(rules);
 965         return ret;
 966 }
 967
 968 int
 969 mlx5_regex_configure(struct rte_regexdev *dev,
 970                      const struct rte_regexdev_config *cfg)
 971 {
 972         struct mlx5_regex_priv *priv = dev->data->dev_private;
 973         int ret;
 974
 975         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 976                 return -1;
 977         priv->nb_queues = cfg->nb_queue_pairs;
 978         dev->data->dev_conf.nb_queue_pairs = priv->nb_queues;
 979         priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) *
 980                                 priv->nb_queues, 0);
 981         if (!priv->nb_queues) {
 982                 DRV_LOG(ERR, "can't allocate qps memory");
 983                 rte_errno = ENOMEM;
 984                 return -rte_errno;
 985         }
 986         priv->nb_max_matches = cfg->nb_max_matches;
 987         /* Setup rxp db memories. */
 988         if (rxp_db_setup(priv)) {
 989                 DRV_LOG(ERR, "Failed to setup RXP db memory");
 990                 rte_errno = ENOMEM;
 991                 return -rte_errno;
 992         }
 993         if (cfg->rule_db != NULL) {
 994                 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db,
 995                                                  cfg->rule_db_len);
 996                 if (ret < 0) {
 997                         DRV_LOG(ERR, "Failed to program rxp rules.");
 998                         rte_errno = ENODEV;
 999                         goto configure_error;
1000                 }
1001         } else
1002                 DRV_LOG(DEBUG, "Regex config without rules programming!");
1003         return 0;
1004 configure_error:
1005         if (priv->qps)
1006                 rte_free(priv->qps);
1007         return -rte_errno;
1008 }