drivers/regex/mlx5/mlx5_rxp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_log.h>
   6 #include <rte_errno.h>
   7 #include <rte_malloc.h>
   8 #include <rte_regexdev.h>
   9 #include <rte_regexdev_core.h>
  10 #include <rte_regexdev_driver.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_devx_cmds.h>
  14 #include <mlx5_prm.h>
  15 #include <mlx5_common_os.h>
  16
  17 #include "mlx5_regex.h"
  18 #include "mlx5_regex_utils.h"
  19 #include "mlx5_rxp_csrs.h"
  20 #include "mlx5_rxp.h"
  21
  22 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES
  23 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH
  24 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX
  25 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS
  26
  27 /* Private Declarations */
  28 static int
  29 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
  30                        uint32_t address, uint32_t expected_value,
  31                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id);
  32 static int
  33 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use);
  34 static int
  35 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id);
  36 static int
  37 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id);
  38 static int
  39 program_rxp_rules(struct mlx5_regex_priv *priv,
  40                   struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id);
  41 static int
  42 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id);
  43 static int
  44 write_private_rules(struct mlx5_regex_priv *priv,
  45                     struct mlx5_rxp_ctl_rules_pgm *rules,
  46                     uint8_t id);
  47 static int
  48 write_shared_rules(struct mlx5_regex_priv *priv,
  49                    struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
  50                    uint8_t db_to_program);
  51 static int
  52 rxp_db_setup(struct mlx5_regex_priv *priv);
  53 static void
  54 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id);
  55 static int
  56 rxp_write_rules_via_cp(struct ibv_context *ctx,
  57                        struct mlx5_rxp_rof_entry *rules,
  58                        int count, uint8_t id);
  59 static int
  60 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules,
  61                 int count, uint8_t id);
  62 static int
  63 rxp_start_engine(struct ibv_context *ctx, uint8_t id);
  64 static int
  65 rxp_stop_engine(struct ibv_context *ctx, uint8_t id);
  66
  67 static void __rte_unused
  68 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused)
  69 {
  70         uint32_t reg, i;
  71
  72         /* Main CSRs*/
  73         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  74                 if (mlx5_devx_regex_register_read(ctx, id,
  75                                                   (MLX5_RXP_CSR_WIDTH * i) +
  76                                                   MLX5_RXP_CSR_BASE_ADDRESS,
  77                                                   &reg)) {
  78                         DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id);
  79                         return;
  80                 }
  81                 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x",
  82                         id, i, reg);
  83         }
  84         /* RTRU CSRs*/
  85         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  86                 if (mlx5_devx_regex_register_read(ctx, id,
  87                                                   (MLX5_RXP_CSR_WIDTH * i) +
  88                                                  MLX5_RXP_RTRU_CSR_BASE_ADDRESS,
  89                                                   &reg)) {
  90                         DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id);
  91                         return;
  92                 }
  93                 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x",
  94                         id, i, reg);
  95         }
  96         /* STAT CSRs */
  97         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  98                 if (mlx5_devx_regex_register_read(ctx, id,
  99                                                   (MLX5_RXP_CSR_WIDTH * i) +
 100                                                 MLX5_RXP_STATS_CSR_BASE_ADDRESS,
 101                                                   &reg)) {
 102                         DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id);
 103                         return;
 104                 }
 105                 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x",
 106                         id, i, reg);
 107         }
 108 }
 109
 110 int
 111 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused,
 112                     struct rte_regexdev_info *info)
 113 {
 114         info->max_matches = MLX5_REGEX_MAX_MATCHES;
 115         info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE;
 116         info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP;
 117         info->max_groups = MLX5_REGEX_MAX_GROUPS;
 118         info->max_queue_pairs = 1;
 119         info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F;
 120         info->rule_flags = 0;
 121         info->max_queue_pairs = 10;
 122         return 0;
 123 }
 124
 125 /**
 126  * Actual writing of RXP instructions to RXP via CSRs.
 127  */
 128 static int
 129 rxp_write_rules_via_cp(struct ibv_context *ctx,
 130                        struct mlx5_rxp_rof_entry *rules,
 131                        int count, uint8_t id)
 132 {
 133         int i, ret = 0;
 134         uint32_t tmp;
 135
 136         for (i = 0; i < count; i++) {
 137                 tmp = (uint32_t)rules[i].value;
 138                 ret |= mlx5_devx_regex_register_write(ctx, id,
 139                                                       MLX5_RXP_RTRU_CSR_DATA_0,
 140                                                       tmp);
 141                 tmp = (uint32_t)(rules[i].value >> 32);
 142                 ret |= mlx5_devx_regex_register_write(ctx, id,
 143                                                       MLX5_RXP_RTRU_CSR_DATA_0 +
 144                                                       MLX5_RXP_CSR_WIDTH, tmp);
 145                 tmp = rules[i].addr;
 146                 ret |= mlx5_devx_regex_register_write(ctx, id,
 147                                                       MLX5_RXP_RTRU_CSR_ADDR,
 148                                                       tmp);
 149                 if (ret) {
 150                         DRV_LOG(ERR, "Failed to copy instructions to RXP.");
 151                         return -1;
 152                 }
 153         }
 154         DRV_LOG(DEBUG, "Written %d instructions", count);
 155         return 0;
 156 }
 157
 158 static int
 159 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules,
 160                 int count, uint8_t id)
 161 {
 162         uint32_t val, fifo_depth;
 163         int ret;
 164
 165         ret = rxp_write_rules_via_cp(ctx, rules, count, id);
 166         if (ret < 0) {
 167                 DRV_LOG(ERR, "Failed to write rules via CSRs.");
 168                 return -1;
 169         }
 170         ret = mlx5_devx_regex_register_read(ctx, id,
 171                                             MLX5_RXP_RTRU_CSR_CAPABILITY,
 172                                             &fifo_depth);
 173         if (ret) {
 174                 DRV_LOG(ERR, "CSR read failed!");
 175                 return -1;
 176         }
 177         ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_FIFO_STAT,
 178                                      count, ~0,
 179                                      MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 180         if (ret < 0) {
 181                 DRV_LOG(ERR, "Rules not rx by RXP: credit: %d, depth: %d", val,
 182                         fifo_depth);
 183                 return ret;
 184         }
 185         DRV_LOG(DEBUG, "RTRU FIFO depth: 0x%x", fifo_depth);
 186         DRV_LOG(DEBUG, "Rules flush took %d cycles.", ret);
 187         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 188                                             &val);
 189         if (ret) {
 190                 DRV_LOG(ERR, "CSR read failed!");
 191                 return -1;
 192         }
 193         val |= MLX5_RXP_RTRU_CSR_CTRL_GO;
 194         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 195                                              val);
 196         ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_STATUS,
 197                                      MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 198                                      MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 199                                      MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 200         if (ret < 0) {
 201                 DRV_LOG(ERR, "Rules update timeout: 0x%08X", val);
 202                 return ret;
 203         }
 204         DRV_LOG(DEBUG, "Rules update took %d cycles", ret);
 205         if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 206                                           &val)) {
 207                 DRV_LOG(ERR, "CSR read failed!");
 208                 return -1;
 209         }
 210         val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO);
 211         if (mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 212                                            val)) {
 213                 DRV_LOG(ERR, "CSR write write failed!");
 214                 return -1;
 215         }
 216
 217         DRV_LOG(DEBUG, "RXP Flush rules finished.");
 218         return 0;
 219 }
 220
 221 static int
 222 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
 223                        uint32_t address, uint32_t expected_value,
 224                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id)
 225 {
 226         unsigned int i;
 227         int ret = 0;
 228
 229         ret = -EBUSY;
 230         for (i = 0; i < timeout_ms; i++) {
 231                 if (mlx5_devx_regex_register_read(ctx, id, address, value))
 232                         return -1;
 233                 if ((*value & expected_mask) == expected_value) {
 234                         ret = 0;
 235                         break;
 236                 }
 237                 rte_delay_us(1000);
 238         }
 239         return ret;
 240 }
 241
 242 static int
 243 rxp_start_engine(struct ibv_context *ctx, uint8_t id)
 244 {
 245         uint32_t ctrl;
 246         int ret;
 247
 248         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 249         if (ret)
 250                 return ret;
 251         ctrl |= MLX5_RXP_CSR_CTRL_GO;
 252         ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C;
 253         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 254         return ret;
 255 }
 256
 257 static int
 258 rxp_stop_engine(struct ibv_context *ctx, uint8_t id)
 259 {
 260         uint32_t ctrl;
 261         int ret;
 262
 263         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 264         if (ret)
 265                 return ret;
 266         ctrl &= ~MLX5_RXP_CSR_CTRL_GO;
 267         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 268         return ret;
 269 }
 270
 271 static int
 272 rxp_init_rtru(struct ibv_context *ctx, uint8_t id, uint32_t init_bits)
 273 {
 274         uint32_t ctrl_value;
 275         uint32_t poll_value;
 276         uint32_t expected_value;
 277         uint32_t expected_mask;
 278         int ret = 0;
 279
 280         /* Read the rtru ctrl CSR. */
 281         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 282                                             &ctrl_value);
 283         if (ret)
 284                 return -1;
 285         /* Clear any previous init modes. */
 286         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK);
 287         if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) {
 288                 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 289                 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 290                                                ctrl_value);
 291         }
 292         /* Set the init_mode bits in the rtru ctrl CSR. */
 293         ctrl_value |= init_bits;
 294         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 295                                        ctrl_value);
 296         /* Need to sleep for a short period after pulsing the rtru init bit. */
 297         rte_delay_us(20000);
 298         /* Poll the rtru status CSR until all the init done bits are set. */
 299         DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init");
 300         /* Set the init bit in the rtru ctrl CSR. */
 301         ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT;
 302         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 303                                        ctrl_value);
 304         /* Clear the init bit in the rtru ctrl CSR */
 305         ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT;
 306         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 307                                        ctrl_value);
 308         /* Check that the following bits are set in the RTRU_CSR. */
 309         if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) {
 310                 /* Must be incremental mode */
 311                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
 312                         MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 313         } else {
 314                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE |
 315                         MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
 316                         MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 317         }
 318         expected_mask = expected_value;
 319         ret = rxp_poll_csr_for_value(ctx, &poll_value,
 320                                      MLX5_RXP_RTRU_CSR_STATUS,
 321                                      expected_value, expected_mask,
 322                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 323         if (ret)
 324                 return ret;
 325         DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value);
 326         /* Clear the init bit in the rtru ctrl CSR */
 327         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 328         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 329                                        ctrl_value);
 330         return 0;
 331 }
 332
 333 static int
 334 rxp_parse_rof(const char *buf, uint32_t len,
 335               struct mlx5_rxp_ctl_rules_pgm **rules)
 336 {
 337         static const char del[] = "\n\r";
 338         char *line;
 339         char *tmp;
 340         char *cur_pos;
 341         uint32_t lines = 0;
 342         uint32_t entries;
 343         struct mlx5_rxp_rof_entry *curentry;
 344
 345         tmp = rte_malloc("", len, 0);
 346         if (!tmp)
 347                 return -ENOMEM;
 348         memcpy(tmp, buf, len);
 349         line = strtok(tmp, del);
 350         while (line) {
 351                 if (line[0] != '#' && line[0] != '\0')
 352                         lines++;
 353                 line = strtok(NULL, del);
 354         }
 355         *rules = rte_malloc("", lines * sizeof(*curentry) + sizeof(**rules), 0);
 356         if (!(*rules)) {
 357                 rte_free(tmp);
 358                 return -ENOMEM;
 359         }
 360         memset(*rules, 0, lines * sizeof(curentry) + sizeof(**rules));
 361         curentry = (*rules)->rules;
 362         (*rules)->hdr.cmd = MLX5_RXP_CTL_RULES_PGM;
 363         entries = 0;
 364         memcpy(tmp, buf, len);
 365         line = strtok(tmp, del);
 366         while (line) {
 367                 if (line[0] == '#' || line[0] == '\0') {
 368                         line = strtok(NULL, del);
 369                         continue;
 370                 }
 371                 curentry->type = strtoul(line, &cur_pos, 10);
 372                 if (cur_pos == line || cur_pos[0] != ',')
 373                         goto parse_error;
 374                 cur_pos++;
 375                 curentry->addr = strtoul(cur_pos, &cur_pos, 16);
 376                 if (cur_pos[0] != ',')
 377                         goto parse_error;
 378                 cur_pos++;
 379                 curentry->value = strtoull(cur_pos, &cur_pos, 16);
 380                 if (cur_pos[0] != '\0' && cur_pos[0] != '\n')
 381                         goto parse_error;
 382                 curentry++;
 383                 entries++;
 384                 if (entries > lines)
 385                         goto parse_error;
 386                 line = strtok(NULL, del);
 387         }
 388         (*rules)->count = entries;
 389         (*rules)->hdr.len = entries * sizeof(*curentry) + sizeof(**rules);
 390         rte_free(tmp);
 391         return 0;
 392 parse_error:
 393         rte_free(tmp);
 394         if (*rules)
 395                 rte_free(*rules);
 396         return -EINVAL;
 397 }
 398
 399 static int
 400 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use)
 401 {
 402         int ret;
 403         uint32_t umem_id;
 404
 405         ret = mlx5_devx_regex_database_stop(priv->ctx, id);
 406         if (ret < 0) {
 407                 DRV_LOG(ERR, "stop engine failed!");
 408                 return ret;
 409         }
 410         umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem);
 411         ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0);
 412         if (ret < 0) {
 413                 DRV_LOG(ERR, "program db failed!");
 414                 return ret;
 415         }
 416         return 0;
 417 }
 418
 419 static int
 420 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id)
 421 {
 422         mlx5_devx_regex_database_resume(priv->ctx, id);
 423         return 0;
 424 }
 425
 426 /*
 427  * Assign db memory for RXP programming.
 428  */
 429 static int
 430 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id)
 431 {
 432         unsigned int i;
 433         uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED;
 434         uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED;
 435
 436         /* Check which database rxp_eng is currently located if any? */
 437         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 438              i++) {
 439                 if (priv->db[i].db_assigned_to_eng_num == id) {
 440                         eng_assigned = i;
 441                         break;
 442                 }
 443         }
 444         /*
 445          * If private mode then, we can keep the same db ptr as RXP will be
 446          * programming EM itself if necessary, however need to see if
 447          * programmed yet.
 448          */
 449         if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) &&
 450             (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED))
 451                 return eng_assigned;
 452         /* Check for inactive db memory to use. */
 453         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 454              i++) {
 455                 if (priv->db[i].active == true)
 456                         continue; /* Already in use, so skip db. */
 457                 /* Set this db to active now as free to use. */
 458                 priv->db[i].active = true;
 459                 /* Now unassign last db index in use by RXP Eng. */
 460                 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) {
 461                         priv->db[eng_assigned].active = false;
 462                         priv->db[eng_assigned].db_assigned_to_eng_num =
 463                                 MLX5_RXP_DB_NOT_ASSIGNED;
 464
 465                         /* Set all DB memory to 0's before setting up DB. */
 466                         memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 467                 }
 468                 /* Now reassign new db index with RXP Engine. */
 469                 priv->db[i].db_assigned_to_eng_num = id;
 470                 db_free = i;
 471                 break;
 472         }
 473         if (db_free == MLX5_RXP_DB_NOT_ASSIGNED)
 474                 return -1;
 475         return db_free;
 476 }
 477
 478 /*
 479  * Program RXP instruction db to RXP engine/s.
 480  */
 481 static int
 482 program_rxp_rules(struct mlx5_regex_priv *priv,
 483                   struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id)
 484 {
 485         int ret, db_free;
 486         uint32_t rule_cnt;
 487
 488         rule_cnt = rules->count;
 489         db_free = mlnx_update_database(priv, id);
 490         if (db_free < 0) {
 491                 DRV_LOG(ERR, "Failed to setup db memory!");
 492                 return db_free;
 493         }
 494         if (priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) {
 495                 /* Register early to ensure RXP writes to EM use valid addr. */
 496                 ret = mlnx_set_database(priv, id, db_free);
 497                 if (ret < 0) {
 498                         DRV_LOG(ERR, "Failed to register db memory!");
 499                         return ret;
 500                 }
 501         }
 502         ret = write_private_rules(priv, rules, id);
 503         if (ret < 0) {
 504                 DRV_LOG(ERR, "Failed to write rules!");
 505                 return ret;
 506         }
 507         if (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) {
 508                 /* Write external rules directly to EM. */
 509                 rules->count = rule_cnt;
 510                /* Now write external instructions to EM. */
 511                 ret = write_shared_rules(priv, rules, rules->hdr.len, db_free);
 512                 if (ret < 0) {
 513                         DRV_LOG(ERR, "Failed to write EM rules!");
 514                         return ret;
 515                 }
 516                 ret = mlnx_set_database(priv, id, db_free);
 517                 if (ret < 0) {
 518                         DRV_LOG(ERR, "Failed to register db memory!");
 519                         return ret;
 520                 }
 521         }
 522         ret = mlnx_resume_database(priv, id);
 523         if (ret < 0) {
 524                 DRV_LOG(ERR, "Failed to resume engine!");
 525                 return ret;
 526         }
 527         DRV_LOG(DEBUG, "Programmed RXP Engine %d\n", id);
 528         rules->count = rule_cnt;
 529         return 0;
 530 }
 531
 532 static int
 533 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id)
 534 {
 535         uint32_t ctrl;
 536         uint32_t reg;
 537         struct ibv_context *ctx = priv->ctx;
 538         int ret;
 539
 540         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 541         if (ret)
 542                 return ret;
 543         if (ctrl & MLX5_RXP_CSR_CTRL_INIT) {
 544                 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 545                 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 546                                                      ctrl);
 547                 if (ret)
 548                         return ret;
 549         }
 550         ctrl |= MLX5_RXP_CSR_CTRL_INIT;
 551         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 552         if (ret)
 553                 return ret;
 554         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 555         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 556         rte_delay_us(20000);
 557         ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS,
 558                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 559                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 560                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 561         if (ret)
 562                 return ret;
 563         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 564         if (ret)
 565                 return ret;
 566         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 567         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 568                                              ctrl);
 569         if (ret)
 570                 return ret;
 571         ret = rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2);
 572         if (ret)
 573                 return ret;
 574         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5,
 575                                             &reg);
 576         if (ret)
 577                 return ret;
 578         DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16,
 579                 reg & 0xffff);
 580         if ((reg >> 16) >= priv->nb_max_matches)
 581                 ret = mlx5_devx_regex_register_write(ctx, id,
 582                                                      MLX5_RXP_CSR_MAX_MATCH,
 583                                                      priv->nb_max_matches);
 584         else
 585                 ret = mlx5_devx_regex_register_write(ctx, id,
 586                                                      MLX5_RXP_CSR_MAX_MATCH,
 587                                                      (reg >> 16));
 588         ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX,
 589                                          (reg & 0xFFFF));
 590         ret |= mlx5_devx_regex_register_write(ctx, id,
 591                                               MLX5_RXP_CSR_MAX_LATENCY, 0);
 592         ret |= mlx5_devx_regex_register_write(ctx, id,
 593                                               MLX5_RXP_CSR_MAX_PRI_THREAD, 0);
 594         return ret;
 595 }
 596
 597 static int
 598 write_private_rules(struct mlx5_regex_priv *priv,
 599                     struct mlx5_rxp_ctl_rules_pgm *rules,
 600                     uint8_t id)
 601 {
 602         unsigned int pending;
 603         uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries;
 604         int ret = 1;
 605
 606         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 607                 return -EINVAL;
 608         if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM ||
 609                                    rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR)
 610                 return -EINVAL;
 611         /* For a non-incremental rules program, re-init the RXP. */
 612         if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) {
 613                 ret = rxp_init_eng(priv, id);
 614                 if (ret < 0)
 615                         return ret;
 616         } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) {
 617                 /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */
 618                 ret = rxp_init_rtru(priv->ctx, id,
 619                                     MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
 620                 if (ret < 0)
 621                         return ret;
 622         }
 623         if (rules->count == 0)
 624                 return -EINVAL;
 625         /* Confirm the RXP is initialised. */
 626         if (mlx5_devx_regex_register_read(priv->ctx, id,
 627                                             MLX5_RXP_CSR_STATUS, &val)) {
 628                 DRV_LOG(ERR, "Failed to read from RXP!");
 629                 return -ENODEV;
 630         }
 631         if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) {
 632                 DRV_LOG(ERR, "RXP not initialised...");
 633                 return -EBUSY;
 634         }
 635         /* Get the RTRU maximum number of entries allowed. */
 636         if (mlx5_devx_regex_register_read(priv->ctx, id,
 637                         MLX5_RXP_RTRU_CSR_CAPABILITY, &rtru_max_num_entries)) {
 638                 DRV_LOG(ERR, "Failed to read RTRU capability!");
 639                 return -ENODEV;
 640         }
 641         rtru_max_num_entries = (rtru_max_num_entries & 0x00FF);
 642         rule_cnt = 0;
 643         pending = 0;
 644         while (rules->count > 0) {
 645                 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_INST) ||
 646                     (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_IM) ||
 647                     (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM)) {
 648                         if ((rules->rules[rule_cnt].type ==
 649                              MLX5_RXP_ROF_ENTRY_EM) &&
 650                             (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
 651                                 /* Skip EM rules programming. */
 652                                 if (pending > 0) {
 653                                         /* Flush any rules that are pending. */
 654                                         rule_offset = (rule_cnt - pending);
 655                                         ret = rxp_flush_rules(priv->ctx,
 656                                                 &rules->rules[rule_offset],
 657                                                 pending, id);
 658                                         if (ret < 0) {
 659                                                 DRV_LOG(ERR, "Flushing rules.");
 660                                                 return -ENODEV;
 661                                         }
 662                                         pending = 0;
 663                                 }
 664                                 rule_cnt++;
 665                         } else {
 666                                 pending++;
 667                                 rule_cnt++;
 668                                 /*
 669                                  * If parsing the last rule, or if reached the
 670                                  * maximum number of rules for this batch, then
 671                                  * flush the rules batch to the RXP.
 672                                  */
 673                                 if ((rules->count == 1) ||
 674                                     (pending == rtru_max_num_entries)) {
 675                                         rule_offset = (rule_cnt - pending);
 676                                         ret = rxp_flush_rules(priv->ctx,
 677                                                 &rules->rules[rule_offset],
 678                                                 pending, id);
 679                                         if (ret < 0) {
 680                                                 DRV_LOG(ERR, "Flushing rules.");
 681                                                 return -ENODEV;
 682                                         }
 683                                         pending = 0;
 684                                 }
 685                         }
 686                 } else if ((rules->rules[rule_cnt].type ==
 687                                 MLX5_RXP_ROF_ENTRY_EQ) ||
 688                          (rules->rules[rule_cnt].type ==
 689                                 MLX5_RXP_ROF_ENTRY_GTE) ||
 690                          (rules->rules[rule_cnt].type ==
 691                                 MLX5_RXP_ROF_ENTRY_LTE) ||
 692                          (rules->rules[rule_cnt].type ==
 693                                 MLX5_RXP_ROF_ENTRY_CHECKSUM) ||
 694                          (rules->rules[rule_cnt].type ==
 695                                 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM)) {
 696                         if (pending) {
 697                                 /* Flush rules before checking reg values. */
 698                                 rule_offset = (rule_cnt - pending);
 699                                 ret = rxp_flush_rules(priv->ctx,
 700                                         &rules->rules[rule_offset],
 701                                         pending, id);
 702                                 if (ret < 0) {
 703                                         DRV_LOG(ERR, "Failed to flush rules.");
 704                                         return -ENODEV;
 705                                 }
 706                         }
 707                         block = (rules->rules[rule_cnt].addr >> 16) & 0xFFFF;
 708                         if (block == 0)
 709                                 reg = MLX5_RXP_CSR_BASE_ADDRESS;
 710                         else if (block == 1)
 711                                 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS;
 712                         else {
 713                                 DRV_LOG(ERR, "Invalid ROF register 0x%08X!",
 714                                         rules->rules[rule_cnt].addr);
 715                                 return -EINVAL;
 716                         }
 717                         reg += (rules->rules[rule_cnt].addr & 0xFFFF) *
 718                                 MLX5_RXP_CSR_WIDTH;
 719                         ret = mlx5_devx_regex_register_read(priv->ctx, id,
 720                                                             reg, &val);
 721                         if (ret) {
 722                                 DRV_LOG(ERR, "RXP CSR read failed!");
 723                                 return ret;
 724                         }
 725                         if ((priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) &&
 726                             ((rules->rules[rule_cnt].type ==
 727                             MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM) &&
 728                             (val != rules->rules[rule_cnt].value))) {
 729                                 DRV_LOG(ERR, "Unexpected value for register:");
 730                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 731                                         rules->rules[rule_cnt].addr, val);
 732                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 733                                         rules->rules[rule_cnt].value);
 734                                         return -EINVAL;
 735                         } else if ((priv->prog_mode ==
 736                                  MLX5_RXP_PRIVATE_PROG_MODE) &&
 737                                  (rules->rules[rule_cnt].type ==
 738                                  MLX5_RXP_ROF_ENTRY_CHECKSUM) &&
 739                                  (val != rules->rules[rule_cnt].value)) {
 740                                 DRV_LOG(ERR, "Unexpected value for register:");
 741                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 742                                         rules->rules[rule_cnt].addr, val);
 743                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 744                                         rules->rules[rule_cnt].value);
 745                                 return -EINVAL;
 746                         } else if ((rules->rules[rule_cnt].type ==
 747                                         MLX5_RXP_ROF_ENTRY_EQ) &&
 748                                   (val != rules->rules[rule_cnt].value)) {
 749                                 DRV_LOG(ERR, "Unexpected value for register:");
 750                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 751                                         rules->rules[rule_cnt].addr, val);
 752                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 753                                         rules->rules[rule_cnt].value);
 754                                         return -EINVAL;
 755                         } else if ((rules->rules[rule_cnt].type ==
 756                                         MLX5_RXP_ROF_ENTRY_GTE) &&
 757                                  (val < rules->rules[rule_cnt].value)) {
 758                                 DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
 759                                         rules->rules[rule_cnt].addr);
 760                                 DRV_LOG(ERR, "got %X, expected >= %" PRIx64 ".",
 761                                         val, rules->rules[rule_cnt].value);
 762                                 return -EINVAL;
 763                         } else if ((rules->rules[rule_cnt].type ==
 764                                         MLX5_RXP_ROF_ENTRY_LTE) &&
 765                                  (val > rules->rules[rule_cnt].value)) {
 766                                 DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
 767                                         rules->rules[rule_cnt].addr);
 768                                 DRV_LOG(ERR, "got %08X expected <= %" PRIx64,
 769                                         val, rules->rules[rule_cnt].value);
 770                                 return -EINVAL;
 771                         }
 772                         rule_cnt++;
 773                         pending = 0;
 774                 } else {
 775                         DRV_LOG(ERR, "Error: Invalid rule type %d!",
 776                                 rules->rules[rule_cnt].type);
 777                         return -EINVAL;
 778                 }
 779                 rules->count--;
 780         }
 781         return ret;
 782 }
 783
 784 /*
 785  * Shared memory programming mode, here all external db instructions are written
 786  * to EM via the host.
 787  */
 788 static int
 789 write_shared_rules(struct mlx5_regex_priv *priv,
 790                    struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
 791                    uint8_t db_to_program)
 792 {
 793         uint32_t rule_cnt, rof_rule_addr;
 794         uint64_t tmp_write_swap[4];
 795
 796         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 797                 return -EINVAL;
 798         if ((rules->count == 0) || (count == 0))
 799                 return -EINVAL;
 800         rule_cnt = 0;
 801         /*
 802          * Note the following section of code carries out a 32byte swap of
 803          * instruction to coincide with HW 32byte swap. This may need removed
 804          * in new variants of this programming function!
 805          */
 806         while (rule_cnt < rules->count) {
 807                 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM) &&
 808                     (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
 809                         /*
 810                          * Note there are always blocks of 8 instructions for
 811                          * 7's written sequentially. However there is no
 812                          * guarantee that all blocks are sequential!
 813                          */
 814                         if (count >= (rule_cnt + MLX5_RXP_INST_BLOCK_SIZE)) {
 815                                 /*
 816                                  * Ensure memory write not exceeding boundary
 817                                  * Check essential to ensure 0x10000 offset
 818                                  * accounted for!
 819                                  */
 820                                 if ((uint8_t *)((uint8_t *)
 821                                     priv->db[db_to_program].ptr +
 822                                     ((rules->rules[rule_cnt + 7].addr <<
 823                                     MLX5_RXP_INST_OFFSET))) >=
 824                                     ((uint8_t *)((uint8_t *)
 825                                     priv->db[db_to_program].ptr +
 826                                     MLX5_MAX_DB_SIZE))) {
 827                                         DRV_LOG(ERR, "DB exceeded memory!");
 828                                         return -ENODEV;
 829                                 }
 830                                 /*
 831                                  * Rule address Offset to align with RXP
 832                                  * external instruction offset.
 833                                  */
 834                                 rof_rule_addr = (rules->rules[rule_cnt].addr <<
 835                                                  MLX5_RXP_INST_OFFSET);
 836                                 /* 32 byte instruction swap (sw work around)! */
 837                                 tmp_write_swap[0] = le64toh(
 838                                         rules->rules[(rule_cnt + 4)].value);
 839                                 tmp_write_swap[1] = le64toh(
 840                                         rules->rules[(rule_cnt + 5)].value);
 841                                 tmp_write_swap[2] = le64toh(
 842                                         rules->rules[(rule_cnt + 6)].value);
 843                                 tmp_write_swap[3] = le64toh(
 844                                         rules->rules[(rule_cnt + 7)].value);
 845                                 /* Write only 4 of the 8 instructions. */
 846                                 memcpy((uint8_t *)((uint8_t *)
 847                                        priv->db[db_to_program].ptr +
 848                                        rof_rule_addr), &tmp_write_swap,
 849                                        (sizeof(uint64_t) * 4));
 850                                 /* Write 1st 4 rules of block after last 4. */
 851                                 rof_rule_addr = (rules->rules[
 852                                                  (rule_cnt + 4)].addr <<
 853                                                  MLX5_RXP_INST_OFFSET);
 854                                 tmp_write_swap[0] = le64toh(
 855                                         rules->rules[(rule_cnt + 0)].value);
 856                                 tmp_write_swap[1] = le64toh(
 857                                         rules->rules[(rule_cnt + 1)].value);
 858                                 tmp_write_swap[2] = le64toh(
 859                                         rules->rules[(rule_cnt + 2)].value);
 860                                 tmp_write_swap[3] = le64toh(
 861                                         rules->rules[(rule_cnt + 3)].value);
 862                                 memcpy((uint8_t *)((uint8_t *)
 863                                        priv->db[db_to_program].ptr +
 864                                        rof_rule_addr), &tmp_write_swap,
 865                                        (sizeof(uint64_t) * 4));
 866                         } else
 867                                 return -1;
 868                         /* Fast forward as already handled block of 8. */
 869                         rule_cnt += MLX5_RXP_INST_BLOCK_SIZE;
 870                 } else
 871                         rule_cnt++; /* Must be something other than EM rule. */
 872         }
 873         return 0;
 874 }
 875
 876 static int
 877 rxp_db_setup(struct mlx5_regex_priv *priv)
 878 {
 879         int ret;
 880         uint8_t i;
 881
 882         /* Setup database memories for both RXP engines + reprogram memory. */
 883         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 884                 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 0);
 885                 if (!priv->db[i].ptr) {
 886                         DRV_LOG(ERR, "Failed to alloc db memory!");
 887                         ret = ENODEV;
 888                         goto tidyup_error;
 889                 }
 890                 /* Register the memory. */
 891                 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx,
 892                                                         priv->db[i].ptr,
 893                                                         MLX5_MAX_DB_SIZE, 7);
 894                 if (!priv->db[i].umem.umem) {
 895                         DRV_LOG(ERR, "Failed to register memory!");
 896                         ret = ENODEV;
 897                         goto tidyup_error;
 898                 }
 899                 /* Ensure set all DB memory to 0's before setting up DB. */
 900                 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 901                 /* No data currently in database. */
 902                 priv->db[i].len = 0;
 903                 priv->db[i].active = false;
 904                 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED;
 905         }
 906         return 0;
 907 tidyup_error:
 908         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 909                 if (priv->db[i].ptr)
 910                         rte_free(priv->db[i].ptr);
 911                 if (priv->db[i].umem.umem)
 912                         mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem);
 913         }
 914         return -ret;
 915 }
 916
 917 int
 918 mlx5_regex_rules_db_import(struct rte_regexdev *dev,
 919                      const char *rule_db, uint32_t rule_db_len)
 920 {
 921         struct mlx5_regex_priv *priv = dev->data->dev_private;
 922         struct mlx5_rxp_ctl_rules_pgm *rules = NULL;
 923         uint8_t id;
 924         int ret;
 925
 926         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
 927                 DRV_LOG(ERR, "RXP programming mode not set!");
 928                 return -1;
 929         }
 930         if (rule_db == NULL) {
 931                 DRV_LOG(ERR, "Database empty!");
 932                 return -ENODEV;
 933         }
 934         if (rule_db_len == 0)
 935                 return -EINVAL;
 936         ret = rxp_parse_rof(rule_db, rule_db_len, &rules);
 937         if (ret) {
 938                 DRV_LOG(ERR, "Can't parse ROF file.");
 939                 return ret;
 940         }
 941         /* Need to ensure RXP not busy before stop! */
 942         for (id = 0; id < priv->nb_engines; id++) {
 943                 ret = rxp_stop_engine(priv->ctx, id);
 944                 if (ret) {
 945                         DRV_LOG(ERR, "Can't stop engine.");
 946                         ret = -ENODEV;
 947                         goto tidyup_error;
 948                 }
 949                 ret = program_rxp_rules(priv, rules, id);
 950                 if (ret < 0) {
 951                         DRV_LOG(ERR, "Failed to program rxp rules.");
 952                         ret = -ENODEV;
 953                         goto tidyup_error;
 954                 }
 955                 ret = rxp_start_engine(priv->ctx, id);
 956                 if (ret) {
 957                         DRV_LOG(ERR, "Can't start engine.");
 958                         ret = -ENODEV;
 959                         goto tidyup_error;
 960                 }
 961         }
 962         rte_free(rules);
 963         return 0;
 964 tidyup_error:
 965         rte_free(rules);
 966         return ret;
 967 }
 968
 969 int
 970 mlx5_regex_configure(struct rte_regexdev *dev,
 971                      const struct rte_regexdev_config *cfg)
 972 {
 973         struct mlx5_regex_priv *priv = dev->data->dev_private;
 974         int ret;
 975
 976         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 977                 return -1;
 978         priv->nb_queues = cfg->nb_queue_pairs;
 979         dev->data->dev_conf.nb_queue_pairs = priv->nb_queues;
 980         priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) *
 981                                 priv->nb_queues, 0);
 982         if (!priv->nb_queues) {
 983                 DRV_LOG(ERR, "can't allocate qps memory");
 984                 rte_errno = ENOMEM;
 985                 return -rte_errno;
 986         }
 987         priv->nb_max_matches = cfg->nb_max_matches;
 988         /* Setup rxp db memories. */
 989         if (rxp_db_setup(priv)) {
 990                 DRV_LOG(ERR, "Failed to setup RXP db memory");
 991                 rte_errno = ENOMEM;
 992                 return -rte_errno;
 993         }
 994         if (cfg->rule_db != NULL) {
 995                 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db,
 996                                                  cfg->rule_db_len);
 997                 if (ret < 0) {
 998                         DRV_LOG(ERR, "Failed to program rxp rules.");
 999                         rte_errno = ENODEV;
1000                         goto configure_error;
1001                 }
1002         } else
1003                 DRV_LOG(DEBUG, "Regex config without rules programming!");
1004         return 0;
1005 configure_error:
1006         if (priv->qps)
1007                 rte_free(priv->qps);
1008         return -rte_errno;
1009 }