drivers/regex/mlx5/mlx5_rxp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_log.h>
   6 #include <rte_errno.h>
   7 #include <rte_malloc.h>
   8 #include <rte_regexdev.h>
   9 #include <rte_regexdev_core.h>
  10 #include <rte_regexdev_driver.h>
  11
  12 #include <mlx5_glue.h>
  13 #include <mlx5_devx_cmds.h>
  14 #include <mlx5_prm.h>
  15 #include <mlx5_common_os.h>
  16
  17 #include "mlx5_regex.h"
  18 #include "mlx5_regex_utils.h"
  19 #include "mlx5_rxp_csrs.h"
  20 #include "mlx5_rxp.h"
  21
  22 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES
  23 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH
  24 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX
  25 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS
  26
  27 /* Private Declarations */
  28 static int
  29 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
  30                        uint32_t address, uint32_t expected_value,
  31                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id);
  32 static int
  33 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use);
  34 static int
  35 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id);
  36 static int
  37 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id);
  38 static int
  39 program_rxp_rules(struct mlx5_regex_priv *priv,
  40                   struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id);
  41 static int
  42 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id);
  43 static int
  44 write_private_rules(struct mlx5_regex_priv *priv,
  45                     struct mlx5_rxp_ctl_rules_pgm *rules,
  46                     uint8_t id);
  47 static int
  48 write_shared_rules(struct mlx5_regex_priv *priv,
  49                    struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
  50                    uint8_t db_to_program);
  51 static int
  52 rxp_db_setup(struct mlx5_regex_priv *priv);
  53 static void
  54 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id);
  55 static int
  56 rxp_write_rules_via_cp(struct ibv_context *ctx,
  57                        struct mlx5_rxp_rof_entry *rules,
  58                        int count, uint8_t id);
  59 static int
  60 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules,
  61                 int count, uint8_t id);
  62 static int
  63 rxp_start_engine(struct ibv_context *ctx, uint8_t id);
  64 static int
  65 rxp_stop_engine(struct ibv_context *ctx, uint8_t id);
  66
  67 static void __rte_unused
  68 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused)
  69 {
  70         uint32_t reg, i;
  71
  72         /* Main CSRs*/
  73         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  74                 if (mlx5_devx_regex_register_read(ctx, id,
  75                                                   (MLX5_RXP_CSR_WIDTH * i) +
  76                                                   MLX5_RXP_CSR_BASE_ADDRESS,
  77                                                   &reg)) {
  78                         DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id);
  79                         return;
  80                 }
  81                 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x",
  82                         id, i, reg);
  83         }
  84         /* RTRU CSRs*/
  85         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  86                 if (mlx5_devx_regex_register_read(ctx, id,
  87                                                   (MLX5_RXP_CSR_WIDTH * i) +
  88                                                  MLX5_RXP_RTRU_CSR_BASE_ADDRESS,
  89                                                   &reg)) {
  90                         DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id);
  91                         return;
  92                 }
  93                 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x",
  94                         id, i, reg);
  95         }
  96         /* STAT CSRs */
  97         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  98                 if (mlx5_devx_regex_register_read(ctx, id,
  99                                                   (MLX5_RXP_CSR_WIDTH * i) +
 100                                                 MLX5_RXP_STATS_CSR_BASE_ADDRESS,
 101                                                   &reg)) {
 102                         DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id);
 103                         return;
 104                 }
 105                 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x",
 106                         id, i, reg);
 107         }
 108 }
 109
 110 int
 111 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused,
 112                     struct rte_regexdev_info *info)
 113 {
 114         info->max_matches = MLX5_REGEX_MAX_MATCHES;
 115         info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE;
 116         info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP;
 117         info->max_groups = MLX5_REGEX_MAX_GROUPS;
 118         info->max_queue_pairs = 1;
 119         info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F |
 120                               RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F;
 121         info->rule_flags = 0;
 122         info->max_queue_pairs = 10;
 123         return 0;
 124 }
 125
 126 /**
 127  * Actual writing of RXP instructions to RXP via CSRs.
 128  */
 129 static int
 130 rxp_write_rules_via_cp(struct ibv_context *ctx,
 131                        struct mlx5_rxp_rof_entry *rules,
 132                        int count, uint8_t id)
 133 {
 134         int i, ret = 0;
 135         uint32_t tmp;
 136
 137         for (i = 0; i < count; i++) {
 138                 tmp = (uint32_t)rules[i].value;
 139                 ret |= mlx5_devx_regex_register_write(ctx, id,
 140                                                       MLX5_RXP_RTRU_CSR_DATA_0,
 141                                                       tmp);
 142                 tmp = (uint32_t)(rules[i].value >> 32);
 143                 ret |= mlx5_devx_regex_register_write(ctx, id,
 144                                                       MLX5_RXP_RTRU_CSR_DATA_0 +
 145                                                       MLX5_RXP_CSR_WIDTH, tmp);
 146                 tmp = rules[i].addr;
 147                 ret |= mlx5_devx_regex_register_write(ctx, id,
 148                                                       MLX5_RXP_RTRU_CSR_ADDR,
 149                                                       tmp);
 150                 if (ret) {
 151                         DRV_LOG(ERR, "Failed to copy instructions to RXP.");
 152                         return -1;
 153                 }
 154         }
 155         DRV_LOG(DEBUG, "Written %d instructions", count);
 156         return 0;
 157 }
 158
 159 static int
 160 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules,
 161                 int count, uint8_t id)
 162 {
 163         uint32_t val, fifo_depth;
 164         int ret;
 165
 166         ret = rxp_write_rules_via_cp(ctx, rules, count, id);
 167         if (ret < 0) {
 168                 DRV_LOG(ERR, "Failed to write rules via CSRs.");
 169                 return -1;
 170         }
 171         ret = mlx5_devx_regex_register_read(ctx, id,
 172                                             MLX5_RXP_RTRU_CSR_CAPABILITY,
 173                                             &fifo_depth);
 174         if (ret) {
 175                 DRV_LOG(ERR, "CSR read failed!");
 176                 return -1;
 177         }
 178         ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_FIFO_STAT,
 179                                      count, ~0,
 180                                      MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 181         if (ret < 0) {
 182                 if (ret == -EBUSY)
 183                         DRV_LOG(ERR, "Rules not rx by RXP: credit: %d, depth:"
 184                                 " %d", val, fifo_depth);
 185                 else
 186                         DRV_LOG(ERR, "CSR poll failed, can't read value!");
 187                 return ret;
 188         }
 189         DRV_LOG(DEBUG, "RTRU FIFO depth: 0x%x", fifo_depth);
 190         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 191                                             &val);
 192         if (ret) {
 193                 DRV_LOG(ERR, "CSR read failed!");
 194                 return -1;
 195         }
 196         val |= MLX5_RXP_RTRU_CSR_CTRL_GO;
 197         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 198                                              val);
 199         if (ret) {
 200                 DRV_LOG(ERR, "CSR write failed!");
 201                 return -1;
 202         }
 203         ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_STATUS,
 204                                      MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 205                                      MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 206                                      MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 207         if (ret < 0) {
 208                 if (ret == -EBUSY)
 209                         DRV_LOG(ERR, "Rules update timeout: 0x%08X", val);
 210                 else
 211                         DRV_LOG(ERR, "CSR poll failed, can't read value!");
 212                 return ret;
 213         }
 214         if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 215                                           &val)) {
 216                 DRV_LOG(ERR, "CSR read failed!");
 217                 return -1;
 218         }
 219         val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO);
 220         if (mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 221                                            val)) {
 222                 DRV_LOG(ERR, "CSR write failed!");
 223                 return -1;
 224         }
 225
 226         DRV_LOG(DEBUG, "RXP Flush rules finished.");
 227         return 0;
 228 }
 229
 230 static int
 231 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
 232                        uint32_t address, uint32_t expected_value,
 233                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id)
 234 {
 235         unsigned int i;
 236         int ret;
 237
 238         ret = -EBUSY;
 239         for (i = 0; i < timeout_ms; i++) {
 240                 if (mlx5_devx_regex_register_read(ctx, id, address, value))
 241                         return -1;
 242                 if ((*value & expected_mask) == expected_value) {
 243                         ret = 0;
 244                         break;
 245                 }
 246                 rte_delay_us(1000);
 247         }
 248         return ret;
 249 }
 250
 251 static int
 252 rxp_start_engine(struct ibv_context *ctx, uint8_t id)
 253 {
 254         uint32_t ctrl;
 255         int ret;
 256
 257         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 258         if (ret)
 259                 return ret;
 260         ctrl |= MLX5_RXP_CSR_CTRL_GO;
 261         ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C;
 262         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 263         return ret;
 264 }
 265
 266 static int
 267 rxp_stop_engine(struct ibv_context *ctx, uint8_t id)
 268 {
 269         uint32_t ctrl;
 270         int ret;
 271
 272         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 273         if (ret)
 274                 return ret;
 275         ctrl &= ~MLX5_RXP_CSR_CTRL_GO;
 276         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 277         return ret;
 278 }
 279
 280 static int
 281 rxp_init_rtru(struct ibv_context *ctx, uint8_t id, uint32_t init_bits)
 282 {
 283         uint32_t ctrl_value;
 284         uint32_t poll_value;
 285         uint32_t expected_value;
 286         uint32_t expected_mask;
 287         int ret;
 288
 289         /* Read the rtru ctrl CSR. */
 290         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 291                                             &ctrl_value);
 292         if (ret)
 293                 return -1;
 294         /* Clear any previous init modes. */
 295         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK);
 296         if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) {
 297                 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 298                 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 299                                                ctrl_value);
 300         }
 301         /* Set the init_mode bits in the rtru ctrl CSR. */
 302         ctrl_value |= init_bits;
 303         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 304                                        ctrl_value);
 305         /* Need to sleep for a short period after pulsing the rtru init bit. */
 306         rte_delay_us(20000);
 307         /* Poll the rtru status CSR until all the init done bits are set. */
 308         DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init");
 309         /* Set the init bit in the rtru ctrl CSR. */
 310         ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT;
 311         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 312                                        ctrl_value);
 313         /* Clear the init bit in the rtru ctrl CSR */
 314         ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT;
 315         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 316                                        ctrl_value);
 317         /* Check that the following bits are set in the RTRU_CSR. */
 318         if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) {
 319                 /* Must be incremental mode */
 320                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
 321                         MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 322         } else {
 323                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE |
 324                         MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
 325                         MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 326         }
 327         expected_mask = expected_value;
 328         ret = rxp_poll_csr_for_value(ctx, &poll_value,
 329                                      MLX5_RXP_RTRU_CSR_STATUS,
 330                                      expected_value, expected_mask,
 331                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 332         if (ret)
 333                 return ret;
 334         DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value);
 335         /* Clear the init bit in the rtru ctrl CSR */
 336         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 337         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 338                                        ctrl_value);
 339         return 0;
 340 }
 341
 342 static int
 343 rxp_parse_rof(const char *buf, uint32_t len,
 344               struct mlx5_rxp_ctl_rules_pgm **rules)
 345 {
 346         static const char del[] = "\n\r";
 347         char *line;
 348         char *tmp;
 349         char *cur_pos;
 350         uint32_t lines = 0;
 351         uint32_t entries;
 352         struct mlx5_rxp_rof_entry *curentry;
 353
 354         tmp = rte_malloc("", len, 0);
 355         if (!tmp)
 356                 return -ENOMEM;
 357         memcpy(tmp, buf, len);
 358         line = strtok(tmp, del);
 359         while (line) {
 360                 if (line[0] != '#' && line[0] != '\0')
 361                         lines++;
 362                 line = strtok(NULL, del);
 363         }
 364         *rules = rte_malloc("", lines * sizeof(*curentry) + sizeof(**rules), 0);
 365         if (!(*rules)) {
 366                 rte_free(tmp);
 367                 return -ENOMEM;
 368         }
 369         memset(*rules, 0, lines * sizeof(curentry) + sizeof(**rules));
 370         curentry = (*rules)->rules;
 371         (*rules)->hdr.cmd = MLX5_RXP_CTL_RULES_PGM;
 372         entries = 0;
 373         memcpy(tmp, buf, len);
 374         line = strtok(tmp, del);
 375         while (line) {
 376                 if (line[0] == '#' || line[0] == '\0') {
 377                         line = strtok(NULL, del);
 378                         continue;
 379                 }
 380                 curentry->type = strtoul(line, &cur_pos, 10);
 381                 if (cur_pos == line || cur_pos[0] != ',')
 382                         goto parse_error;
 383                 cur_pos++;
 384                 curentry->addr = strtoul(cur_pos, &cur_pos, 16);
 385                 if (cur_pos[0] != ',')
 386                         goto parse_error;
 387                 cur_pos++;
 388                 curentry->value = strtoull(cur_pos, &cur_pos, 16);
 389                 if (cur_pos[0] != '\0' && cur_pos[0] != '\n')
 390                         goto parse_error;
 391                 curentry++;
 392                 entries++;
 393                 if (entries > lines)
 394                         goto parse_error;
 395                 line = strtok(NULL, del);
 396         }
 397         (*rules)->count = entries;
 398         (*rules)->hdr.len = entries * sizeof(*curentry) + sizeof(**rules);
 399         rte_free(tmp);
 400         return 0;
 401 parse_error:
 402         rte_free(tmp);
 403         if (*rules)
 404                 rte_free(*rules);
 405         return -EINVAL;
 406 }
 407
 408 static int
 409 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use)
 410 {
 411         int ret;
 412         uint32_t umem_id;
 413
 414         ret = mlx5_devx_regex_database_stop(priv->ctx, id);
 415         if (ret < 0) {
 416                 DRV_LOG(ERR, "stop engine failed!");
 417                 return ret;
 418         }
 419         umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem);
 420         ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0);
 421         if (ret < 0) {
 422                 DRV_LOG(ERR, "program db failed!");
 423                 return ret;
 424         }
 425         return 0;
 426 }
 427
 428 static int
 429 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id)
 430 {
 431         mlx5_devx_regex_database_resume(priv->ctx, id);
 432         return 0;
 433 }
 434
 435 /*
 436  * Assign db memory for RXP programming.
 437  */
 438 static int
 439 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id)
 440 {
 441         unsigned int i;
 442         uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED;
 443         uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED;
 444
 445         /* Check which database rxp_eng is currently located if any? */
 446         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 447              i++) {
 448                 if (priv->db[i].db_assigned_to_eng_num == id) {
 449                         eng_assigned = i;
 450                         break;
 451                 }
 452         }
 453         /*
 454          * If private mode then, we can keep the same db ptr as RXP will be
 455          * programming EM itself if necessary, however need to see if
 456          * programmed yet.
 457          */
 458         if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) &&
 459             (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED))
 460                 return eng_assigned;
 461         /* Check for inactive db memory to use. */
 462         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 463              i++) {
 464                 if (priv->db[i].active == true)
 465                         continue; /* Already in use, so skip db. */
 466                 /* Set this db to active now as free to use. */
 467                 priv->db[i].active = true;
 468                 /* Now unassign last db index in use by RXP Eng. */
 469                 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) {
 470                         priv->db[eng_assigned].active = false;
 471                         priv->db[eng_assigned].db_assigned_to_eng_num =
 472                                 MLX5_RXP_DB_NOT_ASSIGNED;
 473
 474                         /* Set all DB memory to 0's before setting up DB. */
 475                         memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 476                 }
 477                 /* Now reassign new db index with RXP Engine. */
 478                 priv->db[i].db_assigned_to_eng_num = id;
 479                 db_free = i;
 480                 break;
 481         }
 482         if (db_free == MLX5_RXP_DB_NOT_ASSIGNED)
 483                 return -1;
 484         return db_free;
 485 }
 486
 487 /*
 488  * Program RXP instruction db to RXP engine/s.
 489  */
 490 static int
 491 program_rxp_rules(struct mlx5_regex_priv *priv,
 492                   struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id)
 493 {
 494         int ret, db_free;
 495         uint32_t rule_cnt;
 496
 497         rule_cnt = rules->count;
 498         db_free = mlnx_update_database(priv, id);
 499         if (db_free < 0) {
 500                 DRV_LOG(ERR, "Failed to setup db memory!");
 501                 return db_free;
 502         }
 503         if (priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) {
 504                 /* Register early to ensure RXP writes to EM use valid addr. */
 505                 ret = mlnx_set_database(priv, id, db_free);
 506                 if (ret < 0) {
 507                         DRV_LOG(ERR, "Failed to register db memory!");
 508                         return ret;
 509                 }
 510         }
 511         ret = write_private_rules(priv, rules, id);
 512         if (ret < 0) {
 513                 DRV_LOG(ERR, "Failed to write rules!");
 514                 return ret;
 515         }
 516         if (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) {
 517                 /* Write external rules directly to EM. */
 518                 rules->count = rule_cnt;
 519                /* Now write external instructions to EM. */
 520                 ret = write_shared_rules(priv, rules, rules->hdr.len, db_free);
 521                 if (ret < 0) {
 522                         DRV_LOG(ERR, "Failed to write EM rules!");
 523                         return ret;
 524                 }
 525                 ret = mlnx_set_database(priv, id, db_free);
 526                 if (ret < 0) {
 527                         DRV_LOG(ERR, "Failed to register db memory!");
 528                         return ret;
 529                 }
 530         }
 531         ret = mlnx_resume_database(priv, id);
 532         if (ret < 0) {
 533                 DRV_LOG(ERR, "Failed to resume engine!");
 534                 return ret;
 535         }
 536         DRV_LOG(DEBUG, "Programmed RXP Engine %d\n", id);
 537         rules->count = rule_cnt;
 538         return 0;
 539 }
 540
 541 static int
 542 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id)
 543 {
 544         uint32_t ctrl;
 545         uint32_t reg;
 546         struct ibv_context *ctx = priv->ctx;
 547         int ret;
 548
 549         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 550         if (ret)
 551                 return ret;
 552         if (ctrl & MLX5_RXP_CSR_CTRL_INIT) {
 553                 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 554                 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 555                                                      ctrl);
 556                 if (ret)
 557                         return ret;
 558         }
 559         ctrl |= MLX5_RXP_CSR_CTRL_INIT;
 560         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 561         if (ret)
 562                 return ret;
 563         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 564         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 565         if (ret)
 566                 return ret;
 567         rte_delay_us(20000);
 568         ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS,
 569                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 570                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 571                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 572         if (ret)
 573                 return ret;
 574         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 575         if (ret)
 576                 return ret;
 577         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 578         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 579                                              ctrl);
 580         if (ret)
 581                 return ret;
 582         ret = rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2);
 583         if (ret)
 584                 return ret;
 585         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5,
 586                                             &reg);
 587         if (ret)
 588                 return ret;
 589         DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16,
 590                 reg & 0xffff);
 591         if ((reg >> 16) >= priv->nb_max_matches)
 592                 ret = mlx5_devx_regex_register_write(ctx, id,
 593                                                      MLX5_RXP_CSR_MAX_MATCH,
 594                                                      priv->nb_max_matches);
 595         else
 596                 ret = mlx5_devx_regex_register_write(ctx, id,
 597                                                      MLX5_RXP_CSR_MAX_MATCH,
 598                                                      (reg >> 16));
 599         ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX,
 600                                          (reg & 0xFFFF));
 601         ret |= mlx5_devx_regex_register_write(ctx, id,
 602                                               MLX5_RXP_CSR_MAX_LATENCY, 0);
 603         ret |= mlx5_devx_regex_register_write(ctx, id,
 604                                               MLX5_RXP_CSR_MAX_PRI_THREAD, 0);
 605         return ret;
 606 }
 607
 608 static int
 609 write_private_rules(struct mlx5_regex_priv *priv,
 610                     struct mlx5_rxp_ctl_rules_pgm *rules,
 611                     uint8_t id)
 612 {
 613         unsigned int pending;
 614         uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries;
 615         int ret = 1;
 616
 617         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 618                 return -EINVAL;
 619         if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM ||
 620                                    rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR)
 621                 return -EINVAL;
 622         /* For a non-incremental rules program, re-init the RXP. */
 623         if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) {
 624                 ret = rxp_init_eng(priv, id);
 625                 if (ret < 0)
 626                         return ret;
 627         } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) {
 628                 /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */
 629                 ret = rxp_init_rtru(priv->ctx, id,
 630                                     MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
 631                 if (ret < 0)
 632                         return ret;
 633         }
 634         if (rules->count == 0)
 635                 return -EINVAL;
 636         /* Confirm the RXP is initialised. */
 637         if (mlx5_devx_regex_register_read(priv->ctx, id,
 638                                             MLX5_RXP_CSR_STATUS, &val)) {
 639                 DRV_LOG(ERR, "Failed to read from RXP!");
 640                 return -ENODEV;
 641         }
 642         if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) {
 643                 DRV_LOG(ERR, "RXP not initialised...");
 644                 return -EBUSY;
 645         }
 646         /* Get the RTRU maximum number of entries allowed. */
 647         if (mlx5_devx_regex_register_read(priv->ctx, id,
 648                         MLX5_RXP_RTRU_CSR_CAPABILITY, &rtru_max_num_entries)) {
 649                 DRV_LOG(ERR, "Failed to read RTRU capability!");
 650                 return -ENODEV;
 651         }
 652         rtru_max_num_entries = (rtru_max_num_entries & 0x00FF);
 653         rule_cnt = 0;
 654         pending = 0;
 655         while (rules->count > 0) {
 656                 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_INST) ||
 657                     (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_IM) ||
 658                     (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM)) {
 659                         if ((rules->rules[rule_cnt].type ==
 660                              MLX5_RXP_ROF_ENTRY_EM) &&
 661                             (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
 662                                 /* Skip EM rules programming. */
 663                                 if (pending > 0) {
 664                                         /* Flush any rules that are pending. */
 665                                         rule_offset = (rule_cnt - pending);
 666                                         ret = rxp_flush_rules(priv->ctx,
 667                                                 &rules->rules[rule_offset],
 668                                                 pending, id);
 669                                         if (ret < 0) {
 670                                                 DRV_LOG(ERR, "Flushing rules.");
 671                                                 return -ENODEV;
 672                                         }
 673                                         pending = 0;
 674                                 }
 675                                 rule_cnt++;
 676                         } else {
 677                                 pending++;
 678                                 rule_cnt++;
 679                                 /*
 680                                  * If parsing the last rule, or if reached the
 681                                  * maximum number of rules for this batch, then
 682                                  * flush the rules batch to the RXP.
 683                                  */
 684                                 if ((rules->count == 1) ||
 685                                     (pending == rtru_max_num_entries)) {
 686                                         rule_offset = (rule_cnt - pending);
 687                                         ret = rxp_flush_rules(priv->ctx,
 688                                                 &rules->rules[rule_offset],
 689                                                 pending, id);
 690                                         if (ret < 0) {
 691                                                 DRV_LOG(ERR, "Flushing rules.");
 692                                                 return -ENODEV;
 693                                         }
 694                                         pending = 0;
 695                                 }
 696                         }
 697                 } else if ((rules->rules[rule_cnt].type ==
 698                                 MLX5_RXP_ROF_ENTRY_EQ) ||
 699                          (rules->rules[rule_cnt].type ==
 700                                 MLX5_RXP_ROF_ENTRY_GTE) ||
 701                          (rules->rules[rule_cnt].type ==
 702                                 MLX5_RXP_ROF_ENTRY_LTE) ||
 703                          (rules->rules[rule_cnt].type ==
 704                                 MLX5_RXP_ROF_ENTRY_CHECKSUM) ||
 705                          (rules->rules[rule_cnt].type ==
 706                                 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM)) {
 707                         if (pending) {
 708                                 /* Flush rules before checking reg values. */
 709                                 rule_offset = (rule_cnt - pending);
 710                                 ret = rxp_flush_rules(priv->ctx,
 711                                         &rules->rules[rule_offset],
 712                                         pending, id);
 713                                 if (ret < 0) {
 714                                         DRV_LOG(ERR, "Failed to flush rules.");
 715                                         return -ENODEV;
 716                                 }
 717                         }
 718                         block = (rules->rules[rule_cnt].addr >> 16) & 0xFFFF;
 719                         if (block == 0)
 720                                 reg = MLX5_RXP_CSR_BASE_ADDRESS;
 721                         else if (block == 1)
 722                                 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS;
 723                         else {
 724                                 DRV_LOG(ERR, "Invalid ROF register 0x%08X!",
 725                                         rules->rules[rule_cnt].addr);
 726                                 return -EINVAL;
 727                         }
 728                         reg += (rules->rules[rule_cnt].addr & 0xFFFF) *
 729                                 MLX5_RXP_CSR_WIDTH;
 730                         ret = mlx5_devx_regex_register_read(priv->ctx, id,
 731                                                             reg, &val);
 732                         if (ret) {
 733                                 DRV_LOG(ERR, "RXP CSR read failed!");
 734                                 return ret;
 735                         }
 736                         if ((priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) &&
 737                             ((rules->rules[rule_cnt].type ==
 738                             MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM) &&
 739                             (val != rules->rules[rule_cnt].value))) {
 740                                 DRV_LOG(ERR, "Unexpected value for register:");
 741                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 742                                         rules->rules[rule_cnt].addr, val);
 743                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 744                                         rules->rules[rule_cnt].value);
 745                                         return -EINVAL;
 746                         } else if ((priv->prog_mode ==
 747                                  MLX5_RXP_PRIVATE_PROG_MODE) &&
 748                                  (rules->rules[rule_cnt].type ==
 749                                  MLX5_RXP_ROF_ENTRY_CHECKSUM) &&
 750                                  (val != rules->rules[rule_cnt].value)) {
 751                                 DRV_LOG(ERR, "Unexpected value for register:");
 752                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 753                                         rules->rules[rule_cnt].addr, val);
 754                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 755                                         rules->rules[rule_cnt].value);
 756                                 return -EINVAL;
 757                         } else if ((rules->rules[rule_cnt].type ==
 758                                         MLX5_RXP_ROF_ENTRY_EQ) &&
 759                                   (val != rules->rules[rule_cnt].value)) {
 760                                 DRV_LOG(ERR, "Unexpected value for register:");
 761                                 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
 762                                         rules->rules[rule_cnt].addr, val);
 763                                 DRV_LOG(ERR, "expected %" PRIx64 ".",
 764                                         rules->rules[rule_cnt].value);
 765                                         return -EINVAL;
 766                         } else if ((rules->rules[rule_cnt].type ==
 767                                         MLX5_RXP_ROF_ENTRY_GTE) &&
 768                                  (val < rules->rules[rule_cnt].value)) {
 769                                 DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
 770                                         rules->rules[rule_cnt].addr);
 771                                 DRV_LOG(ERR, "got %X, expected >= %" PRIx64 ".",
 772                                         val, rules->rules[rule_cnt].value);
 773                                 return -EINVAL;
 774                         } else if ((rules->rules[rule_cnt].type ==
 775                                         MLX5_RXP_ROF_ENTRY_LTE) &&
 776                                  (val > rules->rules[rule_cnt].value)) {
 777                                 DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
 778                                         rules->rules[rule_cnt].addr);
 779                                 DRV_LOG(ERR, "got %08X expected <= %" PRIx64,
 780                                         val, rules->rules[rule_cnt].value);
 781                                 return -EINVAL;
 782                         }
 783                         rule_cnt++;
 784                         pending = 0;
 785                 } else {
 786                         DRV_LOG(ERR, "Error: Invalid rule type %d!",
 787                                 rules->rules[rule_cnt].type);
 788                         return -EINVAL;
 789                 }
 790                 rules->count--;
 791         }
 792         return ret;
 793 }
 794
 795 /*
 796  * Shared memory programming mode, here all external db instructions are written
 797  * to EM via the host.
 798  */
 799 static int
 800 write_shared_rules(struct mlx5_regex_priv *priv,
 801                    struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
 802                    uint8_t db_to_program)
 803 {
 804         uint32_t rule_cnt, rof_rule_addr;
 805         uint64_t tmp_write_swap[4];
 806
 807         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 808                 return -EINVAL;
 809         if ((rules->count == 0) || (count == 0))
 810                 return -EINVAL;
 811         rule_cnt = 0;
 812         /*
 813          * Note the following section of code carries out a 32byte swap of
 814          * instruction to coincide with HW 32byte swap. This may need removed
 815          * in new variants of this programming function!
 816          */
 817         while (rule_cnt < rules->count) {
 818                 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM) &&
 819                     (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
 820                         /*
 821                          * Note there are always blocks of 8 instructions for
 822                          * 7's written sequentially. However there is no
 823                          * guarantee that all blocks are sequential!
 824                          */
 825                         if (count >= (rule_cnt + MLX5_RXP_INST_BLOCK_SIZE)) {
 826                                 /*
 827                                  * Ensure memory write not exceeding boundary
 828                                  * Check essential to ensure 0x10000 offset
 829                                  * accounted for!
 830                                  */
 831                                 if ((uint8_t *)((uint8_t *)
 832                                     priv->db[db_to_program].ptr +
 833                                     ((rules->rules[rule_cnt + 7].addr <<
 834                                     MLX5_RXP_INST_OFFSET))) >=
 835                                     ((uint8_t *)((uint8_t *)
 836                                     priv->db[db_to_program].ptr +
 837                                     MLX5_MAX_DB_SIZE))) {
 838                                         DRV_LOG(ERR, "DB exceeded memory!");
 839                                         return -ENODEV;
 840                                 }
 841                                 /*
 842                                  * Rule address Offset to align with RXP
 843                                  * external instruction offset.
 844                                  */
 845                                 rof_rule_addr = (rules->rules[rule_cnt].addr <<
 846                                                  MLX5_RXP_INST_OFFSET);
 847                                 /* 32 byte instruction swap (sw work around)! */
 848                                 tmp_write_swap[0] = le64toh(
 849                                         rules->rules[(rule_cnt + 4)].value);
 850                                 tmp_write_swap[1] = le64toh(
 851                                         rules->rules[(rule_cnt + 5)].value);
 852                                 tmp_write_swap[2] = le64toh(
 853                                         rules->rules[(rule_cnt + 6)].value);
 854                                 tmp_write_swap[3] = le64toh(
 855                                         rules->rules[(rule_cnt + 7)].value);
 856                                 /* Write only 4 of the 8 instructions. */
 857                                 memcpy((uint8_t *)((uint8_t *)
 858                                        priv->db[db_to_program].ptr +
 859                                        rof_rule_addr), &tmp_write_swap,
 860                                        (sizeof(uint64_t) * 4));
 861                                 /* Write 1st 4 rules of block after last 4. */
 862                                 rof_rule_addr = (rules->rules[
 863                                                  (rule_cnt + 4)].addr <<
 864                                                  MLX5_RXP_INST_OFFSET);
 865                                 tmp_write_swap[0] = le64toh(
 866                                         rules->rules[(rule_cnt + 0)].value);
 867                                 tmp_write_swap[1] = le64toh(
 868                                         rules->rules[(rule_cnt + 1)].value);
 869                                 tmp_write_swap[2] = le64toh(
 870                                         rules->rules[(rule_cnt + 2)].value);
 871                                 tmp_write_swap[3] = le64toh(
 872                                         rules->rules[(rule_cnt + 3)].value);
 873                                 memcpy((uint8_t *)((uint8_t *)
 874                                        priv->db[db_to_program].ptr +
 875                                        rof_rule_addr), &tmp_write_swap,
 876                                        (sizeof(uint64_t) * 4));
 877                         } else
 878                                 return -1;
 879                         /* Fast forward as already handled block of 8. */
 880                         rule_cnt += MLX5_RXP_INST_BLOCK_SIZE;
 881                 } else
 882                         rule_cnt++; /* Must be something other than EM rule. */
 883         }
 884         return 0;
 885 }
 886
 887 static int
 888 rxp_db_setup(struct mlx5_regex_priv *priv)
 889 {
 890         int ret;
 891         uint8_t i;
 892
 893         /* Setup database memories for both RXP engines + reprogram memory. */
 894         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 895                 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 0);
 896                 if (!priv->db[i].ptr) {
 897                         DRV_LOG(ERR, "Failed to alloc db memory!");
 898                         ret = ENODEV;
 899                         goto tidyup_error;
 900                 }
 901                 /* Register the memory. */
 902                 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx,
 903                                                         priv->db[i].ptr,
 904                                                         MLX5_MAX_DB_SIZE, 7);
 905                 if (!priv->db[i].umem.umem) {
 906                         DRV_LOG(ERR, "Failed to register memory!");
 907                         ret = ENODEV;
 908                         goto tidyup_error;
 909                 }
 910                 /* Ensure set all DB memory to 0's before setting up DB. */
 911                 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 912                 /* No data currently in database. */
 913                 priv->db[i].len = 0;
 914                 priv->db[i].active = false;
 915                 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED;
 916         }
 917         return 0;
 918 tidyup_error:
 919         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 920                 if (priv->db[i].ptr)
 921                         rte_free(priv->db[i].ptr);
 922                 if (priv->db[i].umem.umem)
 923                         mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem);
 924         }
 925         return -ret;
 926 }
 927
 928 int
 929 mlx5_regex_rules_db_import(struct rte_regexdev *dev,
 930                      const char *rule_db, uint32_t rule_db_len)
 931 {
 932         struct mlx5_regex_priv *priv = dev->data->dev_private;
 933         struct mlx5_rxp_ctl_rules_pgm *rules = NULL;
 934         uint32_t id;
 935         int ret;
 936
 937         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
 938                 DRV_LOG(ERR, "RXP programming mode not set!");
 939                 return -1;
 940         }
 941         if (rule_db == NULL) {
 942                 DRV_LOG(ERR, "Database empty!");
 943                 return -ENODEV;
 944         }
 945         if (rule_db_len == 0)
 946                 return -EINVAL;
 947         ret = rxp_parse_rof(rule_db, rule_db_len, &rules);
 948         if (ret) {
 949                 DRV_LOG(ERR, "Can't parse ROF file.");
 950                 return ret;
 951         }
 952         /* Need to ensure RXP not busy before stop! */
 953         for (id = 0; id < priv->nb_engines; id++) {
 954                 ret = rxp_stop_engine(priv->ctx, id);
 955                 if (ret) {
 956                         DRV_LOG(ERR, "Can't stop engine.");
 957                         ret = -ENODEV;
 958                         goto tidyup_error;
 959                 }
 960                 ret = program_rxp_rules(priv, rules, id);
 961                 if (ret < 0) {
 962                         DRV_LOG(ERR, "Failed to program rxp rules.");
 963                         ret = -ENODEV;
 964                         goto tidyup_error;
 965                 }
 966                 ret = rxp_start_engine(priv->ctx, id);
 967                 if (ret) {
 968                         DRV_LOG(ERR, "Can't start engine.");
 969                         ret = -ENODEV;
 970                         goto tidyup_error;
 971                 }
 972         }
 973         rte_free(rules);
 974         return 0;
 975 tidyup_error:
 976         rte_free(rules);
 977         return ret;
 978 }
 979
 980 int
 981 mlx5_regex_configure(struct rte_regexdev *dev,
 982                      const struct rte_regexdev_config *cfg)
 983 {
 984         struct mlx5_regex_priv *priv = dev->data->dev_private;
 985         int ret;
 986
 987         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 988                 return -1;
 989         priv->nb_queues = cfg->nb_queue_pairs;
 990         dev->data->dev_conf.nb_queue_pairs = priv->nb_queues;
 991         priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) *
 992                                 priv->nb_queues, 0);
 993         if (!priv->nb_queues) {
 994                 DRV_LOG(ERR, "can't allocate qps memory");
 995                 rte_errno = ENOMEM;
 996                 return -rte_errno;
 997         }
 998         priv->nb_max_matches = cfg->nb_max_matches;
 999         /* Setup rxp db memories. */
1000         if (rxp_db_setup(priv)) {
1001                 DRV_LOG(ERR, "Failed to setup RXP db memory");
1002                 rte_errno = ENOMEM;
1003                 return -rte_errno;
1004         }
1005         if (cfg->rule_db != NULL) {
1006                 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db,
1007                                                  cfg->rule_db_len);
1008                 if (ret < 0) {
1009                         DRV_LOG(ERR, "Failed to program rxp rules.");
1010                         rte_errno = ENODEV;
1011                         goto configure_error;
1012                 }
1013         } else
1014                 DRV_LOG(DEBUG, "Regex config without rules programming!");
1015         return 0;
1016 configure_error:
1017         if (priv->qps)
1018                 rte_free(priv->qps);
1019         return -rte_errno;
1020 }