drivers/regex/mlx5/mlx5_rxp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_log.h>
   6 #include <rte_errno.h>
   7 #include <rte_malloc.h>
   8 #include <rte_regexdev.h>
   9 #include <rte_regexdev_core.h>
  10 #include <rte_regexdev_driver.h>
  11 #include <sys/mman.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_devx_cmds.h>
  15 #include <mlx5_prm.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_regex.h"
  19 #include "mlx5_regex_utils.h"
  20 #include "mlx5_rxp_csrs.h"
  21 #include "mlx5_rxp.h"
  22
  23 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES
  24 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH
  25 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX
  26 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS
  27
  28 #define MLX5_REGEX_RXP_ROF2_LINE_LEN 34
  29
  30 /* Private Declarations */
  31 static int
  32 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
  33                        uint32_t address, uint32_t expected_value,
  34                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id);
  35 static int
  36 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use);
  37 static int
  38 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id);
  39 static int
  40 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id);
  41 static int
  42 program_rxp_rules(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
  43                   uint8_t id);
  44 static int
  45 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id);
  46 static int
  47 rxp_db_setup(struct mlx5_regex_priv *priv);
  48 static void
  49 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id);
  50 static int
  51 rxp_start_engine(struct ibv_context *ctx, uint8_t id);
  52 static int
  53 rxp_stop_engine(struct ibv_context *ctx, uint8_t id);
  54
  55 static void __rte_unused
  56 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused)
  57 {
  58         uint32_t reg, i;
  59
  60         /* Main CSRs*/
  61         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  62                 if (mlx5_devx_regex_register_read(ctx, id,
  63                                                   (MLX5_RXP_CSR_WIDTH * i) +
  64                                                   MLX5_RXP_CSR_BASE_ADDRESS,
  65                                                   &reg)) {
  66                         DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id);
  67                         return;
  68                 }
  69                 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x",
  70                         id, i, reg);
  71         }
  72         /* RTRU CSRs*/
  73         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  74                 if (mlx5_devx_regex_register_read(ctx, id,
  75                                                   (MLX5_RXP_CSR_WIDTH * i) +
  76                                                  MLX5_RXP_RTRU_CSR_BASE_ADDRESS,
  77                                                   &reg)) {
  78                         DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id);
  79                         return;
  80                 }
  81                 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x",
  82                         id, i, reg);
  83         }
  84         /* STAT CSRs */
  85         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  86                 if (mlx5_devx_regex_register_read(ctx, id,
  87                                                   (MLX5_RXP_CSR_WIDTH * i) +
  88                                                 MLX5_RXP_STATS_CSR_BASE_ADDRESS,
  89                                                   &reg)) {
  90                         DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id);
  91                         return;
  92                 }
  93                 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x",
  94                         id, i, reg);
  95         }
  96 }
  97
  98 int
  99 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused,
 100                     struct rte_regexdev_info *info)
 101 {
 102         info->max_matches = MLX5_REGEX_MAX_MATCHES;
 103         info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE;
 104         info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP;
 105         info->max_groups = MLX5_REGEX_MAX_GROUPS;
 106         info->max_queue_pairs = 1;
 107         info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F |
 108                               RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F;
 109         info->rule_flags = 0;
 110         info->max_queue_pairs = 10;
 111         return 0;
 112 }
 113
 114 static int
 115 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
 116                        uint32_t address, uint32_t expected_value,
 117                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id)
 118 {
 119         unsigned int i;
 120         int ret;
 121
 122         ret = -EBUSY;
 123         for (i = 0; i < timeout_ms; i++) {
 124                 if (mlx5_devx_regex_register_read(ctx, id, address, value))
 125                         return -1;
 126                 if ((*value & expected_mask) == expected_value) {
 127                         ret = 0;
 128                         break;
 129                 }
 130                 rte_delay_us(1000);
 131         }
 132         return ret;
 133 }
 134
 135 static int
 136 rxp_start_engine(struct ibv_context *ctx, uint8_t id)
 137 {
 138         uint32_t ctrl;
 139         int ret;
 140
 141         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 142         if (ret)
 143                 return ret;
 144         ctrl |= MLX5_RXP_CSR_CTRL_GO;
 145         ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C;
 146         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 147         return ret;
 148 }
 149
 150 static int
 151 rxp_stop_engine(struct ibv_context *ctx, uint8_t id)
 152 {
 153         uint32_t ctrl;
 154         int ret;
 155
 156         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 157         if (ret)
 158                 return ret;
 159         ctrl &= ~MLX5_RXP_CSR_CTRL_GO;
 160         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 161         return ret;
 162 }
 163
 164 static int
 165 rxp_init_rtru(struct mlx5_regex_priv *priv, uint8_t id, uint32_t init_bits)
 166 {
 167         uint32_t ctrl_value;
 168         uint32_t poll_value;
 169         uint32_t expected_value;
 170         uint32_t expected_mask;
 171         struct ibv_context *ctx = priv->ctx;
 172         int ret = 0;
 173
 174         /* Read the rtru ctrl CSR. */
 175         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 176                                             &ctrl_value);
 177         if (ret)
 178                 return -1;
 179         /* Clear any previous init modes. */
 180         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK);
 181         if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) {
 182                 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 183                 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 184                                                ctrl_value);
 185         }
 186         /* Set the init_mode bits in the rtru ctrl CSR. */
 187         ctrl_value |= init_bits;
 188         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 189                                        ctrl_value);
 190         /* Need to sleep for a short period after pulsing the rtru init bit. */
 191         rte_delay_us(20000);
 192         /* Poll the rtru status CSR until all the init done bits are set. */
 193         DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init");
 194         /* Set the init bit in the rtru ctrl CSR. */
 195         ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT;
 196         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 197                                        ctrl_value);
 198         /* Clear the init bit in the rtru ctrl CSR */
 199         ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT;
 200         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 201                                        ctrl_value);
 202         /* Check that the following bits are set in the RTRU_CSR. */
 203         if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) {
 204                 /* Must be incremental mode */
 205                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE;
 206         } else {
 207                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE |
 208                         MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE;
 209         }
 210         if (priv->is_bf2)
 211                 expected_value |= MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 212
 213
 214         expected_mask = expected_value;
 215         ret = rxp_poll_csr_for_value(ctx, &poll_value,
 216                                      MLX5_RXP_RTRU_CSR_STATUS,
 217                                      expected_value, expected_mask,
 218                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 219         if (ret)
 220                 return ret;
 221         DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value);
 222         /* Clear the init bit in the rtru ctrl CSR */
 223         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 224         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 225                                        ctrl_value);
 226         return 0;
 227 }
 228
 229 static int
 230 rxp_parse_line(char *line, uint32_t *type, uint32_t *address, uint64_t *value)
 231 {
 232         char *cur_pos;
 233
 234         if (*line == '\0' || *line == '#')
 235                 return  1;
 236         *type = strtoul(line, &cur_pos, 10);
 237         if (*cur_pos != ',' && *cur_pos != '\0')
 238                 return -1;
 239         *address = strtoul(cur_pos+1, &cur_pos, 16);
 240         if (*cur_pos != ',' && *cur_pos != '\0')
 241                 return -1;
 242         *value = strtoul(cur_pos+1, &cur_pos, 16);
 243         if (*cur_pos != ',' && *cur_pos != '\0')
 244                 return -1;
 245         return 0;
 246 }
 247
 248 static uint32_t
 249 rxp_get_reg_address(uint32_t address)
 250 {
 251         uint32_t block;
 252         uint32_t reg;
 253
 254         block = (address >> 16) & 0xFFFF;
 255         if (block == 0)
 256                 reg = MLX5_RXP_CSR_BASE_ADDRESS;
 257         else if (block == 1)
 258                 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS;
 259         else {
 260                 DRV_LOG(ERR, "Invalid ROF register 0x%08X!", address);
 261                         return UINT32_MAX;
 262         }
 263         reg += (address & 0xFFFF) * MLX5_RXP_CSR_WIDTH;
 264         return reg;
 265 }
 266
 267 #define MLX5_RXP_NUM_LINES_PER_BLOCK 8
 268
 269 static int
 270 rxp_program_rof(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
 271                 uint8_t id)
 272 {
 273         static const char del[] = "\n\r";
 274         char *line;
 275         char *tmp;
 276         uint32_t type = 0;
 277         uint32_t address;
 278         uint64_t val;
 279         uint32_t reg_val;
 280         int ret;
 281         int skip = -1;
 282         int last = 0;
 283         uint32_t temp;
 284         uint32_t tmp_addr;
 285         uint32_t rof_rule_addr;
 286         uint64_t tmp_write_swap[4];
 287         struct mlx5_rxp_rof_entry rules[8];
 288         int i;
 289         int db_free;
 290         int j;
 291
 292         tmp = rte_malloc("", len, 0);
 293         if (!tmp)
 294                 return -ENOMEM;
 295         memcpy(tmp, buf, len);
 296         db_free = mlnx_update_database(priv, id);
 297         if (db_free < 0) {
 298                 DRV_LOG(ERR, "Failed to setup db memory!");
 299                 rte_free(tmp);
 300                 return db_free;
 301         }
 302         for (line = strtok(tmp, del), j = 0; line; line = strtok(NULL, del),
 303              j++, last = type) {
 304                 ret = rxp_parse_line(line, &type, &address, &val);
 305                 if (ret != 0) {
 306                         if (ret < 0)
 307                                 goto parse_error;
 308                         continue;
 309                 }
 310                 switch (type) {
 311                 case MLX5_RXP_ROF_ENTRY_EQ:
 312                         if (skip == 0 && address == 0)
 313                                 skip = 1;
 314                         tmp_addr = rxp_get_reg_address(address);
 315                         if (tmp_addr == UINT32_MAX)
 316                                 goto parse_error;
 317                         ret = mlx5_devx_regex_register_read(priv->ctx, id,
 318                                                             tmp_addr, &reg_val);
 319                         if (ret)
 320                                 goto parse_error;
 321                         if (skip == -1 && address == 0) {
 322                                 if (val == reg_val) {
 323                                         skip = 0;
 324                                         continue;
 325                                 }
 326                         } else if (skip == 0) {
 327                                 if (val != reg_val) {
 328                                         DRV_LOG(ERR,
 329                                                 "got %08X expected == %" PRIx64,
 330                                                 reg_val, val);
 331                                         goto parse_error;
 332                                 }
 333                         }
 334                         break;
 335                 case MLX5_RXP_ROF_ENTRY_GTE:
 336                         if (skip == 0 && address == 0)
 337                                 skip = 1;
 338                         tmp_addr = rxp_get_reg_address(address);
 339                         if (tmp_addr == UINT32_MAX)
 340                                 goto parse_error;
 341                         ret = mlx5_devx_regex_register_read(priv->ctx, id,
 342                                                             tmp_addr, &reg_val);
 343                         if (ret)
 344                                 goto parse_error;
 345                         if (skip == -1 && address == 0) {
 346                                 if (reg_val >= val) {
 347                                         skip = 0;
 348                                         continue;
 349                                 }
 350                         } else if (skip == 0) {
 351                                 if (reg_val < val) {
 352                                         DRV_LOG(ERR,
 353                                                 "got %08X expected >= %" PRIx64,
 354                                                 reg_val, val);
 355                                         goto parse_error;
 356                                 }
 357                         }
 358                         break;
 359                 case MLX5_RXP_ROF_ENTRY_LTE:
 360                         tmp_addr = rxp_get_reg_address(address);
 361                         if (tmp_addr == UINT32_MAX)
 362                                 goto parse_error;
 363                         ret = mlx5_devx_regex_register_read(priv->ctx, id,
 364                                                             tmp_addr, &reg_val);
 365                         if (ret)
 366                                 goto parse_error;
 367                         if (skip == 0 && address == 0 &&
 368                             last != MLX5_RXP_ROF_ENTRY_GTE) {
 369                                 skip = 1;
 370                         } else if (skip == 0 && address == 0 &&
 371                                    last == MLX5_RXP_ROF_ENTRY_GTE) {
 372                                 if (reg_val > val)
 373                                         skip = -1;
 374                                 continue;
 375                         }
 376                         if (skip == -1 && address == 0) {
 377                                 if (reg_val <= val) {
 378                                         skip = 0;
 379                                         continue;
 380                                 }
 381                         } else if (skip == 0) {
 382                                 if (reg_val > val) {
 383                                         DRV_LOG(ERR,
 384                                                 "got %08X expected <= %" PRIx64,
 385                                                 reg_val, val);
 386                                         goto parse_error;
 387                                 }
 388                         }
 389                         break;
 390                 case MLX5_RXP_ROF_ENTRY_CHECKSUM:
 391                         break;
 392                 case MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM:
 393                         if (skip)
 394                                 continue;
 395                         tmp_addr = rxp_get_reg_address(address);
 396                         if (tmp_addr == UINT32_MAX)
 397                                 goto parse_error;
 398
 399                         ret = mlx5_devx_regex_register_read(priv->ctx, id,
 400                                                             tmp_addr, &reg_val);
 401                         if (ret) {
 402                                 DRV_LOG(ERR, "RXP CSR read failed!");
 403                                 return ret;
 404                         }
 405                         if (reg_val != val) {
 406                                 DRV_LOG(ERR, "got %08X expected <= %" PRIx64,
 407                                         reg_val, val);
 408                                 goto parse_error;
 409                         }
 410                         break;
 411                 case MLX5_RXP_ROF_ENTRY_IM:
 412                         if (skip)
 413                                 continue;
 414                         /*
 415                          * NOTE: All rules written to RXP must be carried out in
 416                          * triplets of: 2xData + 1xAddr.
 417                          * No optimisation is currently allowed in this
 418                          * sequence to perform less writes.
 419                          */
 420                         temp = val;
 421                         ret |= mlx5_devx_regex_register_write
 422                                         (priv->ctx, id,
 423                                          MLX5_RXP_RTRU_CSR_DATA_0, temp);
 424                         temp = (uint32_t)(val >> 32);
 425                         ret |= mlx5_devx_regex_register_write
 426                                         (priv->ctx, id,
 427                                          MLX5_RXP_RTRU_CSR_DATA_0 +
 428                                          MLX5_RXP_CSR_WIDTH, temp);
 429                         temp = address;
 430                         ret |= mlx5_devx_regex_register_write
 431                                         (priv->ctx, id, MLX5_RXP_RTRU_CSR_ADDR,
 432                                          temp);
 433                         if (ret) {
 434                                 DRV_LOG(ERR,
 435                                         "Failed to copy instructions to RXP.");
 436                                 goto parse_error;
 437                         }
 438                         break;
 439                 case MLX5_RXP_ROF_ENTRY_EM:
 440                         if (skip)
 441                                 continue;
 442                         for (i = 0; i < MLX5_RXP_NUM_LINES_PER_BLOCK; i++) {
 443                                 ret = rxp_parse_line(line, &type,
 444                                                      &rules[i].addr,
 445                                                      &rules[i].value);
 446                                 if (ret != 0)
 447                                         goto parse_error;
 448                                 if (i < (MLX5_RXP_NUM_LINES_PER_BLOCK - 1)) {
 449                                         line = strtok(NULL, del);
 450                                         if (!line)
 451                                                 goto parse_error;
 452                                 }
 453                         }
 454                         if ((uint8_t *)((uint8_t *)
 455                                         priv->db[id].ptr +
 456                                         ((rules[7].addr <<
 457                                          MLX5_RXP_INST_OFFSET))) >=
 458                                         ((uint8_t *)((uint8_t *)
 459                                         priv->db[id].ptr + MLX5_MAX_DB_SIZE))) {
 460                                 DRV_LOG(ERR, "DB exceeded memory!");
 461                                 goto parse_error;
 462                         }
 463                         /*
 464                          * Rule address Offset to align with RXP
 465                          * external instruction offset.
 466                          */
 467                         rof_rule_addr = (rules[0].addr << MLX5_RXP_INST_OFFSET);
 468                         /* 32 byte instruction swap (sw work around)! */
 469                         tmp_write_swap[0] = le64toh(rules[4].value);
 470                         tmp_write_swap[1] = le64toh(rules[5].value);
 471                         tmp_write_swap[2] = le64toh(rules[6].value);
 472                         tmp_write_swap[3] = le64toh(rules[7].value);
 473                         /* Write only 4 of the 8 instructions. */
 474                         memcpy((uint8_t *)((uint8_t *)
 475                                 priv->db[id].ptr + rof_rule_addr),
 476                                 &tmp_write_swap, (sizeof(uint64_t) * 4));
 477                         /* Write 1st 4 rules of block after last 4. */
 478                         rof_rule_addr = (rules[4].addr << MLX5_RXP_INST_OFFSET);
 479                         tmp_write_swap[0] = le64toh(rules[0].value);
 480                         tmp_write_swap[1] = le64toh(rules[1].value);
 481                         tmp_write_swap[2] = le64toh(rules[2].value);
 482                         tmp_write_swap[3] = le64toh(rules[3].value);
 483                         memcpy((uint8_t *)((uint8_t *)
 484                                 priv->db[id].ptr + rof_rule_addr),
 485                                 &tmp_write_swap, (sizeof(uint64_t) * 4));
 486                         break;
 487                 default:
 488                         break;
 489                 }
 490
 491         }
 492         ret = mlnx_set_database(priv, id, db_free);
 493         if (ret < 0) {
 494                 DRV_LOG(ERR, "Failed to register db memory!");
 495                 goto parse_error;
 496         }
 497         rte_free(tmp);
 498         return 0;
 499 parse_error:
 500         rte_free(tmp);
 501         return ret;
 502 }
 503
 504 static int
 505 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use)
 506 {
 507         int ret;
 508         uint32_t umem_id;
 509
 510         ret = mlx5_devx_regex_database_stop(priv->ctx, id);
 511         if (ret < 0) {
 512                 DRV_LOG(ERR, "stop engine failed!");
 513                 return ret;
 514         }
 515         umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem);
 516         ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0);
 517         if (ret < 0) {
 518                 DRV_LOG(ERR, "program db failed!");
 519                 return ret;
 520         }
 521         return 0;
 522 }
 523
 524 static int
 525 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id)
 526 {
 527         mlx5_devx_regex_database_resume(priv->ctx, id);
 528         return 0;
 529 }
 530
 531 /*
 532  * Assign db memory for RXP programming.
 533  */
 534 static int
 535 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id)
 536 {
 537         unsigned int i;
 538         uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED;
 539         uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED;
 540
 541         /* Check which database rxp_eng is currently located if any? */
 542         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 543              i++) {
 544                 if (priv->db[i].db_assigned_to_eng_num == id) {
 545                         eng_assigned = i;
 546                         break;
 547                 }
 548         }
 549         /*
 550          * If private mode then, we can keep the same db ptr as RXP will be
 551          * programming EM itself if necessary, however need to see if
 552          * programmed yet.
 553          */
 554         if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) &&
 555             (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED))
 556                 return eng_assigned;
 557         /* Check for inactive db memory to use. */
 558         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 559              i++) {
 560                 if (priv->db[i].active == true)
 561                         continue; /* Already in use, so skip db. */
 562                 /* Set this db to active now as free to use. */
 563                 priv->db[i].active = true;
 564                 /* Now unassign last db index in use by RXP Eng. */
 565                 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) {
 566                         priv->db[eng_assigned].active = false;
 567                         priv->db[eng_assigned].db_assigned_to_eng_num =
 568                                 MLX5_RXP_DB_NOT_ASSIGNED;
 569
 570                         /* Set all DB memory to 0's before setting up DB. */
 571                         memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 572                 }
 573                 /* Now reassign new db index with RXP Engine. */
 574                 priv->db[i].db_assigned_to_eng_num = id;
 575                 db_free = i;
 576                 break;
 577         }
 578         if (db_free == MLX5_RXP_DB_NOT_ASSIGNED)
 579                 return -1;
 580         return db_free;
 581 }
 582
 583 /*
 584  * Program RXP instruction db to RXP engine/s.
 585  */
 586 static int
 587 program_rxp_rules(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
 588                   uint8_t id)
 589 {
 590         int ret;
 591         uint32_t val;
 592
 593         ret = rxp_init_eng(priv, id);
 594         if (ret < 0)
 595                 return ret;
 596         /* Confirm the RXP is initialised. */
 597         if (mlx5_devx_regex_register_read(priv->ctx, id,
 598                                             MLX5_RXP_CSR_STATUS, &val)) {
 599                 DRV_LOG(ERR, "Failed to read from RXP!");
 600                 return -ENODEV;
 601         }
 602         if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) {
 603                 DRV_LOG(ERR, "RXP not initialised...");
 604                 return -EBUSY;
 605         }
 606         ret = mlx5_devx_regex_register_read(priv->ctx, id,
 607                                             MLX5_RXP_RTRU_CSR_CTRL, &val);
 608         if (ret) {
 609                 DRV_LOG(ERR, "CSR read failed!");
 610                 return -1;
 611         }
 612         val |= MLX5_RXP_RTRU_CSR_CTRL_GO;
 613         ret = mlx5_devx_regex_register_write(priv->ctx, id,
 614                                              MLX5_RXP_RTRU_CSR_CTRL, val);
 615         if (ret) {
 616                 DRV_LOG(ERR, "Can't program rof file!");
 617                 return -1;
 618         }
 619         ret = rxp_program_rof(priv, buf, len, id);
 620         if (ret) {
 621                 DRV_LOG(ERR, "Can't program rof file!");
 622                 return -1;
 623         }
 624         if (priv->is_bf2) {
 625                 ret = rxp_poll_csr_for_value
 626                         (priv->ctx, &val, MLX5_RXP_RTRU_CSR_STATUS,
 627                          MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 628                          MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 629                          MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 630                 if (ret < 0) {
 631                         DRV_LOG(ERR, "Rules update timeout: 0x%08X", val);
 632                         return ret;
 633                 }
 634                 DRV_LOG(DEBUG, "Rules update took %d cycles", ret);
 635         }
 636         if (mlx5_devx_regex_register_read(priv->ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 637                                           &val)) {
 638                 DRV_LOG(ERR, "CSR read failed!");
 639                 return -1;
 640         }
 641         val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO);
 642         if (mlx5_devx_regex_register_write(priv->ctx, id,
 643                                            MLX5_RXP_RTRU_CSR_CTRL, val)) {
 644                 DRV_LOG(ERR, "CSR write failed!");
 645                 return -1;
 646         }
 647         ret = mlx5_devx_regex_register_read(priv->ctx, id, MLX5_RXP_CSR_CTRL,
 648                                             &val);
 649         if (ret)
 650                 return ret;
 651         val &= ~MLX5_RXP_CSR_CTRL_INIT;
 652         ret = mlx5_devx_regex_register_write(priv->ctx, id, MLX5_RXP_CSR_CTRL,
 653                                              val);
 654         if (ret)
 655                 return ret;
 656         rxp_init_rtru(priv, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
 657         if (priv->is_bf2) {
 658                 ret = rxp_poll_csr_for_value(priv->ctx, &val,
 659                                              MLX5_RXP_CSR_STATUS,
 660                                              MLX5_RXP_CSR_STATUS_INIT_DONE,
 661                                              MLX5_RXP_CSR_STATUS_INIT_DONE,
 662                                              MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT,
 663                                              id);
 664                 if (ret) {
 665                         DRV_LOG(ERR, "Device init failed!");
 666                         return ret;
 667                 }
 668         }
 669         ret = mlnx_resume_database(priv, id);
 670         if (ret < 0) {
 671                 DRV_LOG(ERR, "Failed to resume engine!");
 672                 return ret;
 673         }
 674
 675         return ret;
 676
 677 }
 678
 679 static int
 680 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id)
 681 {
 682         uint32_t ctrl;
 683         uint32_t reg;
 684         struct ibv_context *ctx = priv->ctx;
 685         int ret;
 686
 687         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 688         if (ret)
 689                 return ret;
 690         if (ctrl & MLX5_RXP_CSR_CTRL_INIT) {
 691                 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 692                 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 693                                                      ctrl);
 694                 if (ret)
 695                         return ret;
 696         }
 697         ctrl |= MLX5_RXP_CSR_CTRL_INIT;
 698         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 699         if (ret)
 700                 return ret;
 701         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 702         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 703         if (ret)
 704                 return ret;
 705         rte_delay_us(20000);
 706         ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS,
 707                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 708                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 709                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 710         if (ret)
 711                 return ret;
 712         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 713         if (ret)
 714                 return ret;
 715         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 716         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 717                                              ctrl);
 718         if (ret)
 719                 return ret;
 720         ret = rxp_init_rtru(priv, id,
 721                             MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2);
 722         if (ret)
 723                 return ret;
 724         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5,
 725                                             &reg);
 726         if (ret)
 727                 return ret;
 728         DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16,
 729                 reg & 0xffff);
 730         if ((reg >> 16) >= priv->nb_max_matches)
 731                 ret = mlx5_devx_regex_register_write(ctx, id,
 732                                                      MLX5_RXP_CSR_MAX_MATCH,
 733                                                      priv->nb_max_matches);
 734         else
 735                 ret = mlx5_devx_regex_register_write(ctx, id,
 736                                                      MLX5_RXP_CSR_MAX_MATCH,
 737                                                      (reg >> 16));
 738         ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX,
 739                                          (reg & 0xFFFF));
 740         ret |= mlx5_devx_regex_register_write(ctx, id,
 741                                               MLX5_RXP_CSR_MAX_LATENCY, 0);
 742         ret |= mlx5_devx_regex_register_write(ctx, id,
 743                                               MLX5_RXP_CSR_MAX_PRI_THREAD, 0);
 744         return ret;
 745 }
 746
 747 static int
 748 rxp_db_setup(struct mlx5_regex_priv *priv)
 749 {
 750         int ret;
 751         uint8_t i;
 752
 753         /* Setup database memories for both RXP engines + reprogram memory. */
 754         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 755                 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 1 << 21);
 756                 if (!priv->db[i].ptr) {
 757                         DRV_LOG(ERR, "Failed to alloc db memory!");
 758                         ret = ENODEV;
 759                         goto tidyup_error;
 760                 }
 761                 /* Register the memory. */
 762                 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx,
 763                                                         priv->db[i].ptr,
 764                                                         MLX5_MAX_DB_SIZE, 7);
 765                 if (!priv->db[i].umem.umem) {
 766                         DRV_LOG(ERR, "Failed to register memory!");
 767                         ret = ENODEV;
 768                         goto tidyup_error;
 769                 }
 770                 /* Ensure set all DB memory to 0's before setting up DB. */
 771                 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 772                 /* No data currently in database. */
 773                 priv->db[i].len = 0;
 774                 priv->db[i].active = false;
 775                 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED;
 776         }
 777         return 0;
 778 tidyup_error:
 779         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 780                 if (priv->db[i].ptr)
 781                         rte_free(priv->db[i].ptr);
 782                 if (priv->db[i].umem.umem)
 783                         mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem);
 784         }
 785         return -ret;
 786 }
 787
 788 int
 789 mlx5_regex_rules_db_import(struct rte_regexdev *dev,
 790                      const char *rule_db, uint32_t rule_db_len)
 791 {
 792         struct mlx5_regex_priv *priv = dev->data->dev_private;
 793         struct mlx5_rxp_ctl_rules_pgm *rules = NULL;
 794         uint32_t id;
 795         int ret;
 796         uint32_t ver;
 797
 798         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
 799                 DRV_LOG(ERR, "RXP programming mode not set!");
 800                 return -1;
 801         }
 802         if (rule_db == NULL) {
 803                 DRV_LOG(ERR, "Database empty!");
 804                 return -ENODEV;
 805         }
 806         if (rule_db_len == 0)
 807                 return -EINVAL;
 808         if (mlx5_devx_regex_register_read(priv->ctx, 0,
 809                                           MLX5_RXP_CSR_BASE_ADDRESS, &ver)) {
 810                 DRV_LOG(ERR, "Failed to read Main CSRs Engine 0!");
 811                 return -1;
 812         }
 813         /* Need to ensure RXP not busy before stop! */
 814         for (id = 0; id < priv->nb_engines; id++) {
 815                 ret = rxp_stop_engine(priv->ctx, id);
 816                 if (ret) {
 817                         DRV_LOG(ERR, "Can't stop engine.");
 818                         ret = -ENODEV;
 819                         goto tidyup_error;
 820                 }
 821                 ret = program_rxp_rules(priv, rule_db, rule_db_len, id);
 822                 if (ret < 0) {
 823                         DRV_LOG(ERR, "Failed to program rxp rules.");
 824                         ret = -ENODEV;
 825                         goto tidyup_error;
 826                 }
 827                 ret = rxp_start_engine(priv->ctx, id);
 828                 if (ret) {
 829                         DRV_LOG(ERR, "Can't start engine.");
 830                         ret = -ENODEV;
 831                         goto tidyup_error;
 832                 }
 833         }
 834         rte_free(rules);
 835         return 0;
 836 tidyup_error:
 837         rte_free(rules);
 838         return ret;
 839 }
 840
 841 int
 842 mlx5_regex_configure(struct rte_regexdev *dev,
 843                      const struct rte_regexdev_config *cfg)
 844 {
 845         struct mlx5_regex_priv *priv = dev->data->dev_private;
 846         int ret;
 847
 848         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 849                 return -1;
 850         priv->nb_queues = cfg->nb_queue_pairs;
 851         dev->data->dev_conf.nb_queue_pairs = priv->nb_queues;
 852         priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) *
 853                                 priv->nb_queues, 0);
 854         if (!priv->nb_queues) {
 855                 DRV_LOG(ERR, "can't allocate qps memory");
 856                 rte_errno = ENOMEM;
 857                 return -rte_errno;
 858         }
 859         priv->nb_max_matches = cfg->nb_max_matches;
 860         /* Setup rxp db memories. */
 861         if (rxp_db_setup(priv)) {
 862                 DRV_LOG(ERR, "Failed to setup RXP db memory");
 863                 rte_errno = ENOMEM;
 864                 return -rte_errno;
 865         }
 866         if (cfg->rule_db != NULL) {
 867                 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db,
 868                                                  cfg->rule_db_len);
 869                 if (ret < 0) {
 870                         DRV_LOG(ERR, "Failed to program rxp rules.");
 871                         rte_errno = ENODEV;
 872                         goto configure_error;
 873                 }
 874         } else
 875                 DRV_LOG(DEBUG, "Regex config without rules programming!");
 876         return 0;
 877 configure_error:
 878         if (priv->qps)
 879                 rte_free(priv->qps);
 880         return -rte_errno;
 881 }