drivers/regex/mlx5/mlx5_rxp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_log.h>
   6 #include <rte_errno.h>
   7 #include <rte_malloc.h>
   8 #include <rte_regexdev.h>
   9 #include <rte_regexdev_core.h>
  10 #include <rte_regexdev_driver.h>
  11 #include <sys/mman.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_devx_cmds.h>
  15 #include <mlx5_prm.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_regex.h"
  19 #include "mlx5_regex_utils.h"
  20 #include "mlx5_rxp_csrs.h"
  21 #include "mlx5_rxp.h"
  22
  23 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES
  24 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH
  25 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX
  26 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS
  27
  28 #define MLX5_REGEX_RXP_ROF2_LINE_LEN 34
  29
  30 /* Private Declarations */
  31 static int
  32 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
  33                        uint32_t address, uint32_t expected_value,
  34                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id);
  35 static int
  36 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use);
  37 static int
  38 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id);
  39 static int
  40 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id);
  41 static int
  42 program_rxp_rules(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
  43                   uint8_t id);
  44 static int
  45 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id);
  46 static int
  47 rxp_db_setup(struct mlx5_regex_priv *priv);
  48 static void
  49 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id);
  50 static int
  51 rxp_start_engine(struct ibv_context *ctx, uint8_t id);
  52 static int
  53 rxp_stop_engine(struct ibv_context *ctx, uint8_t id);
  54
  55 static void __rte_unused
  56 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused)
  57 {
  58         uint32_t reg, i;
  59
  60         /* Main CSRs*/
  61         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  62                 if (mlx5_devx_regex_register_read(ctx, id,
  63                                                   (MLX5_RXP_CSR_WIDTH * i) +
  64                                                   MLX5_RXP_CSR_BASE_ADDRESS,
  65                                                   &reg)) {
  66                         DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id);
  67                         return;
  68                 }
  69                 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x",
  70                         id, i, reg);
  71         }
  72         /* RTRU CSRs*/
  73         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  74                 if (mlx5_devx_regex_register_read(ctx, id,
  75                                                   (MLX5_RXP_CSR_WIDTH * i) +
  76                                                  MLX5_RXP_RTRU_CSR_BASE_ADDRESS,
  77                                                   &reg)) {
  78                         DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id);
  79                         return;
  80                 }
  81                 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x",
  82                         id, i, reg);
  83         }
  84         /* STAT CSRs */
  85         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  86                 if (mlx5_devx_regex_register_read(ctx, id,
  87                                                   (MLX5_RXP_CSR_WIDTH * i) +
  88                                                 MLX5_RXP_STATS_CSR_BASE_ADDRESS,
  89                                                   &reg)) {
  90                         DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id);
  91                         return;
  92                 }
  93                 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x",
  94                         id, i, reg);
  95         }
  96 }
  97
  98 int
  99 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused,
 100                     struct rte_regexdev_info *info)
 101 {
 102         info->max_matches = MLX5_REGEX_MAX_MATCHES;
 103         info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE;
 104         info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP;
 105         info->max_groups = MLX5_REGEX_MAX_GROUPS;
 106         info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F |
 107                               RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F;
 108         info->rule_flags = 0;
 109         info->max_queue_pairs = UINT16_MAX;
 110         return 0;
 111 }
 112
 113 static int
 114 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
 115                        uint32_t address, uint32_t expected_value,
 116                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id)
 117 {
 118         unsigned int i;
 119         int ret;
 120
 121         ret = -EBUSY;
 122         for (i = 0; i < timeout_ms; i++) {
 123                 if (mlx5_devx_regex_register_read(ctx, id, address, value))
 124                         return -1;
 125                 if ((*value & expected_mask) == expected_value) {
 126                         ret = 0;
 127                         break;
 128                 }
 129                 rte_delay_us(1000);
 130         }
 131         return ret;
 132 }
 133
 134 static int
 135 rxp_start_engine(struct ibv_context *ctx, uint8_t id)
 136 {
 137         uint32_t ctrl;
 138         int ret;
 139
 140         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 141         if (ret)
 142                 return ret;
 143         ctrl |= MLX5_RXP_CSR_CTRL_GO;
 144         ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C;
 145         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 146         return ret;
 147 }
 148
 149 static int
 150 rxp_stop_engine(struct ibv_context *ctx, uint8_t id)
 151 {
 152         uint32_t ctrl;
 153         int ret;
 154
 155         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 156         if (ret)
 157                 return ret;
 158         ctrl &= ~MLX5_RXP_CSR_CTRL_GO;
 159         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 160         return ret;
 161 }
 162
 163 static int
 164 rxp_init_rtru(struct mlx5_regex_priv *priv, uint8_t id, uint32_t init_bits)
 165 {
 166         uint32_t ctrl_value;
 167         uint32_t poll_value;
 168         uint32_t expected_value;
 169         uint32_t expected_mask;
 170         struct ibv_context *ctx = priv->cdev->ctx;
 171         int ret = 0;
 172
 173         /* Read the rtru ctrl CSR. */
 174         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 175                                             &ctrl_value);
 176         if (ret)
 177                 return -1;
 178         /* Clear any previous init modes. */
 179         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK);
 180         if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) {
 181                 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 182                 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 183                                                ctrl_value);
 184         }
 185         /* Set the init_mode bits in the rtru ctrl CSR. */
 186         ctrl_value |= init_bits;
 187         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 188                                        ctrl_value);
 189         /* Need to sleep for a short period after pulsing the rtru init bit. */
 190         rte_delay_us(20000);
 191         /* Poll the rtru status CSR until all the init done bits are set. */
 192         DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init");
 193         /* Set the init bit in the rtru ctrl CSR. */
 194         ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT;
 195         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 196                                        ctrl_value);
 197         /* Clear the init bit in the rtru ctrl CSR */
 198         ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT;
 199         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 200                                        ctrl_value);
 201         /* Check that the following bits are set in the RTRU_CSR. */
 202         if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) {
 203                 /* Must be incremental mode */
 204                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE;
 205         } else {
 206                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE |
 207                         MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE;
 208         }
 209         if (priv->is_bf2)
 210                 expected_value |= MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 211
 212
 213         expected_mask = expected_value;
 214         ret = rxp_poll_csr_for_value(ctx, &poll_value,
 215                                      MLX5_RXP_RTRU_CSR_STATUS,
 216                                      expected_value, expected_mask,
 217                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 218         if (ret)
 219                 return ret;
 220         DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value);
 221         /* Clear the init bit in the rtru ctrl CSR */
 222         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 223         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 224                                        ctrl_value);
 225         return 0;
 226 }
 227
 228 static int
 229 rxp_parse_line(char *line, uint32_t *type, uint32_t *address, uint64_t *value)
 230 {
 231         char *cur_pos;
 232
 233         if (*line == '\0' || *line == '#')
 234                 return  1;
 235         *type = strtoul(line, &cur_pos, 10);
 236         if (*cur_pos != ',' && *cur_pos != '\0')
 237                 return -1;
 238         *address = strtoul(cur_pos+1, &cur_pos, 16);
 239         if (*cur_pos != ',' && *cur_pos != '\0')
 240                 return -1;
 241         *value = strtoul(cur_pos+1, &cur_pos, 16);
 242         if (*cur_pos != ',' && *cur_pos != '\0')
 243                 return -1;
 244         return 0;
 245 }
 246
 247 static uint32_t
 248 rxp_get_reg_address(uint32_t address)
 249 {
 250         uint32_t block;
 251         uint32_t reg;
 252
 253         block = (address >> 16) & 0xFFFF;
 254         if (block == 0)
 255                 reg = MLX5_RXP_CSR_BASE_ADDRESS;
 256         else if (block == 1)
 257                 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS;
 258         else {
 259                 DRV_LOG(ERR, "Invalid ROF register 0x%08X!", address);
 260                         return UINT32_MAX;
 261         }
 262         reg += (address & 0xFFFF) * MLX5_RXP_CSR_WIDTH;
 263         return reg;
 264 }
 265
 266 #define MLX5_RXP_NUM_LINES_PER_BLOCK 8
 267
 268 static int
 269 rxp_program_rof(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
 270                 uint8_t id)
 271 {
 272         static const char del[] = "\n\r";
 273         char *line;
 274         char *tmp;
 275         uint32_t type = 0;
 276         uint32_t address;
 277         uint64_t val;
 278         uint32_t reg_val;
 279         int ret;
 280         int skip = -1;
 281         int last = 0;
 282         uint32_t temp;
 283         uint32_t tmp_addr;
 284         uint32_t rof_rule_addr;
 285         uint64_t tmp_write_swap[4];
 286         struct mlx5_rxp_rof_entry rules[8];
 287         int i;
 288         int db_free;
 289         int j;
 290
 291         tmp = rte_malloc("", len, 0);
 292         if (!tmp)
 293                 return -ENOMEM;
 294         memcpy(tmp, buf, len);
 295         db_free = mlnx_update_database(priv, id);
 296         if (db_free < 0) {
 297                 DRV_LOG(ERR, "Failed to setup db memory!");
 298                 rte_free(tmp);
 299                 return db_free;
 300         }
 301         for (line = strtok(tmp, del), j = 0; line; line = strtok(NULL, del),
 302              j++, last = type) {
 303                 ret = rxp_parse_line(line, &type, &address, &val);
 304                 if (ret != 0) {
 305                         if (ret < 0)
 306                                 goto parse_error;
 307                         continue;
 308                 }
 309                 switch (type) {
 310                 case MLX5_RXP_ROF_ENTRY_EQ:
 311                         if (skip == 0 && address == 0)
 312                                 skip = 1;
 313                         tmp_addr = rxp_get_reg_address(address);
 314                         if (tmp_addr == UINT32_MAX)
 315                                 goto parse_error;
 316                         ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id,
 317                                                             tmp_addr, &reg_val);
 318                         if (ret)
 319                                 goto parse_error;
 320                         if (skip == -1 && address == 0) {
 321                                 if (val == reg_val) {
 322                                         skip = 0;
 323                                         continue;
 324                                 }
 325                         } else if (skip == 0) {
 326                                 if (val != reg_val) {
 327                                         DRV_LOG(ERR,
 328                                                 "got %08X expected == %" PRIx64,
 329                                                 reg_val, val);
 330                                         goto parse_error;
 331                                 }
 332                         }
 333                         break;
 334                 case MLX5_RXP_ROF_ENTRY_GTE:
 335                         if (skip == 0 && address == 0)
 336                                 skip = 1;
 337                         tmp_addr = rxp_get_reg_address(address);
 338                         if (tmp_addr == UINT32_MAX)
 339                                 goto parse_error;
 340                         ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id,
 341                                                             tmp_addr, &reg_val);
 342                         if (ret)
 343                                 goto parse_error;
 344                         if (skip == -1 && address == 0) {
 345                                 if (reg_val >= val) {
 346                                         skip = 0;
 347                                         continue;
 348                                 }
 349                         } else if (skip == 0) {
 350                                 if (reg_val < val) {
 351                                         DRV_LOG(ERR,
 352                                                 "got %08X expected >= %" PRIx64,
 353                                                 reg_val, val);
 354                                         goto parse_error;
 355                                 }
 356                         }
 357                         break;
 358                 case MLX5_RXP_ROF_ENTRY_LTE:
 359                         tmp_addr = rxp_get_reg_address(address);
 360                         if (tmp_addr == UINT32_MAX)
 361                                 goto parse_error;
 362                         ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id,
 363                                                             tmp_addr, &reg_val);
 364                         if (ret)
 365                                 goto parse_error;
 366                         if (skip == 0 && address == 0 &&
 367                             last != MLX5_RXP_ROF_ENTRY_GTE) {
 368                                 skip = 1;
 369                         } else if (skip == 0 && address == 0 &&
 370                                    last == MLX5_RXP_ROF_ENTRY_GTE) {
 371                                 if (reg_val > val)
 372                                         skip = -1;
 373                                 continue;
 374                         }
 375                         if (skip == -1 && address == 0) {
 376                                 if (reg_val <= val) {
 377                                         skip = 0;
 378                                         continue;
 379                                 }
 380                         } else if (skip == 0) {
 381                                 if (reg_val > val) {
 382                                         DRV_LOG(ERR,
 383                                                 "got %08X expected <= %" PRIx64,
 384                                                 reg_val, val);
 385                                         goto parse_error;
 386                                 }
 387                         }
 388                         break;
 389                 case MLX5_RXP_ROF_ENTRY_CHECKSUM:
 390                         break;
 391                 case MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM:
 392                         if (skip)
 393                                 continue;
 394                         tmp_addr = rxp_get_reg_address(address);
 395                         if (tmp_addr == UINT32_MAX)
 396                                 goto parse_error;
 397
 398                         ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id,
 399                                                             tmp_addr, &reg_val);
 400                         if (ret) {
 401                                 DRV_LOG(ERR, "RXP CSR read failed!");
 402                                 return ret;
 403                         }
 404                         if (reg_val != val) {
 405                                 DRV_LOG(ERR, "got %08X expected <= %" PRIx64,
 406                                         reg_val, val);
 407                                 goto parse_error;
 408                         }
 409                         break;
 410                 case MLX5_RXP_ROF_ENTRY_IM:
 411                         if (skip)
 412                                 continue;
 413                         /*
 414                          * NOTE: All rules written to RXP must be carried out in
 415                          * triplets of: 2xData + 1xAddr.
 416                          * No optimisation is currently allowed in this
 417                          * sequence to perform less writes.
 418                          */
 419                         temp = val;
 420                         ret |= mlx5_devx_regex_register_write
 421                                         (priv->cdev->ctx, id,
 422                                          MLX5_RXP_RTRU_CSR_DATA_0, temp);
 423                         temp = (uint32_t)(val >> 32);
 424                         ret |= mlx5_devx_regex_register_write
 425                                         (priv->cdev->ctx, id,
 426                                          MLX5_RXP_RTRU_CSR_DATA_0 +
 427                                          MLX5_RXP_CSR_WIDTH, temp);
 428                         temp = address;
 429                         ret |= mlx5_devx_regex_register_write
 430                                         (priv->cdev->ctx, id,
 431                                          MLX5_RXP_RTRU_CSR_ADDR, temp);
 432                         if (ret) {
 433                                 DRV_LOG(ERR,
 434                                         "Failed to copy instructions to RXP.");
 435                                 goto parse_error;
 436                         }
 437                         break;
 438                 case MLX5_RXP_ROF_ENTRY_EM:
 439                         if (skip)
 440                                 continue;
 441                         for (i = 0; i < MLX5_RXP_NUM_LINES_PER_BLOCK; i++) {
 442                                 ret = rxp_parse_line(line, &type,
 443                                                      &rules[i].addr,
 444                                                      &rules[i].value);
 445                                 if (ret != 0)
 446                                         goto parse_error;
 447                                 if (i < (MLX5_RXP_NUM_LINES_PER_BLOCK - 1)) {
 448                                         line = strtok(NULL, del);
 449                                         if (!line)
 450                                                 goto parse_error;
 451                                 }
 452                         }
 453                         if ((uint8_t *)((uint8_t *)
 454                                         priv->db[id].ptr +
 455                                         ((rules[7].addr <<
 456                                          MLX5_RXP_INST_OFFSET))) >=
 457                                         ((uint8_t *)((uint8_t *)
 458                                         priv->db[id].ptr + MLX5_MAX_DB_SIZE))) {
 459                                 DRV_LOG(ERR, "DB exceeded memory!");
 460                                 goto parse_error;
 461                         }
 462                         /*
 463                          * Rule address Offset to align with RXP
 464                          * external instruction offset.
 465                          */
 466                         rof_rule_addr = (rules[0].addr << MLX5_RXP_INST_OFFSET);
 467                         /* 32 byte instruction swap (sw work around)! */
 468                         tmp_write_swap[0] = le64toh(rules[4].value);
 469                         tmp_write_swap[1] = le64toh(rules[5].value);
 470                         tmp_write_swap[2] = le64toh(rules[6].value);
 471                         tmp_write_swap[3] = le64toh(rules[7].value);
 472                         /* Write only 4 of the 8 instructions. */
 473                         memcpy((uint8_t *)((uint8_t *)
 474                                 priv->db[id].ptr + rof_rule_addr),
 475                                 &tmp_write_swap, (sizeof(uint64_t) * 4));
 476                         /* Write 1st 4 rules of block after last 4. */
 477                         rof_rule_addr = (rules[4].addr << MLX5_RXP_INST_OFFSET);
 478                         tmp_write_swap[0] = le64toh(rules[0].value);
 479                         tmp_write_swap[1] = le64toh(rules[1].value);
 480                         tmp_write_swap[2] = le64toh(rules[2].value);
 481                         tmp_write_swap[3] = le64toh(rules[3].value);
 482                         memcpy((uint8_t *)((uint8_t *)
 483                                 priv->db[id].ptr + rof_rule_addr),
 484                                 &tmp_write_swap, (sizeof(uint64_t) * 4));
 485                         break;
 486                 default:
 487                         break;
 488                 }
 489
 490         }
 491         ret = mlnx_set_database(priv, id, db_free);
 492         if (ret < 0) {
 493                 DRV_LOG(ERR, "Failed to register db memory!");
 494                 goto parse_error;
 495         }
 496         rte_free(tmp);
 497         return 0;
 498 parse_error:
 499         rte_free(tmp);
 500         return ret;
 501 }
 502
 503 static int
 504 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use)
 505 {
 506         int ret;
 507         uint32_t umem_id;
 508
 509         ret = mlx5_devx_regex_database_stop(priv->cdev->ctx, id);
 510         if (ret < 0) {
 511                 DRV_LOG(ERR, "stop engine failed!");
 512                 return ret;
 513         }
 514         umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem);
 515         ret = mlx5_devx_regex_database_program(priv->cdev->ctx, id, umem_id, 0);
 516         if (ret < 0) {
 517                 DRV_LOG(ERR, "program db failed!");
 518                 return ret;
 519         }
 520         return 0;
 521 }
 522
 523 static int
 524 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id)
 525 {
 526         mlx5_devx_regex_database_resume(priv->cdev->ctx, id);
 527         return 0;
 528 }
 529
 530 /*
 531  * Assign db memory for RXP programming.
 532  */
 533 static int
 534 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id)
 535 {
 536         unsigned int i;
 537         uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED;
 538         uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED;
 539
 540         /* Check which database rxp_eng is currently located if any? */
 541         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 542              i++) {
 543                 if (priv->db[i].db_assigned_to_eng_num == id) {
 544                         eng_assigned = i;
 545                         break;
 546                 }
 547         }
 548         /*
 549          * If private mode then, we can keep the same db ptr as RXP will be
 550          * programming EM itself if necessary, however need to see if
 551          * programmed yet.
 552          */
 553         if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) &&
 554             (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED))
 555                 return eng_assigned;
 556         /* Check for inactive db memory to use. */
 557         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 558              i++) {
 559                 if (priv->db[i].active == true)
 560                         continue; /* Already in use, so skip db. */
 561                 /* Set this db to active now as free to use. */
 562                 priv->db[i].active = true;
 563                 /* Now unassign last db index in use by RXP Eng. */
 564                 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) {
 565                         priv->db[eng_assigned].active = false;
 566                         priv->db[eng_assigned].db_assigned_to_eng_num =
 567                                 MLX5_RXP_DB_NOT_ASSIGNED;
 568
 569                         /* Set all DB memory to 0's before setting up DB. */
 570                         memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 571                 }
 572                 /* Now reassign new db index with RXP Engine. */
 573                 priv->db[i].db_assigned_to_eng_num = id;
 574                 db_free = i;
 575                 break;
 576         }
 577         if (db_free == MLX5_RXP_DB_NOT_ASSIGNED)
 578                 return -1;
 579         return db_free;
 580 }
 581
 582 /*
 583  * Program RXP instruction db to RXP engine/s.
 584  */
 585 static int
 586 program_rxp_rules(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
 587                   uint8_t id)
 588 {
 589         int ret;
 590         uint32_t val;
 591         struct ibv_context *ctx = priv->cdev->ctx;
 592
 593         ret = rxp_init_eng(priv, id);
 594         if (ret < 0)
 595                 return ret;
 596         /* Confirm the RXP is initialised. */
 597         if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_STATUS, &val)) {
 598                 DRV_LOG(ERR, "Failed to read from RXP!");
 599                 return -ENODEV;
 600         }
 601         if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) {
 602                 DRV_LOG(ERR, "RXP not initialised...");
 603                 return -EBUSY;
 604         }
 605         ret = mlx5_devx_regex_register_read(ctx, id,
 606                                             MLX5_RXP_RTRU_CSR_CTRL, &val);
 607         if (ret) {
 608                 DRV_LOG(ERR, "CSR read failed!");
 609                 return -1;
 610         }
 611         val |= MLX5_RXP_RTRU_CSR_CTRL_GO;
 612         ret = mlx5_devx_regex_register_write(ctx, id,
 613                                              MLX5_RXP_RTRU_CSR_CTRL, val);
 614         if (ret) {
 615                 DRV_LOG(ERR, "Can't program rof file!");
 616                 return -1;
 617         }
 618         ret = rxp_program_rof(priv, buf, len, id);
 619         if (ret) {
 620                 DRV_LOG(ERR, "Can't program rof file!");
 621                 return -1;
 622         }
 623         if (priv->is_bf2) {
 624                 ret = rxp_poll_csr_for_value
 625                         (ctx, &val, MLX5_RXP_RTRU_CSR_STATUS,
 626                          MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 627                          MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 628                          MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 629                 if (ret < 0) {
 630                         DRV_LOG(ERR, "Rules update timeout: 0x%08X", val);
 631                         return ret;
 632                 }
 633                 DRV_LOG(DEBUG, "Rules update took %d cycles", ret);
 634         }
 635         if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 636                                           &val)) {
 637                 DRV_LOG(ERR, "CSR read failed!");
 638                 return -1;
 639         }
 640         val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO);
 641         if (mlx5_devx_regex_register_write(ctx, id,
 642                                            MLX5_RXP_RTRU_CSR_CTRL, val)) {
 643                 DRV_LOG(ERR, "CSR write failed!");
 644                 return -1;
 645         }
 646         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &val);
 647         if (ret)
 648                 return ret;
 649         val &= ~MLX5_RXP_CSR_CTRL_INIT;
 650         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, val);
 651         if (ret)
 652                 return ret;
 653         rxp_init_rtru(priv, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
 654         if (priv->is_bf2) {
 655                 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_CSR_STATUS,
 656                                              MLX5_RXP_CSR_STATUS_INIT_DONE,
 657                                              MLX5_RXP_CSR_STATUS_INIT_DONE,
 658                                              MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT,
 659                                              id);
 660                 if (ret) {
 661                         DRV_LOG(ERR, "Device init failed!");
 662                         return ret;
 663                 }
 664         }
 665         ret = mlnx_resume_database(priv, id);
 666         if (ret < 0) {
 667                 DRV_LOG(ERR, "Failed to resume engine!");
 668                 return ret;
 669         }
 670
 671         return ret;
 672
 673 }
 674
 675 static int
 676 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id)
 677 {
 678         uint32_t ctrl;
 679         uint32_t reg;
 680         struct ibv_context *ctx = priv->cdev->ctx;
 681         int ret;
 682
 683         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 684         if (ret)
 685                 return ret;
 686         if (ctrl & MLX5_RXP_CSR_CTRL_INIT) {
 687                 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 688                 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 689                                                      ctrl);
 690                 if (ret)
 691                         return ret;
 692         }
 693         ctrl |= MLX5_RXP_CSR_CTRL_INIT;
 694         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 695         if (ret)
 696                 return ret;
 697         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 698         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 699         if (ret)
 700                 return ret;
 701         rte_delay_us(20000);
 702         ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS,
 703                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 704                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 705                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 706         if (ret)
 707                 return ret;
 708         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 709         if (ret)
 710                 return ret;
 711         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 712         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 713                                              ctrl);
 714         if (ret)
 715                 return ret;
 716         ret = rxp_init_rtru(priv, id,
 717                             MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2);
 718         if (ret)
 719                 return ret;
 720         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5,
 721                                             &reg);
 722         if (ret)
 723                 return ret;
 724         DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16,
 725                 reg & 0xffff);
 726         if ((reg >> 16) >= priv->nb_max_matches)
 727                 ret = mlx5_devx_regex_register_write(ctx, id,
 728                                                      MLX5_RXP_CSR_MAX_MATCH,
 729                                                      priv->nb_max_matches);
 730         else
 731                 ret = mlx5_devx_regex_register_write(ctx, id,
 732                                                      MLX5_RXP_CSR_MAX_MATCH,
 733                                                      (reg >> 16));
 734         ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX,
 735                                          (reg & 0xFFFF));
 736         ret |= mlx5_devx_regex_register_write(ctx, id,
 737                                               MLX5_RXP_CSR_MAX_LATENCY, 0);
 738         ret |= mlx5_devx_regex_register_write(ctx, id,
 739                                               MLX5_RXP_CSR_MAX_PRI_THREAD, 0);
 740         return ret;
 741 }
 742
 743 static int
 744 rxp_db_setup(struct mlx5_regex_priv *priv)
 745 {
 746         int ret;
 747         uint8_t i;
 748
 749         /* Setup database memories for both RXP engines + reprogram memory. */
 750         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 751                 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 1 << 21);
 752                 if (!priv->db[i].ptr) {
 753                         DRV_LOG(ERR, "Failed to alloc db memory!");
 754                         ret = ENODEV;
 755                         goto tidyup_error;
 756                 }
 757                 /* Register the memory. */
 758                 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg
 759                                                         (priv->cdev->ctx,
 760                                                          priv->db[i].ptr,
 761                                                          MLX5_MAX_DB_SIZE, 7);
 762                 if (!priv->db[i].umem.umem) {
 763                         DRV_LOG(ERR, "Failed to register memory!");
 764                         ret = ENODEV;
 765                         goto tidyup_error;
 766                 }
 767                 /* Ensure set all DB memory to 0's before setting up DB. */
 768                 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 769                 /* No data currently in database. */
 770                 priv->db[i].len = 0;
 771                 priv->db[i].active = false;
 772                 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED;
 773         }
 774         return 0;
 775 tidyup_error:
 776         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 777                 if (priv->db[i].ptr)
 778                         rte_free(priv->db[i].ptr);
 779                 if (priv->db[i].umem.umem)
 780                         mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem);
 781         }
 782         return -ret;
 783 }
 784
 785 int
 786 mlx5_regex_rules_db_import(struct rte_regexdev *dev,
 787                      const char *rule_db, uint32_t rule_db_len)
 788 {
 789         struct mlx5_regex_priv *priv = dev->data->dev_private;
 790         struct mlx5_rxp_ctl_rules_pgm *rules = NULL;
 791         uint32_t id;
 792         int ret;
 793         uint32_t ver;
 794
 795         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
 796                 DRV_LOG(ERR, "RXP programming mode not set!");
 797                 return -1;
 798         }
 799         if (rule_db == NULL) {
 800                 DRV_LOG(ERR, "Database empty!");
 801                 return -ENODEV;
 802         }
 803         if (rule_db_len == 0)
 804                 return -EINVAL;
 805         if (mlx5_devx_regex_register_read(priv->cdev->ctx, 0,
 806                                           MLX5_RXP_CSR_BASE_ADDRESS, &ver)) {
 807                 DRV_LOG(ERR, "Failed to read Main CSRs Engine 0!");
 808                 return -1;
 809         }
 810         /* Need to ensure RXP not busy before stop! */
 811         for (id = 0; id < priv->nb_engines; id++) {
 812                 ret = rxp_stop_engine(priv->cdev->ctx, id);
 813                 if (ret) {
 814                         DRV_LOG(ERR, "Can't stop engine.");
 815                         ret = -ENODEV;
 816                         goto tidyup_error;
 817                 }
 818                 ret = program_rxp_rules(priv, rule_db, rule_db_len, id);
 819                 if (ret < 0) {
 820                         DRV_LOG(ERR, "Failed to program rxp rules.");
 821                         ret = -ENODEV;
 822                         goto tidyup_error;
 823                 }
 824                 ret = rxp_start_engine(priv->cdev->ctx, id);
 825                 if (ret) {
 826                         DRV_LOG(ERR, "Can't start engine.");
 827                         ret = -ENODEV;
 828                         goto tidyup_error;
 829                 }
 830         }
 831         rte_free(rules);
 832         return 0;
 833 tidyup_error:
 834         rte_free(rules);
 835         return ret;
 836 }
 837
 838 int
 839 mlx5_regex_configure(struct rte_regexdev *dev,
 840                      const struct rte_regexdev_config *cfg)
 841 {
 842         struct mlx5_regex_priv *priv = dev->data->dev_private;
 843         int ret;
 844
 845         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 846                 return -1;
 847         priv->nb_queues = cfg->nb_queue_pairs;
 848         dev->data->dev_conf.nb_queue_pairs = priv->nb_queues;
 849         priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) *
 850                                 priv->nb_queues, 0);
 851         if (!priv->nb_queues) {
 852                 DRV_LOG(ERR, "can't allocate qps memory");
 853                 rte_errno = ENOMEM;
 854                 return -rte_errno;
 855         }
 856         priv->nb_max_matches = cfg->nb_max_matches;
 857         /* Setup rxp db memories. */
 858         if (rxp_db_setup(priv)) {
 859                 DRV_LOG(ERR, "Failed to setup RXP db memory");
 860                 rte_errno = ENOMEM;
 861                 return -rte_errno;
 862         }
 863         if (cfg->rule_db != NULL) {
 864                 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db,
 865                                                  cfg->rule_db_len);
 866                 if (ret < 0) {
 867                         DRV_LOG(ERR, "Failed to program rxp rules.");
 868                         rte_errno = ENODEV;
 869                         goto configure_error;
 870                 }
 871         } else
 872                 DRV_LOG(DEBUG, "Regex config without rules programming!");
 873         return 0;
 874 configure_error:
 875         if (priv->qps)
 876                 rte_free(priv->qps);
 877         return -rte_errno;
 878 }