drivers/regex/mlx5/mlx5_rxp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_log.h>
   6 #include <rte_errno.h>
   7 #include <rte_malloc.h>
   8 #include <rte_regexdev.h>
   9 #include <rte_regexdev_core.h>
  10 #include <rte_regexdev_driver.h>
  11 #include <sys/mman.h>
  12
  13 #include <mlx5_glue.h>
  14 #include <mlx5_devx_cmds.h>
  15 #include <mlx5_prm.h>
  16 #include <mlx5_common_os.h>
  17
  18 #include "mlx5_regex.h"
  19 #include "mlx5_regex_utils.h"
  20 #include "mlx5_rxp_csrs.h"
  21 #include "mlx5_rxp.h"
  22
  23 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES
  24 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH
  25 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX
  26 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS
  27
  28 #define MLX5_REGEX_RXP_ROF2_LINE_LEN 34
  29
  30 /* Private Declarations */
  31 static int
  32 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
  33                        uint32_t address, uint32_t expected_value,
  34                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id);
  35 static int
  36 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id);
  37 static int
  38 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id);
  39 static int
  40 program_rxp_rules(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
  41                   uint8_t id);
  42 static int
  43 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id);
  44 static int
  45 rxp_db_setup(struct mlx5_regex_priv *priv);
  46 static void
  47 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id);
  48 static int
  49 rxp_start_engine(struct ibv_context *ctx, uint8_t id);
  50 static int
  51 rxp_stop_engine(struct ibv_context *ctx, uint8_t id);
  52
  53 static void __rte_unused
  54 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused)
  55 {
  56         uint32_t reg, i;
  57
  58         /* Main CSRs*/
  59         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  60                 if (mlx5_devx_regex_register_read(ctx, id,
  61                                                   (MLX5_RXP_CSR_WIDTH * i) +
  62                                                   MLX5_RXP_CSR_BASE_ADDRESS,
  63                                                   &reg)) {
  64                         DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id);
  65                         return;
  66                 }
  67                 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x",
  68                         id, i, reg);
  69         }
  70         /* RTRU CSRs*/
  71         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  72                 if (mlx5_devx_regex_register_read(ctx, id,
  73                                                   (MLX5_RXP_CSR_WIDTH * i) +
  74                                                  MLX5_RXP_RTRU_CSR_BASE_ADDRESS,
  75                                                   &reg)) {
  76                         DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id);
  77                         return;
  78                 }
  79                 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x",
  80                         id, i, reg);
  81         }
  82         /* STAT CSRs */
  83         for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
  84                 if (mlx5_devx_regex_register_read(ctx, id,
  85                                                   (MLX5_RXP_CSR_WIDTH * i) +
  86                                                 MLX5_RXP_STATS_CSR_BASE_ADDRESS,
  87                                                   &reg)) {
  88                         DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id);
  89                         return;
  90                 }
  91                 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x",
  92                         id, i, reg);
  93         }
  94 }
  95
  96 int
  97 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused,
  98                     struct rte_regexdev_info *info)
  99 {
 100         info->max_matches = MLX5_REGEX_MAX_MATCHES;
 101         info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE;
 102         info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP;
 103         info->max_groups = MLX5_REGEX_MAX_GROUPS;
 104         info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F |
 105                               RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F;
 106         info->rule_flags = 0;
 107         info->max_queue_pairs = UINT16_MAX;
 108         return 0;
 109 }
 110
 111 static int
 112 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
 113                        uint32_t address, uint32_t expected_value,
 114                        uint32_t expected_mask, uint32_t timeout_ms, uint8_t id)
 115 {
 116         unsigned int i;
 117         int ret;
 118
 119         ret = -EBUSY;
 120         for (i = 0; i < timeout_ms; i++) {
 121                 if (mlx5_devx_regex_register_read(ctx, id, address, value))
 122                         return -1;
 123                 if ((*value & expected_mask) == expected_value) {
 124                         ret = 0;
 125                         break;
 126                 }
 127                 rte_delay_us(1000);
 128         }
 129         return ret;
 130 }
 131
 132 static int
 133 rxp_start_engine(struct ibv_context *ctx, uint8_t id)
 134 {
 135         uint32_t ctrl;
 136         int ret;
 137
 138         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 139         if (ret)
 140                 return ret;
 141         ctrl |= MLX5_RXP_CSR_CTRL_GO;
 142         ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C;
 143         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 144         return ret;
 145 }
 146
 147 static int
 148 rxp_stop_engine(struct ibv_context *ctx, uint8_t id)
 149 {
 150         uint32_t ctrl;
 151         int ret;
 152
 153         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 154         if (ret)
 155                 return ret;
 156         ctrl &= ~MLX5_RXP_CSR_CTRL_GO;
 157         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 158         return ret;
 159 }
 160
 161 static int
 162 rxp_init_rtru(struct mlx5_regex_priv *priv, uint8_t id, uint32_t init_bits)
 163 {
 164         uint32_t ctrl_value;
 165         uint32_t poll_value;
 166         uint32_t expected_value;
 167         uint32_t expected_mask;
 168         struct ibv_context *ctx = priv->cdev->ctx;
 169         int ret = 0;
 170
 171         /* Read the rtru ctrl CSR. */
 172         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 173                                             &ctrl_value);
 174         if (ret)
 175                 return -1;
 176         /* Clear any previous init modes. */
 177         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK);
 178         if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) {
 179                 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 180                 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 181                                                ctrl_value);
 182         }
 183         /* Set the init_mode bits in the rtru ctrl CSR. */
 184         ctrl_value |= init_bits;
 185         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 186                                        ctrl_value);
 187         /* Need to sleep for a short period after pulsing the rtru init bit. */
 188         rte_delay_us(20000);
 189         /* Poll the rtru status CSR until all the init done bits are set. */
 190         DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init");
 191         /* Set the init bit in the rtru ctrl CSR. */
 192         ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT;
 193         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 194                                        ctrl_value);
 195         /* Clear the init bit in the rtru ctrl CSR */
 196         ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT;
 197         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 198                                        ctrl_value);
 199         /* Check that the following bits are set in the RTRU_CSR. */
 200         if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) {
 201                 /* Must be incremental mode */
 202                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE;
 203         } else {
 204                 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE |
 205                         MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE;
 206         }
 207         if (priv->is_bf2)
 208                 expected_value |= MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
 209
 210
 211         expected_mask = expected_value;
 212         ret = rxp_poll_csr_for_value(ctx, &poll_value,
 213                                      MLX5_RXP_RTRU_CSR_STATUS,
 214                                      expected_value, expected_mask,
 215                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 216         if (ret)
 217                 return ret;
 218         DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value);
 219         /* Clear the init bit in the rtru ctrl CSR */
 220         ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
 221         mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 222                                        ctrl_value);
 223         return 0;
 224 }
 225
 226 static int
 227 rxp_parse_line(char *line, uint32_t *type, uint32_t *address, uint64_t *value)
 228 {
 229         char *cur_pos;
 230
 231         if (*line == '\0' || *line == '#')
 232                 return  1;
 233         *type = strtoul(line, &cur_pos, 10);
 234         if (*cur_pos != ',' && *cur_pos != '\0')
 235                 return -1;
 236         *address = strtoul(cur_pos+1, &cur_pos, 16);
 237         if (*cur_pos != ',' && *cur_pos != '\0')
 238                 return -1;
 239         *value = strtoul(cur_pos+1, &cur_pos, 16);
 240         if (*cur_pos != ',' && *cur_pos != '\0')
 241                 return -1;
 242         return 0;
 243 }
 244
 245 static uint32_t
 246 rxp_get_reg_address(uint32_t address)
 247 {
 248         uint32_t block;
 249         uint32_t reg;
 250
 251         block = (address >> 16) & 0xFFFF;
 252         if (block == 0)
 253                 reg = MLX5_RXP_CSR_BASE_ADDRESS;
 254         else if (block == 1)
 255                 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS;
 256         else {
 257                 DRV_LOG(ERR, "Invalid ROF register 0x%08X!", address);
 258                         return UINT32_MAX;
 259         }
 260         reg += (address & 0xFFFF) * MLX5_RXP_CSR_WIDTH;
 261         return reg;
 262 }
 263
 264 #define MLX5_RXP_NUM_LINES_PER_BLOCK 8
 265
 266 static int
 267 rxp_program_rof(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
 268                 uint8_t id)
 269 {
 270         static const char del[] = "\n\r";
 271         char *line;
 272         char *tmp;
 273         uint32_t type = 0;
 274         uint32_t address;
 275         uint64_t val;
 276         uint32_t reg_val;
 277         int ret;
 278         int skip = -1;
 279         int last = 0;
 280         uint32_t temp;
 281         uint32_t tmp_addr;
 282         uint32_t rof_rule_addr;
 283         uint64_t tmp_write_swap[4];
 284         struct mlx5_rxp_rof_entry rules[8];
 285         int i;
 286         int db_free;
 287         int j;
 288
 289         tmp = rte_malloc("", len, 0);
 290         if (!tmp)
 291                 return -ENOMEM;
 292         memcpy(tmp, buf, len);
 293         db_free = mlnx_update_database(priv, id);
 294         if (db_free < 0) {
 295                 DRV_LOG(ERR, "Failed to setup db memory!");
 296                 rte_free(tmp);
 297                 return db_free;
 298         }
 299         for (line = strtok(tmp, del), j = 0; line; line = strtok(NULL, del),
 300              j++, last = type) {
 301                 ret = rxp_parse_line(line, &type, &address, &val);
 302                 if (ret != 0) {
 303                         if (ret < 0)
 304                                 goto parse_error;
 305                         continue;
 306                 }
 307                 switch (type) {
 308                 case MLX5_RXP_ROF_ENTRY_EQ:
 309                         if (skip == 0 && address == 0)
 310                                 skip = 1;
 311                         tmp_addr = rxp_get_reg_address(address);
 312                         if (tmp_addr == UINT32_MAX)
 313                                 goto parse_error;
 314                         ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id,
 315                                                             tmp_addr, &reg_val);
 316                         if (ret)
 317                                 goto parse_error;
 318                         if (skip == -1 && address == 0) {
 319                                 if (val == reg_val) {
 320                                         skip = 0;
 321                                         continue;
 322                                 }
 323                         } else if (skip == 0) {
 324                                 if (val != reg_val) {
 325                                         DRV_LOG(ERR,
 326                                                 "got %08X expected == %" PRIx64,
 327                                                 reg_val, val);
 328                                         goto parse_error;
 329                                 }
 330                         }
 331                         break;
 332                 case MLX5_RXP_ROF_ENTRY_GTE:
 333                         if (skip == 0 && address == 0)
 334                                 skip = 1;
 335                         tmp_addr = rxp_get_reg_address(address);
 336                         if (tmp_addr == UINT32_MAX)
 337                                 goto parse_error;
 338                         ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id,
 339                                                             tmp_addr, &reg_val);
 340                         if (ret)
 341                                 goto parse_error;
 342                         if (skip == -1 && address == 0) {
 343                                 if (reg_val >= val) {
 344                                         skip = 0;
 345                                         continue;
 346                                 }
 347                         } else if (skip == 0) {
 348                                 if (reg_val < val) {
 349                                         DRV_LOG(ERR,
 350                                                 "got %08X expected >= %" PRIx64,
 351                                                 reg_val, val);
 352                                         goto parse_error;
 353                                 }
 354                         }
 355                         break;
 356                 case MLX5_RXP_ROF_ENTRY_LTE:
 357                         tmp_addr = rxp_get_reg_address(address);
 358                         if (tmp_addr == UINT32_MAX)
 359                                 goto parse_error;
 360                         ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id,
 361                                                             tmp_addr, &reg_val);
 362                         if (ret)
 363                                 goto parse_error;
 364                         if (skip == 0 && address == 0 &&
 365                             last != MLX5_RXP_ROF_ENTRY_GTE) {
 366                                 skip = 1;
 367                         } else if (skip == 0 && address == 0 &&
 368                                    last == MLX5_RXP_ROF_ENTRY_GTE) {
 369                                 if (reg_val > val)
 370                                         skip = -1;
 371                                 continue;
 372                         }
 373                         if (skip == -1 && address == 0) {
 374                                 if (reg_val <= val) {
 375                                         skip = 0;
 376                                         continue;
 377                                 }
 378                         } else if (skip == 0) {
 379                                 if (reg_val > val) {
 380                                         DRV_LOG(ERR,
 381                                                 "got %08X expected <= %" PRIx64,
 382                                                 reg_val, val);
 383                                         goto parse_error;
 384                                 }
 385                         }
 386                         break;
 387                 case MLX5_RXP_ROF_ENTRY_CHECKSUM:
 388                         break;
 389                 case MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM:
 390                         if (skip)
 391                                 continue;
 392                         tmp_addr = rxp_get_reg_address(address);
 393                         if (tmp_addr == UINT32_MAX)
 394                                 goto parse_error;
 395
 396                         ret = mlx5_devx_regex_register_read(priv->cdev->ctx, id,
 397                                                             tmp_addr, &reg_val);
 398                         if (ret) {
 399                                 DRV_LOG(ERR, "RXP CSR read failed!");
 400                                 return ret;
 401                         }
 402                         if (reg_val != val) {
 403                                 DRV_LOG(ERR, "got %08X expected <= %" PRIx64,
 404                                         reg_val, val);
 405                                 goto parse_error;
 406                         }
 407                         break;
 408                 case MLX5_RXP_ROF_ENTRY_IM:
 409                         if (skip)
 410                                 continue;
 411                         /*
 412                          * NOTE: All rules written to RXP must be carried out in
 413                          * triplets of: 2xData + 1xAddr.
 414                          * No optimisation is currently allowed in this
 415                          * sequence to perform less writes.
 416                          */
 417                         temp = val;
 418                         ret |= mlx5_devx_regex_register_write
 419                                         (priv->cdev->ctx, id,
 420                                          MLX5_RXP_RTRU_CSR_DATA_0, temp);
 421                         temp = (uint32_t)(val >> 32);
 422                         ret |= mlx5_devx_regex_register_write
 423                                         (priv->cdev->ctx, id,
 424                                          MLX5_RXP_RTRU_CSR_DATA_0 +
 425                                          MLX5_RXP_CSR_WIDTH, temp);
 426                         temp = address;
 427                         ret |= mlx5_devx_regex_register_write
 428                                         (priv->cdev->ctx, id,
 429                                          MLX5_RXP_RTRU_CSR_ADDR, temp);
 430                         if (ret) {
 431                                 DRV_LOG(ERR,
 432                                         "Failed to copy instructions to RXP.");
 433                                 goto parse_error;
 434                         }
 435                         break;
 436                 case MLX5_RXP_ROF_ENTRY_EM:
 437                         if (skip)
 438                                 continue;
 439                         for (i = 0; i < MLX5_RXP_NUM_LINES_PER_BLOCK; i++) {
 440                                 ret = rxp_parse_line(line, &type,
 441                                                      &rules[i].addr,
 442                                                      &rules[i].value);
 443                                 if (ret != 0)
 444                                         goto parse_error;
 445                                 if (i < (MLX5_RXP_NUM_LINES_PER_BLOCK - 1)) {
 446                                         line = strtok(NULL, del);
 447                                         if (!line)
 448                                                 goto parse_error;
 449                                 }
 450                         }
 451                         if ((uint8_t *)((uint8_t *)
 452                                         priv->db[id].ptr +
 453                                         ((rules[7].addr <<
 454                                          MLX5_RXP_INST_OFFSET))) >=
 455                                         ((uint8_t *)((uint8_t *)
 456                                         priv->db[id].ptr + MLX5_MAX_DB_SIZE))) {
 457                                 DRV_LOG(ERR, "DB exceeded memory!");
 458                                 goto parse_error;
 459                         }
 460                         /*
 461                          * Rule address Offset to align with RXP
 462                          * external instruction offset.
 463                          */
 464                         rof_rule_addr = (rules[0].addr << MLX5_RXP_INST_OFFSET);
 465                         /* 32 byte instruction swap (sw work around)! */
 466                         tmp_write_swap[0] = le64toh(rules[4].value);
 467                         tmp_write_swap[1] = le64toh(rules[5].value);
 468                         tmp_write_swap[2] = le64toh(rules[6].value);
 469                         tmp_write_swap[3] = le64toh(rules[7].value);
 470                         /* Write only 4 of the 8 instructions. */
 471                         memcpy((uint8_t *)((uint8_t *)
 472                                 priv->db[id].ptr + rof_rule_addr),
 473                                 &tmp_write_swap, (sizeof(uint64_t) * 4));
 474                         /* Write 1st 4 rules of block after last 4. */
 475                         rof_rule_addr = (rules[4].addr << MLX5_RXP_INST_OFFSET);
 476                         tmp_write_swap[0] = le64toh(rules[0].value);
 477                         tmp_write_swap[1] = le64toh(rules[1].value);
 478                         tmp_write_swap[2] = le64toh(rules[2].value);
 479                         tmp_write_swap[3] = le64toh(rules[3].value);
 480                         memcpy((uint8_t *)((uint8_t *)
 481                                 priv->db[id].ptr + rof_rule_addr),
 482                                 &tmp_write_swap, (sizeof(uint64_t) * 4));
 483                         break;
 484                 default:
 485                         break;
 486                 }
 487
 488         }
 489         rte_free(tmp);
 490         return 0;
 491 parse_error:
 492         rte_free(tmp);
 493         return ret;
 494 }
 495
 496 static int
 497 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id)
 498 {
 499         mlx5_devx_regex_database_resume(priv->cdev->ctx, id);
 500         return 0;
 501 }
 502
 503 /*
 504  * Assign db memory for RXP programming.
 505  */
 506 static int
 507 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id)
 508 {
 509         unsigned int i;
 510         uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED;
 511         uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED;
 512
 513         /* Check which database rxp_eng is currently located if any? */
 514         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 515              i++) {
 516                 if (priv->db[i].db_assigned_to_eng_num == id) {
 517                         eng_assigned = i;
 518                         break;
 519                 }
 520         }
 521         /*
 522          * If private mode then, we can keep the same db ptr as RXP will be
 523          * programming EM itself if necessary, however need to see if
 524          * programmed yet.
 525          */
 526         if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) &&
 527             (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED))
 528                 return eng_assigned;
 529         /* Check for inactive db memory to use. */
 530         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
 531              i++) {
 532                 if (priv->db[i].active == true)
 533                         continue; /* Already in use, so skip db. */
 534                 /* Set this db to active now as free to use. */
 535                 priv->db[i].active = true;
 536                 /* Now unassign last db index in use by RXP Eng. */
 537                 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) {
 538                         priv->db[eng_assigned].active = false;
 539                         priv->db[eng_assigned].db_assigned_to_eng_num =
 540                                 MLX5_RXP_DB_NOT_ASSIGNED;
 541
 542                         /* Set all DB memory to 0's before setting up DB. */
 543                         memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 544                 }
 545                 /* Now reassign new db index with RXP Engine. */
 546                 priv->db[i].db_assigned_to_eng_num = id;
 547                 db_free = i;
 548                 break;
 549         }
 550         if (db_free == MLX5_RXP_DB_NOT_ASSIGNED)
 551                 return -1;
 552         return db_free;
 553 }
 554
 555 /*
 556  * Program RXP instruction db to RXP engine/s.
 557  */
 558 static int
 559 program_rxp_rules(struct mlx5_regex_priv *priv, const char *buf, uint32_t len,
 560                   uint8_t id)
 561 {
 562         int ret;
 563         uint32_t val;
 564         struct ibv_context *ctx = priv->cdev->ctx;
 565
 566         ret = rxp_init_eng(priv, id);
 567         if (ret < 0)
 568                 return ret;
 569         /* Confirm the RXP is initialised. */
 570         if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_STATUS, &val)) {
 571                 DRV_LOG(ERR, "Failed to read from RXP!");
 572                 return -ENODEV;
 573         }
 574         if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) {
 575                 DRV_LOG(ERR, "RXP not initialised...");
 576                 return -EBUSY;
 577         }
 578         ret = mlx5_devx_regex_register_read(ctx, id,
 579                                             MLX5_RXP_RTRU_CSR_CTRL, &val);
 580         if (ret) {
 581                 DRV_LOG(ERR, "CSR read failed!");
 582                 return -1;
 583         }
 584         val |= MLX5_RXP_RTRU_CSR_CTRL_GO;
 585         ret = mlx5_devx_regex_register_write(ctx, id,
 586                                              MLX5_RXP_RTRU_CSR_CTRL, val);
 587         if (ret) {
 588                 DRV_LOG(ERR, "Can't program rof file!");
 589                 return -1;
 590         }
 591         ret = rxp_program_rof(priv, buf, len, id);
 592         if (ret) {
 593                 DRV_LOG(ERR, "Can't program rof file!");
 594                 return -1;
 595         }
 596         if (priv->is_bf2) {
 597                 ret = rxp_poll_csr_for_value
 598                         (ctx, &val, MLX5_RXP_RTRU_CSR_STATUS,
 599                          MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 600                          MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
 601                          MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
 602                 if (ret < 0) {
 603                         DRV_LOG(ERR, "Rules update timeout: 0x%08X", val);
 604                         return ret;
 605                 }
 606                 DRV_LOG(DEBUG, "Rules update took %d cycles", ret);
 607         }
 608         if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
 609                                           &val)) {
 610                 DRV_LOG(ERR, "CSR read failed!");
 611                 return -1;
 612         }
 613         val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO);
 614         if (mlx5_devx_regex_register_write(ctx, id,
 615                                            MLX5_RXP_RTRU_CSR_CTRL, val)) {
 616                 DRV_LOG(ERR, "CSR write failed!");
 617                 return -1;
 618         }
 619         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &val);
 620         if (ret)
 621                 return ret;
 622         val &= ~MLX5_RXP_CSR_CTRL_INIT;
 623         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, val);
 624         if (ret)
 625                 return ret;
 626         rxp_init_rtru(priv, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
 627         if (priv->is_bf2) {
 628                 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_CSR_STATUS,
 629                                              MLX5_RXP_CSR_STATUS_INIT_DONE,
 630                                              MLX5_RXP_CSR_STATUS_INIT_DONE,
 631                                              MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT,
 632                                              id);
 633                 if (ret) {
 634                         DRV_LOG(ERR, "Device init failed!");
 635                         return ret;
 636                 }
 637         }
 638         ret = mlnx_resume_database(priv, id);
 639         if (ret < 0) {
 640                 DRV_LOG(ERR, "Failed to resume engine!");
 641                 return ret;
 642         }
 643
 644         return ret;
 645
 646 }
 647
 648 static int
 649 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id)
 650 {
 651         uint32_t ctrl;
 652         uint32_t reg;
 653         struct ibv_context *ctx = priv->cdev->ctx;
 654         int ret;
 655
 656         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 657         if (ret)
 658                 return ret;
 659         if (ctrl & MLX5_RXP_CSR_CTRL_INIT) {
 660                 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 661                 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 662                                                      ctrl);
 663                 if (ret)
 664                         return ret;
 665         }
 666         ctrl |= MLX5_RXP_CSR_CTRL_INIT;
 667         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 668         if (ret)
 669                 return ret;
 670         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 671         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
 672         if (ret)
 673                 return ret;
 674         rte_delay_us(20000);
 675         ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS,
 676                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 677                                      MLX5_RXP_CSR_STATUS_INIT_DONE,
 678                                      MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
 679         if (ret)
 680                 return ret;
 681         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
 682         if (ret)
 683                 return ret;
 684         ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
 685         ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
 686                                              ctrl);
 687         if (ret)
 688                 return ret;
 689         ret = rxp_init_rtru(priv, id,
 690                             MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2);
 691         if (ret)
 692                 return ret;
 693         ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5,
 694                                             &reg);
 695         if (ret)
 696                 return ret;
 697         DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16,
 698                 reg & 0xffff);
 699         if ((reg >> 16) >= priv->nb_max_matches)
 700                 ret = mlx5_devx_regex_register_write(ctx, id,
 701                                                      MLX5_RXP_CSR_MAX_MATCH,
 702                                                      priv->nb_max_matches);
 703         else
 704                 ret = mlx5_devx_regex_register_write(ctx, id,
 705                                                      MLX5_RXP_CSR_MAX_MATCH,
 706                                                      (reg >> 16));
 707         ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX,
 708                                          (reg & 0xFFFF));
 709         ret |= mlx5_devx_regex_register_write(ctx, id,
 710                                               MLX5_RXP_CSR_MAX_LATENCY, 0);
 711         ret |= mlx5_devx_regex_register_write(ctx, id,
 712                                               MLX5_RXP_CSR_MAX_PRI_THREAD, 0);
 713         return ret;
 714 }
 715
 716 static int
 717 rxp_db_setup(struct mlx5_regex_priv *priv)
 718 {
 719         int ret;
 720         uint8_t i;
 721
 722         /* Setup database memories for both RXP engines + reprogram memory. */
 723         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 724                 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 1 << 21);
 725                 if (!priv->db[i].ptr) {
 726                         DRV_LOG(ERR, "Failed to alloc db memory!");
 727                         ret = ENODEV;
 728                         goto tidyup_error;
 729                 }
 730                 /* Register the memory. */
 731                 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg
 732                                                         (priv->cdev->ctx,
 733                                                          priv->db[i].ptr,
 734                                                          MLX5_MAX_DB_SIZE, 7);
 735                 if (!priv->db[i].umem.umem) {
 736                         DRV_LOG(ERR, "Failed to register memory!");
 737                         ret = ENODEV;
 738                         goto tidyup_error;
 739                 }
 740                 /* Ensure set all DB memory to 0's before setting up DB. */
 741                 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
 742                 /* No data currently in database. */
 743                 priv->db[i].len = 0;
 744                 priv->db[i].active = false;
 745                 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED;
 746         }
 747         return 0;
 748 tidyup_error:
 749         for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
 750                 if (priv->db[i].umem.umem)
 751                         mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem);
 752                 rte_free(priv->db[i].ptr);
 753                 priv->db[i].ptr = NULL;
 754         }
 755         return -ret;
 756 }
 757
 758 int
 759 mlx5_regex_rules_db_import(struct rte_regexdev *dev,
 760                      const char *rule_db, uint32_t rule_db_len)
 761 {
 762         struct mlx5_regex_priv *priv = dev->data->dev_private;
 763         struct mlx5_rxp_ctl_rules_pgm *rules = NULL;
 764         uint32_t id;
 765         int ret;
 766         uint32_t ver;
 767
 768         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
 769                 DRV_LOG(ERR, "RXP programming mode not set!");
 770                 return -1;
 771         }
 772         if (rule_db == NULL) {
 773                 DRV_LOG(ERR, "Database empty!");
 774                 return -ENODEV;
 775         }
 776         if (rule_db_len == 0)
 777                 return -EINVAL;
 778         if (mlx5_devx_regex_register_read(priv->cdev->ctx, 0,
 779                                           MLX5_RXP_CSR_BASE_ADDRESS, &ver)) {
 780                 DRV_LOG(ERR, "Failed to read Main CSRs Engine 0!");
 781                 return -1;
 782         }
 783         /* Need to ensure RXP not busy before stop! */
 784         for (id = 0; id < priv->nb_engines; id++) {
 785                 ret = rxp_stop_engine(priv->cdev->ctx, id);
 786                 if (ret) {
 787                         DRV_LOG(ERR, "Can't stop engine.");
 788                         ret = -ENODEV;
 789                         goto tidyup_error;
 790                 }
 791                 ret = program_rxp_rules(priv, rule_db, rule_db_len, id);
 792                 if (ret < 0) {
 793                         DRV_LOG(ERR, "Failed to program rxp rules.");
 794                         ret = -ENODEV;
 795                         goto tidyup_error;
 796                 }
 797                 ret = rxp_start_engine(priv->cdev->ctx, id);
 798                 if (ret) {
 799                         DRV_LOG(ERR, "Can't start engine.");
 800                         ret = -ENODEV;
 801                         goto tidyup_error;
 802                 }
 803         }
 804         rte_free(rules);
 805         return 0;
 806 tidyup_error:
 807         rte_free(rules);
 808         return ret;
 809 }
 810
 811 int
 812 mlx5_regex_configure(struct rte_regexdev *dev,
 813                      const struct rte_regexdev_config *cfg)
 814 {
 815         struct mlx5_regex_priv *priv = dev->data->dev_private;
 816         int ret;
 817
 818         if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
 819                 return -1;
 820         priv->nb_queues = cfg->nb_queue_pairs;
 821         dev->data->dev_conf.nb_queue_pairs = priv->nb_queues;
 822         priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) *
 823                                 priv->nb_queues, 0);
 824         if (!priv->nb_queues) {
 825                 DRV_LOG(ERR, "can't allocate qps memory");
 826                 rte_errno = ENOMEM;
 827                 return -rte_errno;
 828         }
 829         priv->nb_max_matches = cfg->nb_max_matches;
 830         /* Setup rxp db memories. */
 831         if (rxp_db_setup(priv)) {
 832                 DRV_LOG(ERR, "Failed to setup RXP db memory");
 833                 rte_errno = ENOMEM;
 834                 return -rte_errno;
 835         }
 836         if (cfg->rule_db != NULL) {
 837                 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db,
 838                                                  cfg->rule_db_len);
 839                 if (ret < 0) {
 840                         DRV_LOG(ERR, "Failed to program rxp rules.");
 841                         rte_errno = ENODEV;
 842                         goto configure_error;
 843                 }
 844         } else
 845                 DRV_LOG(DEBUG, "Regex config without rules programming!");
 846         return 0;
 847 configure_error:
 848         if (priv->qps)
 849                 rte_free(priv->qps);
 850         return -rte_errno;
 851 }