1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2020 Mellanox Technologies, Ltd
7 #include <rte_malloc.h>
8 #include <rte_regexdev.h>
9 #include <rte_regexdev_core.h>
10 #include <rte_regexdev_driver.h>
12 #include <mlx5_glue.h>
13 #include <mlx5_devx_cmds.h>
15 #include <mlx5_common_os.h>
17 #include "mlx5_regex.h"
18 #include "mlx5_regex_utils.h"
19 #include "mlx5_rxp_csrs.h"
22 #define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES
23 #define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH
24 #define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX
25 #define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS
27 /* Private Declarations */
29 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
30 uint32_t address, uint32_t expected_value,
31 uint32_t expected_mask, uint32_t timeout_ms, uint8_t id);
33 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use);
35 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id);
37 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id);
39 program_rxp_rules(struct mlx5_regex_priv *priv,
40 struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id);
42 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id);
44 write_private_rules(struct mlx5_regex_priv *priv,
45 struct mlx5_rxp_ctl_rules_pgm *rules,
48 write_shared_rules(struct mlx5_regex_priv *priv,
49 struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
50 uint8_t db_to_program);
52 rxp_db_setup(struct mlx5_regex_priv *priv);
54 rxp_dump_csrs(struct ibv_context *ctx, uint8_t id);
56 rxp_write_rules_via_cp(struct ibv_context *ctx,
57 struct mlx5_rxp_rof_entry *rules,
58 int count, uint8_t id);
60 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules,
61 int count, uint8_t id);
63 rxp_start_engine(struct ibv_context *ctx, uint8_t id);
65 rxp_stop_engine(struct ibv_context *ctx, uint8_t id);
67 static void __rte_unused
68 rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused)
73 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
74 if (mlx5_devx_regex_register_read(ctx, id,
75 (MLX5_RXP_CSR_WIDTH * i) +
76 MLX5_RXP_CSR_BASE_ADDRESS,
78 DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id);
81 DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x",
85 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
86 if (mlx5_devx_regex_register_read(ctx, id,
87 (MLX5_RXP_CSR_WIDTH * i) +
88 MLX5_RXP_RTRU_CSR_BASE_ADDRESS,
90 DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id);
93 DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x",
97 for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) {
98 if (mlx5_devx_regex_register_read(ctx, id,
99 (MLX5_RXP_CSR_WIDTH * i) +
100 MLX5_RXP_STATS_CSR_BASE_ADDRESS,
102 DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id);
105 DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x",
111 mlx5_regex_info_get(struct rte_regexdev *dev __rte_unused,
112 struct rte_regexdev_info *info)
114 info->max_matches = MLX5_REGEX_MAX_MATCHES;
115 info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE;
116 info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP;
117 info->max_groups = MLX5_REGEX_MAX_GROUPS;
118 info->max_queue_pairs = 1;
119 info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F |
120 RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F;
121 info->rule_flags = 0;
122 info->max_queue_pairs = 10;
127 * Actual writing of RXP instructions to RXP via CSRs.
130 rxp_write_rules_via_cp(struct ibv_context *ctx,
131 struct mlx5_rxp_rof_entry *rules,
132 int count, uint8_t id)
137 for (i = 0; i < count; i++) {
138 tmp = (uint32_t)rules[i].value;
139 ret |= mlx5_devx_regex_register_write(ctx, id,
140 MLX5_RXP_RTRU_CSR_DATA_0,
142 tmp = (uint32_t)(rules[i].value >> 32);
143 ret |= mlx5_devx_regex_register_write(ctx, id,
144 MLX5_RXP_RTRU_CSR_DATA_0 +
145 MLX5_RXP_CSR_WIDTH, tmp);
147 ret |= mlx5_devx_regex_register_write(ctx, id,
148 MLX5_RXP_RTRU_CSR_ADDR,
151 DRV_LOG(ERR, "Failed to copy instructions to RXP.");
155 DRV_LOG(DEBUG, "Written %d instructions", count);
160 rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules,
161 int count, uint8_t id)
163 uint32_t val, fifo_depth;
166 ret = rxp_write_rules_via_cp(ctx, rules, count, id);
168 DRV_LOG(ERR, "Failed to write rules via CSRs.");
171 ret = mlx5_devx_regex_register_read(ctx, id,
172 MLX5_RXP_RTRU_CSR_CAPABILITY,
175 DRV_LOG(ERR, "CSR read failed!");
178 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_FIFO_STAT,
180 MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
182 DRV_LOG(ERR, "Rules not rx by RXP: credit: %d, depth: %d", val,
186 DRV_LOG(DEBUG, "RTRU FIFO depth: 0x%x", fifo_depth);
187 DRV_LOG(DEBUG, "Rules flush took %d cycles.", ret);
188 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
191 DRV_LOG(ERR, "CSR read failed!");
194 val |= MLX5_RXP_RTRU_CSR_CTRL_GO;
195 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
198 DRV_LOG(ERR, "CSR write failed!");
201 ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_STATUS,
202 MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
203 MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE,
204 MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id);
206 DRV_LOG(ERR, "Rules update timeout: 0x%08X", val);
209 DRV_LOG(DEBUG, "Rules update took %d cycles", ret);
210 if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
212 DRV_LOG(ERR, "CSR read failed!");
215 val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO);
216 if (mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
218 DRV_LOG(ERR, "CSR write write failed!");
222 DRV_LOG(DEBUG, "RXP Flush rules finished.");
227 rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
228 uint32_t address, uint32_t expected_value,
229 uint32_t expected_mask, uint32_t timeout_ms, uint8_t id)
235 for (i = 0; i < timeout_ms; i++) {
236 if (mlx5_devx_regex_register_read(ctx, id, address, value))
238 if ((*value & expected_mask) == expected_value) {
248 rxp_start_engine(struct ibv_context *ctx, uint8_t id)
253 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
256 ctrl |= MLX5_RXP_CSR_CTRL_GO;
257 ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C;
258 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
263 rxp_stop_engine(struct ibv_context *ctx, uint8_t id)
268 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
271 ctrl &= ~MLX5_RXP_CSR_CTRL_GO;
272 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
277 rxp_init_rtru(struct ibv_context *ctx, uint8_t id, uint32_t init_bits)
281 uint32_t expected_value;
282 uint32_t expected_mask;
285 /* Read the rtru ctrl CSR. */
286 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
290 /* Clear any previous init modes. */
291 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK);
292 if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) {
293 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
294 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
297 /* Set the init_mode bits in the rtru ctrl CSR. */
298 ctrl_value |= init_bits;
299 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
301 /* Need to sleep for a short period after pulsing the rtru init bit. */
303 /* Poll the rtru status CSR until all the init done bits are set. */
304 DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init");
305 /* Set the init bit in the rtru ctrl CSR. */
306 ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT;
307 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
309 /* Clear the init bit in the rtru ctrl CSR */
310 ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT;
311 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
313 /* Check that the following bits are set in the RTRU_CSR. */
314 if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) {
315 /* Must be incremental mode */
316 expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
317 MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
319 expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE |
320 MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
321 MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
323 expected_mask = expected_value;
324 ret = rxp_poll_csr_for_value(ctx, &poll_value,
325 MLX5_RXP_RTRU_CSR_STATUS,
326 expected_value, expected_mask,
327 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
330 DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value);
331 /* Clear the init bit in the rtru ctrl CSR */
332 ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
333 mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
339 rxp_parse_rof(const char *buf, uint32_t len,
340 struct mlx5_rxp_ctl_rules_pgm **rules)
342 static const char del[] = "\n\r";
348 struct mlx5_rxp_rof_entry *curentry;
350 tmp = rte_malloc("", len, 0);
353 memcpy(tmp, buf, len);
354 line = strtok(tmp, del);
356 if (line[0] != '#' && line[0] != '\0')
358 line = strtok(NULL, del);
360 *rules = rte_malloc("", lines * sizeof(*curentry) + sizeof(**rules), 0);
365 memset(*rules, 0, lines * sizeof(curentry) + sizeof(**rules));
366 curentry = (*rules)->rules;
367 (*rules)->hdr.cmd = MLX5_RXP_CTL_RULES_PGM;
369 memcpy(tmp, buf, len);
370 line = strtok(tmp, del);
372 if (line[0] == '#' || line[0] == '\0') {
373 line = strtok(NULL, del);
376 curentry->type = strtoul(line, &cur_pos, 10);
377 if (cur_pos == line || cur_pos[0] != ',')
380 curentry->addr = strtoul(cur_pos, &cur_pos, 16);
381 if (cur_pos[0] != ',')
384 curentry->value = strtoull(cur_pos, &cur_pos, 16);
385 if (cur_pos[0] != '\0' && cur_pos[0] != '\n')
391 line = strtok(NULL, del);
393 (*rules)->count = entries;
394 (*rules)->hdr.len = entries * sizeof(*curentry) + sizeof(**rules);
405 mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use)
410 ret = mlx5_devx_regex_database_stop(priv->ctx, id);
412 DRV_LOG(ERR, "stop engine failed!");
415 umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem);
416 ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0);
418 DRV_LOG(ERR, "program db failed!");
425 mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id)
427 mlx5_devx_regex_database_resume(priv->ctx, id);
432 * Assign db memory for RXP programming.
435 mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id)
438 uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED;
439 uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED;
441 /* Check which database rxp_eng is currently located if any? */
442 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
444 if (priv->db[i].db_assigned_to_eng_num == id) {
450 * If private mode then, we can keep the same db ptr as RXP will be
451 * programming EM itself if necessary, however need to see if
454 if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) &&
455 (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED))
457 /* Check for inactive db memory to use. */
458 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
460 if (priv->db[i].active == true)
461 continue; /* Already in use, so skip db. */
462 /* Set this db to active now as free to use. */
463 priv->db[i].active = true;
464 /* Now unassign last db index in use by RXP Eng. */
465 if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) {
466 priv->db[eng_assigned].active = false;
467 priv->db[eng_assigned].db_assigned_to_eng_num =
468 MLX5_RXP_DB_NOT_ASSIGNED;
470 /* Set all DB memory to 0's before setting up DB. */
471 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
473 /* Now reassign new db index with RXP Engine. */
474 priv->db[i].db_assigned_to_eng_num = id;
478 if (db_free == MLX5_RXP_DB_NOT_ASSIGNED)
484 * Program RXP instruction db to RXP engine/s.
487 program_rxp_rules(struct mlx5_regex_priv *priv,
488 struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id)
493 rule_cnt = rules->count;
494 db_free = mlnx_update_database(priv, id);
496 DRV_LOG(ERR, "Failed to setup db memory!");
499 if (priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) {
500 /* Register early to ensure RXP writes to EM use valid addr. */
501 ret = mlnx_set_database(priv, id, db_free);
503 DRV_LOG(ERR, "Failed to register db memory!");
507 ret = write_private_rules(priv, rules, id);
509 DRV_LOG(ERR, "Failed to write rules!");
512 if (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) {
513 /* Write external rules directly to EM. */
514 rules->count = rule_cnt;
515 /* Now write external instructions to EM. */
516 ret = write_shared_rules(priv, rules, rules->hdr.len, db_free);
518 DRV_LOG(ERR, "Failed to write EM rules!");
521 ret = mlnx_set_database(priv, id, db_free);
523 DRV_LOG(ERR, "Failed to register db memory!");
527 ret = mlnx_resume_database(priv, id);
529 DRV_LOG(ERR, "Failed to resume engine!");
532 DRV_LOG(DEBUG, "Programmed RXP Engine %d\n", id);
533 rules->count = rule_cnt;
538 rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id)
542 struct ibv_context *ctx = priv->ctx;
545 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
548 if (ctrl & MLX5_RXP_CSR_CTRL_INIT) {
549 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
550 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
555 ctrl |= MLX5_RXP_CSR_CTRL_INIT;
556 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
559 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
560 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
564 ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS,
565 MLX5_RXP_CSR_STATUS_INIT_DONE,
566 MLX5_RXP_CSR_STATUS_INIT_DONE,
567 MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
570 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
573 ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
574 ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
578 ret = rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2);
581 ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5,
585 DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16,
587 if ((reg >> 16) >= priv->nb_max_matches)
588 ret = mlx5_devx_regex_register_write(ctx, id,
589 MLX5_RXP_CSR_MAX_MATCH,
590 priv->nb_max_matches);
592 ret = mlx5_devx_regex_register_write(ctx, id,
593 MLX5_RXP_CSR_MAX_MATCH,
595 ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX,
597 ret |= mlx5_devx_regex_register_write(ctx, id,
598 MLX5_RXP_CSR_MAX_LATENCY, 0);
599 ret |= mlx5_devx_regex_register_write(ctx, id,
600 MLX5_RXP_CSR_MAX_PRI_THREAD, 0);
605 write_private_rules(struct mlx5_regex_priv *priv,
606 struct mlx5_rxp_ctl_rules_pgm *rules,
609 unsigned int pending;
610 uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries;
613 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
615 if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM ||
616 rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR)
618 /* For a non-incremental rules program, re-init the RXP. */
619 if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) {
620 ret = rxp_init_eng(priv, id);
623 } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) {
624 /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */
625 ret = rxp_init_rtru(priv->ctx, id,
626 MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
630 if (rules->count == 0)
632 /* Confirm the RXP is initialised. */
633 if (mlx5_devx_regex_register_read(priv->ctx, id,
634 MLX5_RXP_CSR_STATUS, &val)) {
635 DRV_LOG(ERR, "Failed to read from RXP!");
638 if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) {
639 DRV_LOG(ERR, "RXP not initialised...");
642 /* Get the RTRU maximum number of entries allowed. */
643 if (mlx5_devx_regex_register_read(priv->ctx, id,
644 MLX5_RXP_RTRU_CSR_CAPABILITY, &rtru_max_num_entries)) {
645 DRV_LOG(ERR, "Failed to read RTRU capability!");
648 rtru_max_num_entries = (rtru_max_num_entries & 0x00FF);
651 while (rules->count > 0) {
652 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_INST) ||
653 (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_IM) ||
654 (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM)) {
655 if ((rules->rules[rule_cnt].type ==
656 MLX5_RXP_ROF_ENTRY_EM) &&
657 (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
658 /* Skip EM rules programming. */
660 /* Flush any rules that are pending. */
661 rule_offset = (rule_cnt - pending);
662 ret = rxp_flush_rules(priv->ctx,
663 &rules->rules[rule_offset],
666 DRV_LOG(ERR, "Flushing rules.");
676 * If parsing the last rule, or if reached the
677 * maximum number of rules for this batch, then
678 * flush the rules batch to the RXP.
680 if ((rules->count == 1) ||
681 (pending == rtru_max_num_entries)) {
682 rule_offset = (rule_cnt - pending);
683 ret = rxp_flush_rules(priv->ctx,
684 &rules->rules[rule_offset],
687 DRV_LOG(ERR, "Flushing rules.");
693 } else if ((rules->rules[rule_cnt].type ==
694 MLX5_RXP_ROF_ENTRY_EQ) ||
695 (rules->rules[rule_cnt].type ==
696 MLX5_RXP_ROF_ENTRY_GTE) ||
697 (rules->rules[rule_cnt].type ==
698 MLX5_RXP_ROF_ENTRY_LTE) ||
699 (rules->rules[rule_cnt].type ==
700 MLX5_RXP_ROF_ENTRY_CHECKSUM) ||
701 (rules->rules[rule_cnt].type ==
702 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM)) {
704 /* Flush rules before checking reg values. */
705 rule_offset = (rule_cnt - pending);
706 ret = rxp_flush_rules(priv->ctx,
707 &rules->rules[rule_offset],
710 DRV_LOG(ERR, "Failed to flush rules.");
714 block = (rules->rules[rule_cnt].addr >> 16) & 0xFFFF;
716 reg = MLX5_RXP_CSR_BASE_ADDRESS;
718 reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS;
720 DRV_LOG(ERR, "Invalid ROF register 0x%08X!",
721 rules->rules[rule_cnt].addr);
724 reg += (rules->rules[rule_cnt].addr & 0xFFFF) *
726 ret = mlx5_devx_regex_register_read(priv->ctx, id,
729 DRV_LOG(ERR, "RXP CSR read failed!");
732 if ((priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) &&
733 ((rules->rules[rule_cnt].type ==
734 MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM) &&
735 (val != rules->rules[rule_cnt].value))) {
736 DRV_LOG(ERR, "Unexpected value for register:");
737 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
738 rules->rules[rule_cnt].addr, val);
739 DRV_LOG(ERR, "expected %" PRIx64 ".",
740 rules->rules[rule_cnt].value);
742 } else if ((priv->prog_mode ==
743 MLX5_RXP_PRIVATE_PROG_MODE) &&
744 (rules->rules[rule_cnt].type ==
745 MLX5_RXP_ROF_ENTRY_CHECKSUM) &&
746 (val != rules->rules[rule_cnt].value)) {
747 DRV_LOG(ERR, "Unexpected value for register:");
748 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
749 rules->rules[rule_cnt].addr, val);
750 DRV_LOG(ERR, "expected %" PRIx64 ".",
751 rules->rules[rule_cnt].value);
753 } else if ((rules->rules[rule_cnt].type ==
754 MLX5_RXP_ROF_ENTRY_EQ) &&
755 (val != rules->rules[rule_cnt].value)) {
756 DRV_LOG(ERR, "Unexpected value for register:");
757 DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
758 rules->rules[rule_cnt].addr, val);
759 DRV_LOG(ERR, "expected %" PRIx64 ".",
760 rules->rules[rule_cnt].value);
762 } else if ((rules->rules[rule_cnt].type ==
763 MLX5_RXP_ROF_ENTRY_GTE) &&
764 (val < rules->rules[rule_cnt].value)) {
765 DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
766 rules->rules[rule_cnt].addr);
767 DRV_LOG(ERR, "got %X, expected >= %" PRIx64 ".",
768 val, rules->rules[rule_cnt].value);
770 } else if ((rules->rules[rule_cnt].type ==
771 MLX5_RXP_ROF_ENTRY_LTE) &&
772 (val > rules->rules[rule_cnt].value)) {
773 DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
774 rules->rules[rule_cnt].addr);
775 DRV_LOG(ERR, "got %08X expected <= %" PRIx64,
776 val, rules->rules[rule_cnt].value);
782 DRV_LOG(ERR, "Error: Invalid rule type %d!",
783 rules->rules[rule_cnt].type);
792 * Shared memory programming mode, here all external db instructions are written
793 * to EM via the host.
796 write_shared_rules(struct mlx5_regex_priv *priv,
797 struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
798 uint8_t db_to_program)
800 uint32_t rule_cnt, rof_rule_addr;
801 uint64_t tmp_write_swap[4];
803 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
805 if ((rules->count == 0) || (count == 0))
809 * Note the following section of code carries out a 32byte swap of
810 * instruction to coincide with HW 32byte swap. This may need removed
811 * in new variants of this programming function!
813 while (rule_cnt < rules->count) {
814 if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM) &&
815 (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
817 * Note there are always blocks of 8 instructions for
818 * 7's written sequentially. However there is no
819 * guarantee that all blocks are sequential!
821 if (count >= (rule_cnt + MLX5_RXP_INST_BLOCK_SIZE)) {
823 * Ensure memory write not exceeding boundary
824 * Check essential to ensure 0x10000 offset
827 if ((uint8_t *)((uint8_t *)
828 priv->db[db_to_program].ptr +
829 ((rules->rules[rule_cnt + 7].addr <<
830 MLX5_RXP_INST_OFFSET))) >=
831 ((uint8_t *)((uint8_t *)
832 priv->db[db_to_program].ptr +
833 MLX5_MAX_DB_SIZE))) {
834 DRV_LOG(ERR, "DB exceeded memory!");
838 * Rule address Offset to align with RXP
839 * external instruction offset.
841 rof_rule_addr = (rules->rules[rule_cnt].addr <<
842 MLX5_RXP_INST_OFFSET);
843 /* 32 byte instruction swap (sw work around)! */
844 tmp_write_swap[0] = le64toh(
845 rules->rules[(rule_cnt + 4)].value);
846 tmp_write_swap[1] = le64toh(
847 rules->rules[(rule_cnt + 5)].value);
848 tmp_write_swap[2] = le64toh(
849 rules->rules[(rule_cnt + 6)].value);
850 tmp_write_swap[3] = le64toh(
851 rules->rules[(rule_cnt + 7)].value);
852 /* Write only 4 of the 8 instructions. */
853 memcpy((uint8_t *)((uint8_t *)
854 priv->db[db_to_program].ptr +
855 rof_rule_addr), &tmp_write_swap,
856 (sizeof(uint64_t) * 4));
857 /* Write 1st 4 rules of block after last 4. */
858 rof_rule_addr = (rules->rules[
859 (rule_cnt + 4)].addr <<
860 MLX5_RXP_INST_OFFSET);
861 tmp_write_swap[0] = le64toh(
862 rules->rules[(rule_cnt + 0)].value);
863 tmp_write_swap[1] = le64toh(
864 rules->rules[(rule_cnt + 1)].value);
865 tmp_write_swap[2] = le64toh(
866 rules->rules[(rule_cnt + 2)].value);
867 tmp_write_swap[3] = le64toh(
868 rules->rules[(rule_cnt + 3)].value);
869 memcpy((uint8_t *)((uint8_t *)
870 priv->db[db_to_program].ptr +
871 rof_rule_addr), &tmp_write_swap,
872 (sizeof(uint64_t) * 4));
875 /* Fast forward as already handled block of 8. */
876 rule_cnt += MLX5_RXP_INST_BLOCK_SIZE;
878 rule_cnt++; /* Must be something other than EM rule. */
884 rxp_db_setup(struct mlx5_regex_priv *priv)
889 /* Setup database memories for both RXP engines + reprogram memory. */
890 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
891 priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 0);
892 if (!priv->db[i].ptr) {
893 DRV_LOG(ERR, "Failed to alloc db memory!");
897 /* Register the memory. */
898 priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx,
900 MLX5_MAX_DB_SIZE, 7);
901 if (!priv->db[i].umem.umem) {
902 DRV_LOG(ERR, "Failed to register memory!");
906 /* Ensure set all DB memory to 0's before setting up DB. */
907 memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
908 /* No data currently in database. */
910 priv->db[i].active = false;
911 priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED;
915 for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
917 rte_free(priv->db[i].ptr);
918 if (priv->db[i].umem.umem)
919 mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem);
925 mlx5_regex_rules_db_import(struct rte_regexdev *dev,
926 const char *rule_db, uint32_t rule_db_len)
928 struct mlx5_regex_priv *priv = dev->data->dev_private;
929 struct mlx5_rxp_ctl_rules_pgm *rules = NULL;
933 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
934 DRV_LOG(ERR, "RXP programming mode not set!");
937 if (rule_db == NULL) {
938 DRV_LOG(ERR, "Database empty!");
941 if (rule_db_len == 0)
943 ret = rxp_parse_rof(rule_db, rule_db_len, &rules);
945 DRV_LOG(ERR, "Can't parse ROF file.");
948 /* Need to ensure RXP not busy before stop! */
949 for (id = 0; id < priv->nb_engines; id++) {
950 ret = rxp_stop_engine(priv->ctx, id);
952 DRV_LOG(ERR, "Can't stop engine.");
956 ret = program_rxp_rules(priv, rules, id);
958 DRV_LOG(ERR, "Failed to program rxp rules.");
962 ret = rxp_start_engine(priv->ctx, id);
964 DRV_LOG(ERR, "Can't start engine.");
977 mlx5_regex_configure(struct rte_regexdev *dev,
978 const struct rte_regexdev_config *cfg)
980 struct mlx5_regex_priv *priv = dev->data->dev_private;
983 if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
985 priv->nb_queues = cfg->nb_queue_pairs;
986 dev->data->dev_conf.nb_queue_pairs = priv->nb_queues;
987 priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) *
989 if (!priv->nb_queues) {
990 DRV_LOG(ERR, "can't allocate qps memory");
994 priv->nb_max_matches = cfg->nb_max_matches;
995 /* Setup rxp db memories. */
996 if (rxp_db_setup(priv)) {
997 DRV_LOG(ERR, "Failed to setup RXP db memory");
1001 if (cfg->rule_db != NULL) {
1002 ret = mlx5_regex_rules_db_import(dev, cfg->rule_db,
1005 DRV_LOG(ERR, "Failed to program rxp rules.");
1007 goto configure_error;
1010 DRV_LOG(DEBUG, "Regex config without rules programming!");
1014 rte_free(priv->qps);