-static int
-rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value,
- uint32_t address, uint32_t expected_value,
- uint32_t expected_mask, uint32_t timeout_ms, uint8_t id)
-{
- unsigned int i;
- int ret = 0;
-
- ret = -EBUSY;
- for (i = 0; i < timeout_ms; i++) {
- if (mlx5_devx_regex_register_read(ctx, id, address, value))
- return -1;
- if ((*value & expected_mask) == expected_value) {
- ret = 0;
- break;
- }
- rte_delay_us(1000);
- }
- return ret;
-}
-
-static int
-rxp_start_engine(struct ibv_context *ctx, uint8_t id)
-{
- uint32_t ctrl;
- int ret;
-
- ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
- if (ret)
- return ret;
- ctrl |= MLX5_RXP_CSR_CTRL_GO;
- ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C;
- ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
- return ret;
-}
-
-static int
-rxp_stop_engine(struct ibv_context *ctx, uint8_t id)
-{
- uint32_t ctrl;
- int ret;
-
- ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
- if (ret)
- return ret;
- ctrl &= ~MLX5_RXP_CSR_CTRL_GO;
- ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
- return ret;
-}
-
-static int
-rxp_init_rtru(struct ibv_context *ctx, uint8_t id, uint32_t init_bits)
-{
- uint32_t ctrl_value;
- uint32_t poll_value;
- uint32_t expected_value;
- uint32_t expected_mask;
- int ret = 0;
-
- /* Read the rtru ctrl CSR. */
- ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
- &ctrl_value);
- if (ret)
- return -1;
- /* Clear any previous init modes. */
- ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK);
- if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) {
- ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
- mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
- ctrl_value);
- }
- /* Set the init_mode bits in the rtru ctrl CSR. */
- ctrl_value |= init_bits;
- mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
- ctrl_value);
- /* Need to sleep for a short period after pulsing the rtru init bit. */
- rte_delay_us(20000);
- /* Poll the rtru status CSR until all the init done bits are set. */
- DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init");
- /* Set the init bit in the rtru ctrl CSR. */
- ctrl_value |= MLX5_RXP_RTRU_CSR_CTRL_INIT;
- mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
- ctrl_value);
- /* Clear the init bit in the rtru ctrl CSR */
- ctrl_value &= ~MLX5_RXP_RTRU_CSR_CTRL_INIT;
- mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
- ctrl_value);
- /* Check that the following bits are set in the RTRU_CSR. */
- if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) {
- /* Must be incremental mode */
- expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
- MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
- } else {
- expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE |
- MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE |
- MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE;
- }
- expected_mask = expected_value;
- ret = rxp_poll_csr_for_value(ctx, &poll_value,
- MLX5_RXP_RTRU_CSR_STATUS,
- expected_value, expected_mask,
- MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
- if (ret)
- return ret;
- DRV_LOG(DEBUG, "rule memory initialise: 0x%08X", poll_value);
- /* Clear the init bit in the rtru ctrl CSR */
- ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT);
- mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL,
- ctrl_value);
- return 0;
-}
-
-static int
-rxp_parse_rof(const char *buf, uint32_t len,
- struct mlx5_rxp_ctl_rules_pgm **rules)
-{
- static const char del[] = "\n\r";
- char *line;
- char *tmp;
- char *cur_pos;
- uint32_t lines = 0;
- uint32_t entries;
- struct mlx5_rxp_rof_entry *curentry;
-
- tmp = rte_malloc("", len, 0);
- if (!tmp)
- return -ENOMEM;
- memcpy(tmp, buf, len);
- line = strtok(tmp, del);
- while (line) {
- if (line[0] != '#' && line[0] != '\0')
- lines++;
- line = strtok(NULL, del);
- }
- *rules = rte_malloc("", lines * sizeof(*curentry) + sizeof(**rules), 0);
- if (!(*rules)) {
- rte_free(tmp);
- return -ENOMEM;
- }
- memset(*rules, 0, lines * sizeof(curentry) + sizeof(**rules));
- curentry = (*rules)->rules;
- (*rules)->hdr.cmd = MLX5_RXP_CTL_RULES_PGM;
- entries = 0;
- memcpy(tmp, buf, len);
- line = strtok(tmp, del);
- while (line) {
- if (line[0] == '#' || line[0] == '\0') {
- line = strtok(NULL, del);
- continue;
- }
- curentry->type = strtoul(line, &cur_pos, 10);
- if (cur_pos == line || cur_pos[0] != ',')
- goto parse_error;
- cur_pos++;
- curentry->addr = strtoul(cur_pos, &cur_pos, 16);
- if (cur_pos[0] != ',')
- goto parse_error;
- cur_pos++;
- curentry->value = strtoull(cur_pos, &cur_pos, 16);
- if (cur_pos[0] != '\0' && cur_pos[0] != '\n')
- goto parse_error;
- curentry++;
- entries++;
- if (entries > lines)
- goto parse_error;
- line = strtok(NULL, del);
- }
- (*rules)->count = entries;
- (*rules)->hdr.len = entries * sizeof(*curentry) + sizeof(**rules);
- rte_free(tmp);
- return 0;
-parse_error:
- rte_free(tmp);
- if (*rules)
- rte_free(*rules);
- return -EINVAL;
-}
-
-static int
-mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use)
-{
- int ret;
- uint32_t umem_id;
-
- ret = mlx5_devx_regex_database_stop(priv->ctx, id);
- if (ret < 0) {
- DRV_LOG(ERR, "stop engine failed!");
- return ret;
- }
- umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem);
- ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0);
- if (ret < 0) {
- DRV_LOG(ERR, "program db failed!");
- return ret;
- }
- return 0;
-}
-
-static int
-mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id)
-{
- mlx5_devx_regex_database_resume(priv->ctx, id);
- return 0;
-}
-
-/*
- * Assign db memory for RXP programming.
- */
-static int
-mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id)
-{
- unsigned int i;
- uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED;
- uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED;
-
- /* Check which database rxp_eng is currently located if any? */
- for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
- i++) {
- if (priv->db[i].db_assigned_to_eng_num == id) {
- eng_assigned = i;
- break;
- }
- }
- /*
- * If private mode then, we can keep the same db ptr as RXP will be
- * programming EM itself if necessary, however need to see if
- * programmed yet.
- */
- if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) &&
- (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED))
- return eng_assigned;
- /* Check for inactive db memory to use. */
- for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT);
- i++) {
- if (priv->db[i].active == true)
- continue; /* Already in use, so skip db. */
- /* Set this db to active now as free to use. */
- priv->db[i].active = true;
- /* Now unassign last db index in use by RXP Eng. */
- if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) {
- priv->db[eng_assigned].active = false;
- priv->db[eng_assigned].db_assigned_to_eng_num =
- MLX5_RXP_DB_NOT_ASSIGNED;
-
- /* Set all DB memory to 0's before setting up DB. */
- memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
- }
- /* Now reassign new db index with RXP Engine. */
- priv->db[i].db_assigned_to_eng_num = id;
- db_free = i;
- break;
- }
- if (db_free == MLX5_RXP_DB_NOT_ASSIGNED)
- return -1;
- return db_free;
-}
-
-/*
- * Program RXP instruction db to RXP engine/s.
- */
-static int
-program_rxp_rules(struct mlx5_regex_priv *priv,
- struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id)
-{
- int ret, db_free;
- uint32_t rule_cnt;
-
- rule_cnt = rules->count;
- db_free = mlnx_update_database(priv, id);
- if (db_free < 0) {
- DRV_LOG(ERR, "Failed to setup db memory!");
- return db_free;
- }
- if (priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) {
- /* Register early to ensure RXP writes to EM use valid addr. */
- ret = mlnx_set_database(priv, id, db_free);
- if (ret < 0) {
- DRV_LOG(ERR, "Failed to register db memory!");
- return ret;
- }
- }
- ret = write_private_rules(priv, rules, id);
- if (ret < 0) {
- DRV_LOG(ERR, "Failed to write rules!");
- return ret;
- }
- if (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) {
- /* Write external rules directly to EM. */
- rules->count = rule_cnt;
- /* Now write external instructions to EM. */
- ret = write_shared_rules(priv, rules, rules->hdr.len, db_free);
- if (ret < 0) {
- DRV_LOG(ERR, "Failed to write EM rules!");
- return ret;
- }
- ret = mlnx_set_database(priv, id, db_free);
- if (ret < 0) {
- DRV_LOG(ERR, "Failed to register db memory!");
- return ret;
- }
- }
- ret = mlnx_resume_database(priv, id);
- if (ret < 0) {
- DRV_LOG(ERR, "Failed to resume engine!");
- return ret;
- }
- DRV_LOG(DEBUG, "Programmed RXP Engine %d\n", id);
- rules->count = rule_cnt;
- return 0;
-}
-
-static int
-rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id)
-{
- uint32_t ctrl;
- uint32_t reg;
- struct ibv_context *ctx = priv->ctx;
- int ret;
-
- ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
- if (ret)
- return ret;
- if (ctrl & MLX5_RXP_CSR_CTRL_INIT) {
- ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
- ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
- ctrl);
- if (ret)
- return ret;
- }
- ctrl |= MLX5_RXP_CSR_CTRL_INIT;
- ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
- if (ret)
- return ret;
- ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
- ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl);
- rte_delay_us(20000);
- ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS,
- MLX5_RXP_CSR_STATUS_INIT_DONE,
- MLX5_RXP_CSR_STATUS_INIT_DONE,
- MLX5_RXP_CSR_STATUS_TRIAL_TIMEOUT, id);
- if (ret)
- return ret;
- ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CTRL, &ctrl);
- if (ret)
- return ret;
- ctrl &= ~MLX5_RXP_CSR_CTRL_INIT;
- ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL,
- ctrl);
- if (ret)
- return ret;
- ret = rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2);
- if (ret)
- return ret;
- ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_CSR_CAPABILITY_5,
- ®);
- if (ret)
- return ret;
- DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16,
- reg & 0xffff);
- if ((reg >> 16) >= priv->nb_max_matches)
- ret = mlx5_devx_regex_register_write(ctx, id,
- MLX5_RXP_CSR_MAX_MATCH,
- priv->nb_max_matches);
- else
- ret = mlx5_devx_regex_register_write(ctx, id,
- MLX5_RXP_CSR_MAX_MATCH,
- (reg >> 16));
- ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX,
- (reg & 0xFFFF));
- ret |= mlx5_devx_regex_register_write(ctx, id,
- MLX5_RXP_CSR_MAX_LATENCY, 0);
- ret |= mlx5_devx_regex_register_write(ctx, id,
- MLX5_RXP_CSR_MAX_PRI_THREAD, 0);
- return ret;
-}
-
-static int
-write_private_rules(struct mlx5_regex_priv *priv,
- struct mlx5_rxp_ctl_rules_pgm *rules,
- uint8_t id)
-{
- unsigned int pending;
- uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries;
- int ret = 1;
-
- if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
- return -EINVAL;
- if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM ||
- rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR)
- return -EINVAL;
- /* For a non-incremental rules program, re-init the RXP. */
- if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) {
- ret = rxp_init_eng(priv, id);
- if (ret < 0)
- return ret;
- } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) {
- /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */
- ret = rxp_init_rtru(priv->ctx, id,
- MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
- if (ret < 0)
- return ret;
- }
- if (rules->count == 0)
- return -EINVAL;
- /* Confirm the RXP is initialised. */
- if (mlx5_devx_regex_register_read(priv->ctx, id,
- MLX5_RXP_CSR_STATUS, &val)) {
- DRV_LOG(ERR, "Failed to read from RXP!");