+static int
+write_private_rules(struct mlx5_regex_priv *priv,
+ struct mlx5_rxp_ctl_rules_pgm *rules,
+ uint8_t id)
+{
+ unsigned int pending;
+ uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries;
+ int ret = 1;
+
+ if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
+ return -EINVAL;
+ if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM ||
+ rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR)
+ return -EINVAL;
+ /* For a non-incremental rules program, re-init the RXP. */
+ if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) {
+ ret = rxp_init_eng(priv, id);
+ if (ret < 0)
+ return ret;
+ } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) {
+ /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */
+ ret = rxp_init_rtru(priv->ctx, id,
+ MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2);
+ if (ret < 0)
+ return ret;
+ }
+ if (rules->count == 0)
+ return -EINVAL;
+ /* Confirm the RXP is initialised. */
+ if (mlx5_devx_regex_register_read(priv->ctx, id,
+ MLX5_RXP_CSR_STATUS, &val)) {
+ DRV_LOG(ERR, "Failed to read from RXP!");
+ return -ENODEV;
+ }
+ if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) {
+ DRV_LOG(ERR, "RXP not initialised...");
+ return -EBUSY;
+ }
+ /* Get the RTRU maximum number of entries allowed. */
+ if (mlx5_devx_regex_register_read(priv->ctx, id,
+ MLX5_RXP_RTRU_CSR_CAPABILITY, &rtru_max_num_entries)) {
+ DRV_LOG(ERR, "Failed to read RTRU capability!");
+ return -ENODEV;
+ }
+ rtru_max_num_entries = (rtru_max_num_entries & 0x00FF);
+ rule_cnt = 0;
+ pending = 0;
+ while (rules->count > 0) {
+ if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_INST) ||
+ (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_IM) ||
+ (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM)) {
+ if ((rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_EM) &&
+ (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
+ /* Skip EM rules programming. */
+ if (pending > 0) {
+ /* Flush any rules that are pending. */
+ rule_offset = (rule_cnt - pending);
+ ret = rxp_flush_rules(priv->ctx,
+ &rules->rules[rule_offset],
+ pending, id);
+ if (ret < 0) {
+ DRV_LOG(ERR, "Flushing rules.");
+ return -ENODEV;
+ }
+ pending = 0;
+ }
+ rule_cnt++;
+ } else {
+ pending++;
+ rule_cnt++;
+ /*
+ * If parsing the last rule, or if reached the
+ * maximum number of rules for this batch, then
+ * flush the rules batch to the RXP.
+ */
+ if ((rules->count == 1) ||
+ (pending == rtru_max_num_entries)) {
+ rule_offset = (rule_cnt - pending);
+ ret = rxp_flush_rules(priv->ctx,
+ &rules->rules[rule_offset],
+ pending, id);
+ if (ret < 0) {
+ DRV_LOG(ERR, "Flushing rules.");
+ return -ENODEV;
+ }
+ pending = 0;
+ }
+ }
+ } else if ((rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_EQ) ||
+ (rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_GTE) ||
+ (rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_LTE) ||
+ (rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_CHECKSUM) ||
+ (rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM)) {
+ if (pending) {
+ /* Flush rules before checking reg values. */
+ rule_offset = (rule_cnt - pending);
+ ret = rxp_flush_rules(priv->ctx,
+ &rules->rules[rule_offset],
+ pending, id);
+ if (ret < 0) {
+ DRV_LOG(ERR, "Failed to flush rules.");
+ return -ENODEV;
+ }
+ }
+ block = (rules->rules[rule_cnt].addr >> 16) & 0xFFFF;
+ if (block == 0)
+ reg = MLX5_RXP_CSR_BASE_ADDRESS;
+ else if (block == 1)
+ reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS;
+ else {
+ DRV_LOG(ERR, "Invalid ROF register 0x%08X!",
+ rules->rules[rule_cnt].addr);
+ return -EINVAL;
+ }
+ reg += (rules->rules[rule_cnt].addr & 0xFFFF) *
+ MLX5_RXP_CSR_WIDTH;
+ ret = mlx5_devx_regex_register_read(priv->ctx, id,
+ reg, &val);
+ if (ret) {
+ DRV_LOG(ERR, "RXP CSR read failed!");
+ return ret;
+ }
+ if ((priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) &&
+ ((rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM) &&
+ (val != rules->rules[rule_cnt].value))) {
+ DRV_LOG(ERR, "Unexpected value for register:");
+ DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
+ rules->rules[rule_cnt].addr, val);
+ DRV_LOG(ERR, "expected %" PRIx64 ".",
+ rules->rules[rule_cnt].value);
+ return -EINVAL;
+ } else if ((priv->prog_mode ==
+ MLX5_RXP_PRIVATE_PROG_MODE) &&
+ (rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_CHECKSUM) &&
+ (val != rules->rules[rule_cnt].value)) {
+ DRV_LOG(ERR, "Unexpected value for register:");
+ DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
+ rules->rules[rule_cnt].addr, val);
+ DRV_LOG(ERR, "expected %" PRIx64 ".",
+ rules->rules[rule_cnt].value);
+ return -EINVAL;
+ } else if ((rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_EQ) &&
+ (val != rules->rules[rule_cnt].value)) {
+ DRV_LOG(ERR, "Unexpected value for register:");
+ DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32,
+ rules->rules[rule_cnt].addr, val);
+ DRV_LOG(ERR, "expected %" PRIx64 ".",
+ rules->rules[rule_cnt].value);
+ return -EINVAL;
+ } else if ((rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_GTE) &&
+ (val < rules->rules[rule_cnt].value)) {
+ DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
+ rules->rules[rule_cnt].addr);
+ DRV_LOG(ERR, "got %X, expected >= %" PRIx64 ".",
+ val, rules->rules[rule_cnt].value);
+ return -EINVAL;
+ } else if ((rules->rules[rule_cnt].type ==
+ MLX5_RXP_ROF_ENTRY_LTE) &&
+ (val > rules->rules[rule_cnt].value)) {
+ DRV_LOG(ERR, "Unexpected value reg 0x%08X,",
+ rules->rules[rule_cnt].addr);
+ DRV_LOG(ERR, "got %08X expected <= %" PRIx64,
+ val, rules->rules[rule_cnt].value);
+ return -EINVAL;
+ }
+ rule_cnt++;
+ pending = 0;
+ } else {
+ DRV_LOG(ERR, "Error: Invalid rule type %d!",
+ rules->rules[rule_cnt].type);
+ return -EINVAL;
+ }
+ rules->count--;
+ }
+ return ret;
+}
+
+/*
+ * Shared memory programming mode, here all external db instructions are written
+ * to EM via the host.
+ */
+static int
+write_shared_rules(struct mlx5_regex_priv *priv,
+ struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count,
+ uint8_t db_to_program)
+{
+ uint32_t rule_cnt, rof_rule_addr;
+ uint64_t tmp_write_swap[4];
+
+ if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED)
+ return -EINVAL;
+ if ((rules->count == 0) || (count == 0))
+ return -EINVAL;
+ rule_cnt = 0;
+ /*
+ * Note the following section of code carries out a 32byte swap of
+ * instruction to coincide with HW 32byte swap. This may need removed
+ * in new variants of this programming function!
+ */
+ while (rule_cnt < rules->count) {
+ if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM) &&
+ (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) {
+ /*
+ * Note there are always blocks of 8 instructions for
+ * 7's written sequentially. However there is no
+ * guarantee that all blocks are sequential!
+ */
+ if (count >= (rule_cnt + MLX5_RXP_INST_BLOCK_SIZE)) {
+ /*
+ * Ensure memory write not exceeding boundary
+ * Check essential to ensure 0x10000 offset
+ * accounted for!
+ */
+ if ((uint8_t *)((uint8_t *)
+ priv->db[db_to_program].ptr +
+ ((rules->rules[rule_cnt + 7].addr <<
+ MLX5_RXP_INST_OFFSET))) >=
+ ((uint8_t *)((uint8_t *)
+ priv->db[db_to_program].ptr +
+ MLX5_MAX_DB_SIZE))) {
+ DRV_LOG(ERR, "DB exceeded memory!");
+ return -ENODEV;
+ }
+ /*
+ * Rule address Offset to align with RXP
+ * external instruction offset.
+ */
+ rof_rule_addr = (rules->rules[rule_cnt].addr <<
+ MLX5_RXP_INST_OFFSET);
+ /* 32 byte instruction swap (sw work around)! */
+ tmp_write_swap[0] = le64toh(
+ rules->rules[(rule_cnt + 4)].value);
+ tmp_write_swap[1] = le64toh(
+ rules->rules[(rule_cnt + 5)].value);
+ tmp_write_swap[2] = le64toh(
+ rules->rules[(rule_cnt + 6)].value);
+ tmp_write_swap[3] = le64toh(
+ rules->rules[(rule_cnt + 7)].value);
+ /* Write only 4 of the 8 instructions. */
+ memcpy((uint8_t *)((uint8_t *)
+ priv->db[db_to_program].ptr +
+ rof_rule_addr), &tmp_write_swap,
+ (sizeof(uint64_t) * 4));
+ /* Write 1st 4 rules of block after last 4. */
+ rof_rule_addr = (rules->rules[
+ (rule_cnt + 4)].addr <<
+ MLX5_RXP_INST_OFFSET);
+ tmp_write_swap[0] = le64toh(
+ rules->rules[(rule_cnt + 0)].value);
+ tmp_write_swap[1] = le64toh(
+ rules->rules[(rule_cnt + 1)].value);
+ tmp_write_swap[2] = le64toh(
+ rules->rules[(rule_cnt + 2)].value);
+ tmp_write_swap[3] = le64toh(
+ rules->rules[(rule_cnt + 3)].value);
+ memcpy((uint8_t *)((uint8_t *)
+ priv->db[db_to_program].ptr +
+ rof_rule_addr), &tmp_write_swap,
+ (sizeof(uint64_t) * 4));
+ } else
+ return -1;
+ /* Fast forward as already handled block of 8. */
+ rule_cnt += MLX5_RXP_INST_BLOCK_SIZE;
+ } else
+ rule_cnt++; /* Must be something other than EM rule. */
+ }
+ return 0;
+}
+
+static int
+rxp_db_setup(struct mlx5_regex_priv *priv)
+{
+ int ret;
+ uint8_t i;
+
+ /* Setup database memories for both RXP engines + reprogram memory. */
+ for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
+ priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 0);
+ if (!priv->db[i].ptr) {
+ DRV_LOG(ERR, "Failed to alloc db memory!");
+ ret = ENODEV;
+ goto tidyup_error;
+ }
+ /* Register the memory. */
+ priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx,
+ priv->db[i].ptr,
+ MLX5_MAX_DB_SIZE, 7);
+ if (!priv->db[i].umem.umem) {
+ DRV_LOG(ERR, "Failed to register memory!");
+ ret = ENODEV;
+ goto tidyup_error;
+ }
+ /* Ensure set all DB memory to 0's before setting up DB. */
+ memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE);
+ /* No data currently in database. */
+ priv->db[i].len = 0;
+ priv->db[i].active = false;
+ priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED;
+ }
+ return 0;
+tidyup_error:
+ for (i = 0; i < (priv->nb_engines + MLX5_RXP_EM_COUNT); i++) {
+ if (priv->db[i].ptr)
+ rte_free(priv->db[i].ptr);
+ if (priv->db[i].umem.umem)
+ mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem);
+ }
+ return -ret;
+}
+
+int
+mlx5_regex_rules_db_import(struct rte_regexdev *dev,
+ const char *rule_db, uint32_t rule_db_len)
+{
+ struct mlx5_regex_priv *priv = dev->data->dev_private;
+ struct mlx5_rxp_ctl_rules_pgm *rules = NULL;
+ uint8_t id;
+ int ret;
+
+ if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) {
+ DRV_LOG(ERR, "RXP programming mode not set!");
+ return -1;
+ }
+ if (rule_db == NULL) {
+ DRV_LOG(ERR, "Database empty!");
+ return -ENODEV;
+ }
+ if (rule_db_len == 0)
+ return -EINVAL;
+ ret = rxp_parse_rof(rule_db, rule_db_len, &rules);
+ if (ret) {
+ DRV_LOG(ERR, "Can't parse ROF file.");
+ return ret;
+ }
+ /* Need to ensure RXP not busy before stop! */
+ for (id = 0; id < priv->nb_engines; id++) {
+ ret = rxp_stop_engine(priv->ctx, id);
+ if (ret) {
+ DRV_LOG(ERR, "Can't stop engine.");
+ ret = -ENODEV;
+ goto tidyup_error;
+ }
+ ret = program_rxp_rules(priv, rules, id);
+ if (ret < 0) {
+ DRV_LOG(ERR, "Failed to program rxp rules.");
+ ret = -ENODEV;
+ goto tidyup_error;
+ }
+ ret = rxp_start_engine(priv->ctx, id);
+ if (ret) {
+ DRV_LOG(ERR, "Can't start engine.");
+ ret = -ENODEV;
+ goto tidyup_error;
+ }
+ }
+ rte_free(rules);
+ return 0;
+tidyup_error:
+ rte_free(rules);
+ return ret;
+}
+