+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "age data not available");
+ }
+ resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
+ AGE_TMOUT ? 1 : 0;
+ resp->sec_since_last_hit_valid = !resp->aged;
+ if (resp->sec_since_last_hit_valid)
+ resp->sec_since_last_hit = __atomic_load_n
+ (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
+ return 0;
+}
+
+/**
+ * Query a flow.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+static int
+flow_dv_query(struct rte_eth_dev *dev,
+ struct rte_flow *flow __rte_unused,
+ const struct rte_flow_action *actions __rte_unused,
+ void *data __rte_unused,
+ struct rte_flow_error *error __rte_unused)
+{
+ int ret = -EINVAL;
+
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_VOID:
+ break;
+ case RTE_FLOW_ACTION_TYPE_COUNT:
+ ret = flow_dv_query_count(dev, flow->counter, data,
+ error);
+ break;
+ case RTE_FLOW_ACTION_TYPE_AGE:
+ ret = flow_dv_query_age(dev, flow, data, error);
+ break;
+ default:
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "action not supported");
+ }
+ }
+ return ret;
+}
+
+/**
+ * Destroy the meter table set.
+ * Lock free, (mutex should be acquired by caller).
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] fm
+ * Meter information table.
+ */
+static void
+flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
+ struct mlx5_flow_meter_info *fm)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ int i;
+
+ if (!fm || !priv->config.dv_flow_en)
+ return;
+ for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
+ if (fm->drop_rule[i]) {
+ claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
+ fm->drop_rule[i] = NULL;
+ }
+ }
+}
+
+static void
+flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
+ struct mlx5_flow_tbl_data_entry *tbl;
+ int i, j;
+
+ for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
+ if (mtrmng->def_rule[i]) {
+ claim_zero(mlx5_flow_os_destroy_flow
+ (mtrmng->def_rule[i]));
+ mtrmng->def_rule[i] = NULL;
+ }
+ if (mtrmng->def_matcher[i]) {
+ tbl = container_of(mtrmng->def_matcher[i]->tbl,
+ struct mlx5_flow_tbl_data_entry, tbl);
+ mlx5_list_unregister(tbl->matchers,
+ &mtrmng->def_matcher[i]->entry);
+ mtrmng->def_matcher[i] = NULL;
+ }
+ for (j = 0; j < MLX5_REG_BITS; j++) {
+ if (mtrmng->drop_matcher[i][j]) {
+ tbl =
+ container_of(mtrmng->drop_matcher[i][j]->tbl,
+ struct mlx5_flow_tbl_data_entry,
+ tbl);
+ mlx5_list_unregister(tbl->matchers,
+ &mtrmng->drop_matcher[i][j]->entry);
+ mtrmng->drop_matcher[i][j] = NULL;
+ }
+ }
+ if (mtrmng->drop_tbl[i]) {
+ flow_dv_tbl_resource_release(MLX5_SH(dev),
+ mtrmng->drop_tbl[i]);
+ mtrmng->drop_tbl[i] = NULL;
+ }
+ }
+}
+
+/* Number of meter flow actions, count and jump or count and drop. */
+#define METER_ACTIONS 2
+
+static void
+__flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
+ enum mlx5_meter_domain domain)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_meter_def_policy *def_policy =
+ priv->sh->mtrmng->def_policy[domain];
+
+ __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
+ mlx5_free(def_policy);
+ priv->sh->mtrmng->def_policy[domain] = NULL;
+}
+
+/**
+ * Destroy the default policy table set.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ */
+static void
+flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ int i;
+
+ for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
+ if (priv->sh->mtrmng->def_policy[i])
+ __flow_dv_destroy_domain_def_policy(dev,
+ (enum mlx5_meter_domain)i);
+ priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
+}
+
+static int
+__flow_dv_create_policy_flow(struct rte_eth_dev *dev,
+ uint32_t color_reg_c_idx,
+ enum rte_color color, void *matcher_object,
+ int actions_n, void *actions,
+ bool match_src_port, const struct rte_flow_item *item,
+ void **rule, const struct rte_flow_attr *attr)
+{
+ int ret;
+ struct mlx5_flow_dv_match_params value = {
+ .size = sizeof(value.buf),
+ };
+ struct mlx5_flow_dv_match_params matcher = {
+ .size = sizeof(matcher.buf),
+ };
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint8_t misc_mask;
+
+ if (match_src_port && (priv->representor || priv->master)) {
+ if (flow_dv_translate_item_port_id(dev, matcher.buf,
+ value.buf, item, attr)) {
+ DRV_LOG(ERR,
+ "Failed to create meter policy flow with port.");
+ return -1;
+ }
+ }
+ flow_dv_match_meta_reg(matcher.buf, value.buf,
+ (enum modify_reg)color_reg_c_idx,
+ rte_col_2_mlx5_col(color),
+ UINT32_MAX);
+ misc_mask = flow_dv_matcher_enable(value.buf);
+ __flow_dv_adjust_buf_size(&value.size, misc_mask);
+ ret = mlx5_flow_os_create_flow(matcher_object,
+ (void *)&value, actions_n, actions, rule);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to create meter policy flow.");
+ return -1;
+ }
+ return 0;
+}
+
+static int
+__flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
+ uint32_t color_reg_c_idx,
+ uint16_t priority,
+ struct mlx5_flow_meter_sub_policy *sub_policy,
+ const struct rte_flow_attr *attr,
+ bool match_src_port,
+ const struct rte_flow_item *item,
+ struct mlx5_flow_dv_matcher **policy_matcher,
+ struct rte_flow_error *error)
+{
+ struct mlx5_list_entry *entry;
+ struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
+ struct mlx5_flow_dv_matcher matcher = {
+ .mask = {
+ .size = sizeof(matcher.mask.buf),
+ },
+ .tbl = tbl_rsc,
+ };
+ struct mlx5_flow_dv_match_params value = {
+ .size = sizeof(value.buf),
+ };
+ struct mlx5_flow_cb_ctx ctx = {
+ .error = error,
+ .data = &matcher,
+ };
+ struct mlx5_flow_tbl_data_entry *tbl_data;
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
+
+ if (match_src_port && (priv->representor || priv->master)) {
+ if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
+ value.buf, item, attr)) {
+ DRV_LOG(ERR,
+ "Failed to register meter drop matcher with port.");
+ return -1;
+ }
+ }
+ tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
+ if (priority < RTE_COLOR_RED)
+ flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
+ (enum modify_reg)color_reg_c_idx, 0, color_mask);
+ matcher.priority = priority;
+ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
+ matcher.mask.size);
+ entry = mlx5_list_register(tbl_data->matchers, &ctx);
+ if (!entry) {
+ DRV_LOG(ERR, "Failed to register meter drop matcher.");
+ return -1;
+ }
+ *policy_matcher =
+ container_of(entry, struct mlx5_flow_dv_matcher, entry);
+ return 0;
+}
+
+/**
+ * Create the policy rules per domain.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] sub_policy
+ * Pointer to sub policy table..
+ * @param[in] egress
+ * Direction of the table.
+ * @param[in] transfer
+ * E-Switch or NIC flow.
+ * @param[in] acts
+ * Pointer to policy action list per color.
+ *
+ * @return
+ * 0 on success, -1 otherwise.
+ */
+static int
+__flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
+ struct mlx5_flow_meter_sub_policy *sub_policy,
+ uint8_t egress, uint8_t transfer, bool match_src_port,
+ struct mlx5_meter_policy_acts acts[RTE_COLORS])
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow_error flow_err;
+ uint32_t color_reg_c_idx;
+ struct rte_flow_attr attr = {
+ .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
+ .priority = 0,
+ .ingress = 0,
+ .egress = !!egress,
+ .transfer = !!transfer,
+ .reserved = 0,
+ };
+ int i;
+ int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
+ struct mlx5_sub_policy_color_rule *color_rule;
+
+ if (ret < 0)
+ return -1;
+ /* Create policy table with POLICY level. */
+ if (!sub_policy->tbl_rsc)
+ sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
+ MLX5_FLOW_TABLE_LEVEL_POLICY,
+ egress, transfer, false, NULL, 0, 0,
+ sub_policy->idx, &flow_err);
+ if (!sub_policy->tbl_rsc) {
+ DRV_LOG(ERR,
+ "Failed to create meter sub policy table.");
+ return -1;
+ }
+ /* Prepare matchers. */
+ color_reg_c_idx = ret;
+ for (i = 0; i < RTE_COLORS; i++) {
+ TAILQ_INIT(&sub_policy->color_rules[i]);
+ if (i == RTE_COLOR_YELLOW || !acts[i].actions_n)
+ continue;
+ color_rule = mlx5_malloc(MLX5_MEM_ZERO,
+ sizeof(struct mlx5_sub_policy_color_rule),
+ 0, SOCKET_ID_ANY);
+ if (!color_rule) {
+ DRV_LOG(ERR, "No memory to create color rule.");
+ goto err_exit;
+ }
+ color_rule->src_port = priv->representor_id;
+ attr.priority = i;
+ /* Create matchers for Color. */
+ if (__flow_dv_create_policy_matcher(dev,
+ color_reg_c_idx, i, sub_policy, &attr,
+ (i != RTE_COLOR_RED ? match_src_port : false),
+ NULL, &color_rule->matcher, &flow_err)) {
+ DRV_LOG(ERR, "Failed to create color matcher.");
+ goto err_exit;
+ }
+ /* Create flow, matching color. */
+ if (__flow_dv_create_policy_flow(dev,
+ color_reg_c_idx, (enum rte_color)i,
+ color_rule->matcher->matcher_object,
+ acts[i].actions_n,
+ acts[i].dv_actions,
+ (i != RTE_COLOR_RED ? match_src_port : false),
+ NULL, &color_rule->rule,
+ &attr)) {
+ DRV_LOG(ERR, "Failed to create color rule.");
+ goto err_exit;
+ }
+ TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
+ color_rule, next_port);
+ }
+ return 0;
+err_exit:
+ if (color_rule) {
+ if (color_rule->rule)
+ mlx5_flow_os_destroy_flow(color_rule->rule);
+ if (color_rule->matcher) {
+ struct mlx5_flow_tbl_data_entry *tbl =
+ container_of(color_rule->matcher->tbl,
+ typeof(*tbl), tbl);
+ mlx5_list_unregister(tbl->matchers,
+ &color_rule->matcher->entry);
+ }
+ mlx5_free(color_rule);
+ }
+ return -1;
+}
+
+static int
+__flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
+ struct mlx5_flow_meter_policy *mtr_policy,
+ struct mlx5_flow_meter_sub_policy *sub_policy,
+ uint32_t domain)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_meter_policy_acts acts[RTE_COLORS];
+ struct mlx5_flow_dv_tag_resource *tag;
+ struct mlx5_flow_dv_port_id_action_resource *port_action;
+ struct mlx5_hrxq *hrxq;
+ struct mlx5_flow_meter_info *next_fm = NULL;
+ struct mlx5_flow_meter_policy *next_policy;
+ struct mlx5_flow_meter_sub_policy *next_sub_policy;
+ struct mlx5_flow_tbl_data_entry *tbl_data;
+ struct rte_flow_error error;
+ uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
+ uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
+ bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
+ bool match_src_port = false;
+ int i;
+
+ for (i = 0; i < RTE_COLORS; i++) {
+ acts[i].actions_n = 0;
+ if (i == RTE_COLOR_YELLOW)
+ continue;
+ if (i == RTE_COLOR_RED) {
+ /* Only support drop on red. */
+ acts[i].dv_actions[0] =
+ mtr_policy->dr_drop_action[domain];
+ acts[i].actions_n = 1;
+ continue;
+ }
+ if (mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
+ struct rte_flow_attr attr = {
+ .transfer = transfer
+ };
+
+ next_fm = mlx5_flow_meter_find(priv,
+ mtr_policy->act_cnt[i].next_mtr_id,
+ NULL);
+ if (!next_fm) {
+ DRV_LOG(ERR,
+ "Failed to get next hierarchy meter.");
+ goto err_exit;
+ }
+ if (mlx5_flow_meter_attach(priv, next_fm,
+ &attr, &error)) {
+ DRV_LOG(ERR, "%s", error.message);
+ next_fm = NULL;
+ goto err_exit;
+ }
+ /* Meter action must be the first for TX. */
+ if (mtr_first) {
+ acts[i].dv_actions[acts[i].actions_n] =
+ next_fm->meter_action;
+ acts[i].actions_n++;
+ }
+ }
+ if (mtr_policy->act_cnt[i].rix_mark) {
+ tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
+ mtr_policy->act_cnt[i].rix_mark);
+ if (!tag) {
+ DRV_LOG(ERR, "Failed to find "
+ "mark action for policy.");
+ goto err_exit;
+ }
+ acts[i].dv_actions[acts[i].actions_n] =
+ tag->action;
+ acts[i].actions_n++;
+ }
+ if (mtr_policy->act_cnt[i].modify_hdr) {
+ acts[i].dv_actions[acts[i].actions_n] =
+ mtr_policy->act_cnt[i].modify_hdr->action;
+ acts[i].actions_n++;
+ }
+ if (mtr_policy->act_cnt[i].fate_action) {
+ switch (mtr_policy->act_cnt[i].fate_action) {
+ case MLX5_FLOW_FATE_PORT_ID:
+ port_action = mlx5_ipool_get
+ (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
+ mtr_policy->act_cnt[i].rix_port_id_action);
+ if (!port_action) {
+ DRV_LOG(ERR, "Failed to find "
+ "port action for policy.");
+ goto err_exit;
+ }
+ acts[i].dv_actions[acts[i].actions_n] =
+ port_action->action;
+ acts[i].actions_n++;
+ mtr_policy->dev = dev;
+ match_src_port = true;
+ break;
+ case MLX5_FLOW_FATE_DROP:
+ case MLX5_FLOW_FATE_JUMP:
+ acts[i].dv_actions[acts[i].actions_n] =
+ mtr_policy->act_cnt[i].dr_jump_action[domain];
+ acts[i].actions_n++;
+ break;
+ case MLX5_FLOW_FATE_SHARED_RSS:
+ case MLX5_FLOW_FATE_QUEUE:
+ hrxq = mlx5_ipool_get
+ (priv->sh->ipool[MLX5_IPOOL_HRXQ],
+ sub_policy->rix_hrxq[i]);
+ if (!hrxq) {
+ DRV_LOG(ERR, "Failed to find "
+ "queue action for policy.");
+ goto err_exit;
+ }
+ acts[i].dv_actions[acts[i].actions_n] =
+ hrxq->action;
+ acts[i].actions_n++;
+ break;
+ case MLX5_FLOW_FATE_MTR:
+ if (!next_fm) {
+ DRV_LOG(ERR,
+ "No next hierarchy meter.");
+ goto err_exit;
+ }
+ if (!mtr_first) {
+ acts[i].dv_actions[acts[i].actions_n] =
+ next_fm->meter_action;
+ acts[i].actions_n++;
+ }
+ if (mtr_policy->act_cnt[i].next_sub_policy) {
+ next_sub_policy =
+ mtr_policy->act_cnt[i].next_sub_policy;
+ } else {
+ next_policy =
+ mlx5_flow_meter_policy_find(dev,
+ next_fm->policy_id, NULL);
+ MLX5_ASSERT(next_policy);
+ next_sub_policy =
+ next_policy->sub_policys[domain][0];
+ }
+ tbl_data =
+ container_of(next_sub_policy->tbl_rsc,
+ struct mlx5_flow_tbl_data_entry, tbl);
+ acts[i].dv_actions[acts[i].actions_n++] =
+ tbl_data->jump.action;
+ if (mtr_policy->act_cnt[i].modify_hdr)
+ match_src_port = !!transfer;
+ break;
+ default:
+ /*Queue action do nothing*/
+ break;
+ }
+ }
+ }
+ if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
+ egress, transfer, match_src_port, acts)) {
+ DRV_LOG(ERR,
+ "Failed to create policy rules per domain.");
+ goto err_exit;
+ }
+ return 0;
+err_exit:
+ if (next_fm)
+ mlx5_flow_meter_detach(priv, next_fm);
+ return -1;
+}
+
+/**
+ * Create the policy rules.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in,out] mtr_policy
+ * Pointer to meter policy table.
+ *
+ * @return
+ * 0 on success, -1 otherwise.
+ */
+static int
+flow_dv_create_policy_rules(struct rte_eth_dev *dev,
+ struct mlx5_flow_meter_policy *mtr_policy)
+{
+ int i;
+ uint16_t sub_policy_num;
+
+ for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
+ sub_policy_num = (mtr_policy->sub_policy_num >>
+ (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
+ MLX5_MTR_SUB_POLICY_NUM_MASK;
+ if (!sub_policy_num)
+ continue;
+ /* Prepare actions list and create policy rules. */
+ if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
+ mtr_policy->sub_policys[i][0], i)) {
+ DRV_LOG(ERR,
+ "Failed to create policy action list per domain.");
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int
+__flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
+ struct mlx5_flow_meter_def_policy *def_policy;
+ struct mlx5_flow_tbl_resource *jump_tbl;
+ struct mlx5_flow_tbl_data_entry *tbl_data;
+ uint8_t egress, transfer;
+ struct rte_flow_error error;
+ struct mlx5_meter_policy_acts acts[RTE_COLORS];
+ int ret;
+
+ egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
+ transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
+ def_policy = mtrmng->def_policy[domain];
+ if (!def_policy) {
+ def_policy = mlx5_malloc(MLX5_MEM_ZERO,
+ sizeof(struct mlx5_flow_meter_def_policy),
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+ if (!def_policy) {
+ DRV_LOG(ERR, "Failed to alloc "
+ "default policy table.");
+ goto def_policy_error;
+ }
+ mtrmng->def_policy[domain] = def_policy;
+ /* Create the meter suffix table with SUFFIX level. */
+ jump_tbl = flow_dv_tbl_resource_get(dev,
+ MLX5_FLOW_TABLE_LEVEL_METER,
+ egress, transfer, false, NULL, 0,
+ 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
+ if (!jump_tbl) {
+ DRV_LOG(ERR,
+ "Failed to create meter suffix table.");
+ goto def_policy_error;
+ }
+ def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
+ tbl_data = container_of(jump_tbl,
+ struct mlx5_flow_tbl_data_entry, tbl);
+ def_policy->dr_jump_action[RTE_COLOR_GREEN] =
+ tbl_data->jump.action;
+ acts[RTE_COLOR_GREEN].dv_actions[0] =
+ tbl_data->jump.action;
+ acts[RTE_COLOR_GREEN].actions_n = 1;
+ /* Create jump action to the drop table. */
+ if (!mtrmng->drop_tbl[domain]) {
+ mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
+ (dev, MLX5_FLOW_TABLE_LEVEL_METER,
+ egress, transfer, false, NULL, 0,
+ 0, MLX5_MTR_TABLE_ID_DROP, &error);
+ if (!mtrmng->drop_tbl[domain]) {
+ DRV_LOG(ERR, "Failed to create "
+ "meter drop table for default policy.");
+ goto def_policy_error;
+ }
+ }
+ tbl_data = container_of(mtrmng->drop_tbl[domain],
+ struct mlx5_flow_tbl_data_entry, tbl);
+ def_policy->dr_jump_action[RTE_COLOR_RED] =
+ tbl_data->jump.action;
+ acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
+ acts[RTE_COLOR_RED].actions_n = 1;
+ /* Create default policy rules. */
+ ret = __flow_dv_create_domain_policy_rules(dev,
+ &def_policy->sub_policy,
+ egress, transfer, false, acts);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to create "
+ "default policy rules.");
+ goto def_policy_error;
+ }
+ }
+ return 0;
+def_policy_error:
+ __flow_dv_destroy_domain_def_policy(dev,
+ (enum mlx5_meter_domain)domain);
+ return -1;
+}
+
+/**
+ * Create the default policy table set.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @return
+ * 0 on success, -1 otherwise.
+ */
+static int
+flow_dv_create_def_policy(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ int i;
+
+ /* Non-termination policy table. */
+ for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
+ if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER)
+ continue;
+ if (__flow_dv_create_domain_def_policy(dev, i)) {
+ DRV_LOG(ERR,
+ "Failed to create default policy");
+ return -1;
+ }
+ }
+ return 0;
+}
+
+/**
+ * Create the needed meter tables.
+ * Lock free, (mutex should be acquired by caller).
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] fm
+ * Meter information table.
+ * @param[in] mtr_idx
+ * Meter index.
+ * @param[in] domain_bitmap
+ * Domain bitmap.
+ * @return
+ * 0 on success, -1 otherwise.
+ */
+static int
+flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
+ struct mlx5_flow_meter_info *fm,
+ uint32_t mtr_idx,
+ uint8_t domain_bitmap)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
+ struct rte_flow_error error;
+ struct mlx5_flow_tbl_data_entry *tbl_data;
+ uint8_t egress, transfer;
+ void *actions[METER_ACTIONS];
+ int domain, ret, i;
+ struct mlx5_flow_counter *cnt;
+ struct mlx5_flow_dv_match_params value = {
+ .size = sizeof(value.buf),
+ };
+ struct mlx5_flow_dv_match_params matcher_para = {
+ .size = sizeof(matcher_para.buf),
+ };
+ int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
+ 0, &error);
+ uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
+ uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
+ struct mlx5_list_entry *entry;
+ struct mlx5_flow_dv_matcher matcher = {
+ .mask = {
+ .size = sizeof(matcher.mask.buf),
+ },
+ };
+ struct mlx5_flow_dv_matcher *drop_matcher;
+ struct mlx5_flow_cb_ctx ctx = {
+ .error = &error,
+ .data = &matcher,
+ };
+ uint8_t misc_mask;
+
+ if (!priv->mtr_en || mtr_id_reg_c < 0) {
+ rte_errno = ENOTSUP;
+ return -1;
+ }
+ for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
+ if (!(domain_bitmap & (1 << domain)) ||
+ (mtrmng->def_rule[domain] && !fm->drop_cnt))
+ continue;
+ egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
+ transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
+ /* Create the drop table with METER DROP level. */
+ if (!mtrmng->drop_tbl[domain]) {
+ mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
+ MLX5_FLOW_TABLE_LEVEL_METER,
+ egress, transfer, false, NULL, 0,
+ 0, MLX5_MTR_TABLE_ID_DROP, &error);
+ if (!mtrmng->drop_tbl[domain]) {
+ DRV_LOG(ERR, "Failed to create meter drop table.");
+ goto policy_error;
+ }
+ }
+ /* Create default matcher in drop table. */
+ matcher.tbl = mtrmng->drop_tbl[domain],
+ tbl_data = container_of(mtrmng->drop_tbl[domain],
+ struct mlx5_flow_tbl_data_entry, tbl);
+ if (!mtrmng->def_matcher[domain]) {
+ flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
+ (enum modify_reg)mtr_id_reg_c,
+ 0, 0);
+ matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
+ matcher.crc = rte_raw_cksum
+ ((const void *)matcher.mask.buf,
+ matcher.mask.size);
+ entry = mlx5_list_register(tbl_data->matchers, &ctx);
+ if (!entry) {
+ DRV_LOG(ERR, "Failed to register meter "
+ "drop default matcher.");
+ goto policy_error;
+ }
+ mtrmng->def_matcher[domain] = container_of(entry,
+ struct mlx5_flow_dv_matcher, entry);
+ }
+ /* Create default rule in drop table. */
+ if (!mtrmng->def_rule[domain]) {
+ i = 0;
+ actions[i++] = priv->sh->dr_drop_action;
+ flow_dv_match_meta_reg(matcher_para.buf, value.buf,
+ (enum modify_reg)mtr_id_reg_c, 0, 0);
+ misc_mask = flow_dv_matcher_enable(value.buf);
+ __flow_dv_adjust_buf_size(&value.size, misc_mask);
+ ret = mlx5_flow_os_create_flow
+ (mtrmng->def_matcher[domain]->matcher_object,
+ (void *)&value, i, actions,
+ &mtrmng->def_rule[domain]);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to create meter "
+ "default drop rule for drop table.");
+ goto policy_error;
+ }
+ }
+ if (!fm->drop_cnt)
+ continue;
+ MLX5_ASSERT(mtrmng->max_mtr_bits);
+ if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
+ /* Create matchers for Drop. */
+ flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
+ (enum modify_reg)mtr_id_reg_c, 0,
+ (mtr_id_mask << mtr_id_offset));
+ matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
+ matcher.crc = rte_raw_cksum
+ ((const void *)matcher.mask.buf,
+ matcher.mask.size);
+ entry = mlx5_list_register(tbl_data->matchers, &ctx);
+ if (!entry) {
+ DRV_LOG(ERR,
+ "Failed to register meter drop matcher.");
+ goto policy_error;
+ }
+ mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
+ container_of(entry, struct mlx5_flow_dv_matcher,
+ entry);
+ }
+ drop_matcher =
+ mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
+ /* Create drop rule, matching meter_id only. */
+ flow_dv_match_meta_reg(matcher_para.buf, value.buf,
+ (enum modify_reg)mtr_id_reg_c,
+ (mtr_idx << mtr_id_offset), UINT32_MAX);
+ i = 0;
+ cnt = flow_dv_counter_get_by_idx(dev,
+ fm->drop_cnt, NULL);
+ actions[i++] = cnt->action;
+ actions[i++] = priv->sh->dr_drop_action;
+ misc_mask = flow_dv_matcher_enable(value.buf);
+ __flow_dv_adjust_buf_size(&value.size, misc_mask);
+ ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
+ (void *)&value, i, actions,
+ &fm->drop_rule[domain]);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to create meter "
+ "drop rule for drop table.");
+ goto policy_error;
+ }
+ }
+ return 0;
+policy_error:
+ for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
+ if (fm->drop_rule[i]) {
+ claim_zero(mlx5_flow_os_destroy_flow
+ (fm->drop_rule[i]));
+ fm->drop_rule[i] = NULL;
+ }
+ }
+ return -1;
+}
+
+static struct mlx5_flow_meter_sub_policy *
+__flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
+ struct mlx5_flow_meter_policy *mtr_policy,
+ struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
+ struct mlx5_flow_meter_sub_policy *next_sub_policy,
+ bool *is_reuse)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
+ uint32_t sub_policy_idx = 0;
+ uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
+ uint32_t i, j;
+ struct mlx5_hrxq *hrxq;
+ struct mlx5_flow_handle dh;
+ struct mlx5_meter_policy_action_container *act_cnt;
+ uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
+ uint16_t sub_policy_num;
+
+ rte_spinlock_lock(&mtr_policy->sl);
+ for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
+ if (!rss_desc[i])
+ continue;
+ hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]);
+ if (!hrxq_idx[i]) {
+ rte_spinlock_unlock(&mtr_policy->sl);
+ return NULL;
+ }
+ }
+ sub_policy_num = (mtr_policy->sub_policy_num >>
+ (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
+ MLX5_MTR_SUB_POLICY_NUM_MASK;
+ for (i = 0; i < sub_policy_num;
+ i++) {
+ for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
+ if (rss_desc[j] &&
+ hrxq_idx[j] !=
+ mtr_policy->sub_policys[domain][i]->rix_hrxq[j])
+ break;
+ }
+ if (j >= MLX5_MTR_RTE_COLORS) {
+ /*
+ * Found the sub policy table with
+ * the same queue per color
+ */
+ rte_spinlock_unlock(&mtr_policy->sl);
+ for (j = 0; j < MLX5_MTR_RTE_COLORS; j++)
+ mlx5_hrxq_release(dev, hrxq_idx[j]);
+ *is_reuse = true;
+ return mtr_policy->sub_policys[domain][i];
+ }
+ }
+ /* Create sub policy. */
+ if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
+ /* Reuse the first dummy sub_policy*/
+ sub_policy = mtr_policy->sub_policys[domain][0];
+ sub_policy_idx = sub_policy->idx;
+ } else {
+ sub_policy = mlx5_ipool_zmalloc
+ (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
+ &sub_policy_idx);
+ if (!sub_policy ||
+ sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
+ for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
+ mlx5_hrxq_release(dev, hrxq_idx[i]);
+ goto rss_sub_policy_error;
+ }
+ sub_policy->idx = sub_policy_idx;
+ sub_policy->main_policy = mtr_policy;