X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow.c;h=3194cd5633e07cb7d23f4323999c48ac07593ce1;hb=0a4291173989faa02a9a4250246ab6c9da4ff741;hp=2b5f82a70e129692c3b62a31db61a3689f13c315;hpb=151cbe3aabf9e1a71e02827e5e1ff0814275019a;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 2b5f82a70e..3194cd5633 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -28,8 +28,8 @@ #include "mlx5.h" #include "mlx5_flow.h" #include "mlx5_flow_os.h" -#include "mlx5_rxtx.h" #include "mlx5_rx.h" +#include "mlx5_tx.h" #include "mlx5_common_os.h" #include "rte_pmd_mlx5.h" @@ -567,23 +567,23 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { }, }; -static struct rte_flow_shared_action * -mlx5_shared_action_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, +static struct rte_flow_action_handle * +mlx5_action_handle_create(struct rte_eth_dev *dev, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action *action, struct rte_flow_error *error); -static int mlx5_shared_action_destroy +static int mlx5_action_handle_destroy (struct rte_eth_dev *dev, - struct rte_flow_shared_action *shared_action, + struct rte_flow_action_handle *handle, struct rte_flow_error *error); -static int mlx5_shared_action_update +static int mlx5_action_handle_update (struct rte_eth_dev *dev, - struct rte_flow_shared_action *shared_action, - const struct rte_flow_action *action, + struct rte_flow_action_handle *handle, + const void *update, struct rte_flow_error *error); -static int mlx5_shared_action_query +static int mlx5_action_handle_query (struct rte_eth_dev *dev, - const struct rte_flow_shared_action *action, + const struct rte_flow_action_handle *handle, void *data, struct rte_flow_error *error); static int @@ -622,10 +622,10 @@ static const struct rte_flow_ops mlx5_flow_ops = { .query = mlx5_flow_query, .dev_dump = mlx5_flow_dev_dump, .get_aged_flows = mlx5_flow_get_aged_flows, - .shared_action_create = mlx5_shared_action_create, - .shared_action_destroy = mlx5_shared_action_destroy, - .shared_action_update = mlx5_shared_action_update, - .shared_action_query = mlx5_shared_action_query, + .action_handle_create = mlx5_action_handle_create, + .action_handle_destroy = mlx5_action_handle_destroy, + .action_handle_update = mlx5_action_handle_update, + .action_handle_query = mlx5_action_handle_query, .tunnel_decap_set = mlx5_flow_tunnel_decap_set, .tunnel_match = mlx5_flow_tunnel_match, .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, @@ -748,9 +748,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -759,7 +759,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; case MLX5_MTR_COLOR: - case MLX5_ASO_FLOW_HIT: /* Both features use the same REG_C. */ + case MLX5_ASO_FLOW_HIT: + case MLX5_ASO_CONNTRACK: + /* All features use the same REG_C. */ MLX5_ASSERT(priv->mtr_color_reg != REG_NON); return priv->mtr_color_reg; case MLX5_COPY_MARK: @@ -1028,7 +1030,7 @@ flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) * @param[in] dev_handle * Pointer to device flow handle structure. */ -static void +void flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow_handle *dev_handle) { @@ -1685,6 +1687,37 @@ mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, return 0; } +/* + * Validate the ASO CT action. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] conntrack + * Pointer to the CT action profile. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_validate_action_ct(struct rte_eth_dev *dev, + const struct rte_flow_action_conntrack *conntrack, + struct rte_flow_error *error) +{ + RTE_SET_USED(dev); + + if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Invalid CT state"); + if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Invalid last TCP packet flag"); + return 0; +} + /** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. @@ -3035,7 +3068,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3342,18 +3376,52 @@ flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) fops->destroy(dev, flow); } +/** + * Flow driver find RSS policy tbl API. This abstracts calling driver + * specific functions. Parent flow (rte_flow) should have driver + * type (drv_type). It will find the RSS policy table that has the rss_desc. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in, out] flow + * Pointer to flow structure. + * @param[in] policy + * Pointer to meter policy table. + * @param[in] rss_desc + * Pointer to rss_desc + */ +static struct mlx5_flow_meter_sub_policy * +flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct mlx5_flow_meter_policy *policy, + struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) +{ + const struct mlx5_flow_driver_ops *fops; + enum mlx5_flow_drv_type type = flow->drv_type; + + MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); + fops = flow_get_drv_ops(type); + return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); +} + /** * Get RSS action from the action list. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] actions * Pointer to the list of actions. + * @param[in] flow + * Parent flow structure pointer. * * @return * Pointer to the RSS action if exist, else return NULL. */ static const struct rte_flow_action_rss* -flow_get_rss_action(const struct rte_flow_action actions[]) +flow_get_rss_action(struct rte_eth_dev *dev, + const struct rte_flow_action actions[]) { + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_rss *rss = NULL; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -3371,6 +3439,23 @@ flow_get_rss_action(const struct rte_flow_action actions[]) rss = act->conf; break; } + case RTE_FLOW_ACTION_TYPE_METER: + { + uint32_t mtr_idx; + struct mlx5_flow_meter_info *fm; + struct mlx5_flow_meter_policy *policy; + const struct rte_flow_action_meter *mtr = actions->conf; + + fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); + if (fm) { + policy = mlx5_flow_meter_policy_find(dev, + fm->policy_id, NULL); + if (policy && policy->is_rss) + rss = + policy->act_cnt[RTE_COLOR_GREEN].rss->conf; + } + break; + } default: break; } @@ -3401,31 +3486,31 @@ flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) return &pool->actions[offset - 1]; } -/* maps shared action to translated non shared in some actions array */ -struct mlx5_translated_shared_action { - struct rte_flow_shared_action *action; /**< Shared action */ - int index; /**< Index in related array of rte_flow_action */ +/* maps indirect action to translated direct in some actions array */ +struct mlx5_translated_action_handle { + struct rte_flow_action_handle *action; /**< Indirect action handle. */ + int index; /**< Index in related array of rte_flow_action. */ }; /** - * Translates actions of type RTE_FLOW_ACTION_TYPE_SHARED to related - * non shared action if translation possible. - * This functionality used to run same execution path for both shared & non - * shared actions on flow create. All necessary preparations for shared - * action handling should be preformed on *shared* actions list returned + * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related + * direct action if translation possible. + * This functionality used to run same execution path for both direct and + * indirect actions on flow create. All necessary preparations for indirect + * action handling should be performed on *handle* actions list returned * from this call. * * @param[in] dev * Pointer to Ethernet device. * @param[in] actions * List of actions to translate. - * @param[out] shared - * List to store translated shared actions. - * @param[in, out] shared_n - * Size of *shared* array. On return should be updated with number of shared - * actions retrieved from the *actions* list. + * @param[out] handle + * List to store translated indirect action object handles. + * @param[in, out] indir_n + * Size of *handle* array. On return should be updated with number of + * indirect actions retrieved from the *actions* list. * @param[out] translated_actions - * List of actions where all shared actions were translated to non shared + * List of actions where all indirect actions were translated to direct * if possible. NULL if no translation took place. * @param[out] error * Pointer to the error structure. @@ -3434,10 +3519,10 @@ struct mlx5_translated_shared_action { * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_shared_actions_translate(struct rte_eth_dev *dev, +flow_action_handles_translate(struct rte_eth_dev *dev, const struct rte_flow_action actions[], - struct mlx5_translated_shared_action *shared, - int *shared_n, + struct mlx5_translated_action_handle *handle, + int *indir_n, struct rte_flow_action **translated_actions, struct rte_flow_error *error) { @@ -3446,23 +3531,23 @@ flow_shared_actions_translate(struct rte_eth_dev *dev, size_t actions_size; int n; int copied_n = 0; - struct mlx5_translated_shared_action *shared_end = NULL; + struct mlx5_translated_action_handle *handle_end = NULL; for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { - if (actions[n].type != RTE_FLOW_ACTION_TYPE_SHARED) + if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) continue; - if (copied_n == *shared_n) { + if (copied_n == *indir_n) { return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL, "too many shared actions"); } - rte_memcpy(&shared[copied_n].action, &actions[n].conf, + rte_memcpy(&handle[copied_n].action, &actions[n].conf, sizeof(actions[n].conf)); - shared[copied_n].index = n; + handle[copied_n].index = n; copied_n++; } n++; - *shared_n = copied_n; + *indir_n = copied_n; if (!copied_n) return 0; actions_size = sizeof(struct rte_flow_action) * n; @@ -3472,28 +3557,43 @@ flow_shared_actions_translate(struct rte_eth_dev *dev, return -ENOMEM; } memcpy(translated, actions, actions_size); - for (shared_end = shared + copied_n; shared < shared_end; shared++) { + for (handle_end = handle + copied_n; handle < handle_end; handle++) { struct mlx5_shared_action_rss *shared_rss; - uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; - uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; - uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - - 1); + uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & + ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); switch (type) { - case MLX5_SHARED_ACTION_TYPE_RSS: + case MLX5_INDIRECT_ACTION_TYPE_RSS: shared_rss = mlx5_ipool_get (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); - translated[shared->index].type = + translated[handle->index].type = RTE_FLOW_ACTION_TYPE_RSS; - translated[shared->index].conf = + translated[handle->index].conf = &shared_rss->origin; break; - case MLX5_SHARED_ACTION_TYPE_AGE: + case MLX5_INDIRECT_ACTION_TYPE_COUNT: + translated[handle->index].type = + (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_COUNT; + translated[handle->index].conf = (void *)(uintptr_t)idx; + break; + case MLX5_INDIRECT_ACTION_TYPE_AGE: if (priv->sh->flow_hit_aso_en) { - translated[shared->index].type = + translated[handle->index].type = (enum rte_flow_action_type) MLX5_RTE_FLOW_ACTION_TYPE_AGE; - translated[shared->index].conf = + translated[handle->index].conf = + (void *)(uintptr_t)idx; + break; + } + /* Fall-through */ + case MLX5_INDIRECT_ACTION_TYPE_CT: + if (priv->sh->ct_aso_en) { + translated[handle->index].type = + RTE_FLOW_ACTION_TYPE_CONNTRACK; + translated[handle->index].conf = (void *)(uintptr_t)idx; break; } @@ -3502,7 +3602,7 @@ flow_shared_actions_translate(struct rte_eth_dev *dev, mlx5_free(translated); return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, - NULL, "invalid shared action type"); + NULL, "invalid indirect action type"); } } *translated_actions = translated; @@ -3524,21 +3624,21 @@ flow_shared_actions_translate(struct rte_eth_dev *dev, */ static uint32_t flow_get_shared_rss_action(struct rte_eth_dev *dev, - struct mlx5_translated_shared_action *shared, + struct mlx5_translated_action_handle *handle, int shared_n) { - struct mlx5_translated_shared_action *shared_end; + struct mlx5_translated_action_handle *handle_end; struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_shared_action_rss *shared_rss; - for (shared_end = shared + shared_n; shared < shared_end; shared++) { - uint32_t act_idx = (uint32_t)(uintptr_t)shared->action; - uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; + for (handle_end = handle + shared_n; handle < handle_end; handle++) { + uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; uint32_t idx = act_idx & - ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); switch (type) { - case MLX5_SHARED_ACTION_TYPE_RSS: + case MLX5_INDIRECT_ACTION_TYPE_RSS: shared_rss = mlx5_ipool_get (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); @@ -3669,32 +3769,104 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], return actions_n + 1; } +/** + * Check if the action will change packet. + * + * @param dev + * Pointer to Ethernet device. + * @param[in] type + * action type. + * + * @return + * true if action will change packet, false otherwise. + */ +static bool flow_check_modify_action_type(struct rte_eth_dev *dev, + enum rte_flow_action_type type) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + switch (type) { + case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: + case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: + case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: + case RTE_FLOW_ACTION_TYPE_SET_TP_DST: + case RTE_FLOW_ACTION_TYPE_DEC_TTL: + case RTE_FLOW_ACTION_TYPE_SET_TTL: + case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: + case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: + case RTE_FLOW_ACTION_TYPE_SET_META: + case RTE_FLOW_ACTION_TYPE_SET_TAG: + case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: + case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: + case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: + case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: + case RTE_FLOW_ACTION_TYPE_RAW_DECAP: + case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: + return true; + case RTE_FLOW_ACTION_TYPE_FLAG: + case RTE_FLOW_ACTION_TYPE_MARK: + if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) + return true; + else + return false; + default: + return false; + } +} + /** * Check meter action from the action list. * + * @param dev + * Pointer to Ethernet device. * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] has_modify + * Pointer to the flag showing there's packet change action. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(struct rte_eth_dev *dev, + const struct rte_flow_action actions[], + bool *has_mtr, bool *has_modify, uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; } + if (!*has_mtr) + *has_modify |= flow_check_modify_action_type(dev, + actions->type); actions_n++; } /* Count RTE_FLOW_ACTION_TYPE_END. */ @@ -4250,9 +4422,11 @@ flow_hairpin_split(struct rte_eth_dev *dev, rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); actions_rx++; set_tag = (void *)actions_rx; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), + .data = flow_id, + }; MLX5_ASSERT(set_tag->id > REG_NON); - set_tag->data = flow_id; tag_action->conf = set_tag; /* Create Tx item list. */ rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); @@ -4332,9 +4506,114 @@ flow_create_split_inner(struct rte_eth_dev *dev, dev_flow->handle->mark = 1; if (sub_flow) *sub_flow = dev_flow; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + dev_flow->dv.table_id = flow_split_info->table_id; +#endif return flow_drv_translate(dev, dev_flow, attr, items, actions, error); } +/** + * Get the sub policy of a meter. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. + * @param[in] policy_id; + * Meter Policy id. + * @param[in] attr + * Flow rule attributes. + * @param[in] items + * Pattern specification (list terminated by the END pattern item). + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. + */ +static struct mlx5_flow_meter_sub_policy * +get_meter_sub_policy(struct rte_eth_dev *dev, + struct rte_flow *flow, + uint32_t policy_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + struct rte_flow_error *error) +{ + struct mlx5_flow_meter_policy *policy; + struct mlx5_flow_meter_sub_policy *sub_policy = NULL; + + policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); + if (!policy) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to find Meter Policy."); + goto exit; + } + if (policy->is_rss) { + struct mlx5_flow_workspace *wks = + mlx5_flow_get_thread_workspace(); + struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; + struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; + uint32_t i; + + MLX5_ASSERT(wks); + /** + * This is a tmp dev_flow, + * no need to register any matcher for it in translate. + */ + wks->skip_matcher_reg = 1; + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { + struct mlx5_flow dev_flow = {0}; + struct mlx5_flow_handle dev_handle = { {0} }; + const void *rss_act = policy->act_cnt[i].rss->conf; + struct rte_flow_action rss_actions[2] = { + [0] = { + .type = RTE_FLOW_ACTION_TYPE_RSS, + .conf = rss_act + }, + [1] = { + .type = RTE_FLOW_ACTION_TYPE_END, + .conf = NULL + } + }; + + dev_flow.handle = &dev_handle; + dev_flow.ingress = attr->ingress; + dev_flow.flow = flow; + dev_flow.external = 0; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + dev_flow.dv.transfer = attr->transfer; +#endif + /* Translate RSS action to get rss hash fields. */ + if (flow_drv_translate(dev, &dev_flow, attr, + items, rss_actions, error)) + goto exit; + rss_desc_v[i] = wks->rss_desc; + rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; + rss_desc_v[i].hash_fields = dev_flow.hash_fields; + rss_desc_v[i].queue_num = rss_desc_v[i].hash_fields ? + rss_desc_v[i].queue_num : 1; + rss_desc[i] = &rss_desc_v[i]; + } + sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, + flow, policy, rss_desc); + } else { + enum mlx5_meter_domain mtr_domain = + attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : + attr->egress ? MLX5_MTR_DOMAIN_EGRESS : + MLX5_MTR_DOMAIN_INGRESS; + sub_policy = policy->sub_policys[mtr_domain][0]; + } + if (!sub_policy) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to get meter sub-policy."); + goto exit; + } +exit: + return sub_policy; +} + /** * Split the meter flow. * @@ -4347,8 +4626,14 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. + * @param[in] fm + * Pointer to flow meter structure. + * @param[in] attr + * Flow rule attributes. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4359,60 +4644,83 @@ flow_create_split_inner(struct rte_eth_dev *dev, * Suffix flow actions. * @param[out] actions_pre * Prefix flow actions. - * @param[out] pattern_sfx - * The pattern items for the suffix flow. - * @param[out] tag_sfx - * Pointer to suffix flow tag. + * @param[out] mtr_flow_id + * Pointer to meter flow id. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct rte_flow *flow, + struct mlx5_flow_meter_info *fm, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + uint32_t *mtr_flow_id, + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; - + struct rte_flow_action *hw_mtr_action; + struct rte_flow_action *action_pre_head = NULL; + bool mtr_first = priv->sh->meter_aso_en && + (attr->egress || + (attr->transfer && priv->representor_id != UINT16_MAX)); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint32_t flow_id = 0; + uint32_t flow_id_reversed = 0; + uint8_t flow_id_bits = 0; + int shift; + + /* For ASO meter, meter must be before tag in TX direction. */ + if (mtr_first) { + action_pre_head = actions_pre++; + /* Leave space for tag action. */ + tag_action = actions_pre++; + } /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { - struct rte_flow_action **action_cur = NULL; + struct rte_flow_action *action_cur = NULL; switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; - action_cur = &actions_pre; + if (mtr_first) { + action_cur = action_pre_head; + } else { + /* Leave space for tag action. */ + tag_action = actions_pre++; + action_cur = actions_pre++; + } break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: raw_encap = actions->conf; if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_RAW_DECAP: raw_decap = actions->conf; if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: @@ -4422,31 +4730,72 @@ flow_meter_split_prep(struct rte_eth_dev *dev, break; } if (!action_cur) - action_cur = &actions_sfx; - memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); - (*action_cur)++; + action_cur = (fm->def_policy) ? + actions_sfx++ : actions_pre++; + memcpy(action_cur, actions, sizeof(struct rte_flow_action)); } /* Add end action to the actions. */ actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; + if (priv->sh->meter_aso_en) { + /** + * For ASO meter, need to add an extra jump action explicitly, + * to jump from meter to policer table. + */ + struct mlx5_flow_meter_sub_policy *sub_policy; + struct mlx5_flow_tbl_data_entry *tbl_data; + + if (!fm->def_policy) { + sub_policy = get_meter_sub_policy(dev, flow, + fm->policy_id, attr, + items, error); + if (!sub_policy) + return -rte_errno; + } else { + enum mlx5_meter_domain mtr_domain = + attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : + attr->egress ? MLX5_MTR_DOMAIN_EGRESS : + MLX5_MTR_DOMAIN_INGRESS; + + sub_policy = + &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; + } + tbl_data = container_of(sub_policy->tbl_rsc, + struct mlx5_flow_tbl_data_entry, tbl); + hw_mtr_action = actions_pre++; + hw_mtr_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_JUMP; + hw_mtr_action->conf = tbl_data->jump.action; + } actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + if (!tag_action) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "No tag action space."); + if (!mtr_flow_id) { + tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; + goto exit; + } + /* Only default-policy Meter creates mtr flow id. */ + if (fm->def_policy) { + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id = tag_id - 1; + flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > + mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); + } + if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) + priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4475,17 +4824,42 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* Both flow_id and meter_id share the same register. */ + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, error), + .offset = mtr_id_offset, + .length = mtr_reg_bits, + .data = flow->meter, + }; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_reversed = (flow_id_reversed << 1) | + ((flow_id >> shift) & 0x1); + set_tag->data |= + flow_id_reversed << (mtr_reg_bits - flow_id_bits); + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data << mtr_id_offset; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; - return tag_id; + tag_item->mask = tag_item_mask; +exit: + if (mtr_flow_id) + *mtr_flow_id = tag_id; + return 0; } /** @@ -4743,6 +5117,14 @@ flow_check_match_action(const struct rte_flow_action actions[], case RTE_FLOW_ACTION_TYPE_MARK: case RTE_FLOW_ACTION_TYPE_SET_META: case RTE_FLOW_ACTION_TYPE_SET_TAG: + case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: + case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: + case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: + case RTE_FLOW_ACTION_TYPE_RAW_DECAP: + case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: if (fdb_mirror) *modify_after_mirror = 1; break; @@ -4862,10 +5244,12 @@ flow_sample_split_prep(struct rte_eth_dev *dev, ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); if (ret < 0) return ret; - set_tag->id = ret; mlx5_ipool_malloc(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); - set_tag->data = tag_id; + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = ret, + .data = tag_id, + }; /* Prepare the suffix subflow items. */ tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); tag_spec->data = tag_id; @@ -4998,8 +5382,8 @@ flow_create_split_metadata(struct rte_eth_dev *dev, if (qrss) { /* Check if it is in meter suffix table. */ mtr_sfx = attr->group == (attr->transfer ? - (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : - MLX5_FLOW_TABLE_LEVEL_SUFFIX); + (MLX5_FLOW_TABLE_LEVEL_METER - 1) : + MLX5_FLOW_TABLE_LEVEL_METER); /* * Q/RSS action on NIC Rx should be split in order to pass by * the mreg copy table (RX_CP_TBL) and then it jumps to the @@ -5147,6 +5531,57 @@ exit: return ret; } +/** + * Create meter internal drop flow with the original pattern. + * + * @param dev + * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. + * @param[in] attr + * Flow rule attributes. + * @param[in] items + * Pattern specification (list terminated by the END pattern item). + * @param[in] flow_split_info + * Pointer to flow split info structure. + * @param[in] fm + * Pointer to flow meter structure. + * @param[out] error + * Perform verbose error reporting if not NULL. + * @return + * 0 on success, negative value otherwise + */ +static uint32_t +flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + struct mlx5_flow_split_info *flow_split_info, + struct mlx5_flow_meter_info *fm, + struct rte_flow_error *error) +{ + struct mlx5_flow *dev_flow = NULL; + struct rte_flow_attr drop_attr = *attr; + struct rte_flow_action drop_actions[3]; + struct mlx5_flow_split_info drop_split_info = *flow_split_info; + + MLX5_ASSERT(fm->drop_cnt); + drop_actions[0].type = + (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; + drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; + drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; + drop_actions[1].conf = NULL; + drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; + drop_actions[2].conf = NULL; + drop_split_info.external = false; + drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; + drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; + drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; + return flow_create_split_inner(dev, flow, &dev_flow, + &drop_attr, items, drop_actions, + &drop_split_info, error); +} + /** * The splitting for meter feature. * @@ -5184,25 +5619,62 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; - uint32_t mtr_tag_id = 0; + struct mlx5_flow_meter_info *fm = NULL; + uint8_t skip_scale_restore; + bool has_mtr = false; + bool has_modify = false; + bool set_mtr_reg = true; + uint32_t meter_id = 0; + uint32_t mtr_idx = 0; + uint32_t mtr_flow_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ - act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + + actions_n = flow_check_meter_action(dev, actions, &has_mtr, + &has_modify, &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = flow_dv_meter_find_by_idx(priv, flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = mtr_idx; + } + MLX5_ASSERT(wks); + wks->fm = fm; + /* + * If it isn't default-policy Meter, and + * 1. There's no action in flow to change + * packet (modify/encap/decap etc.), OR + * 2. No drop count needed for this meter. + * no need to use regC to save meter id anymore. + */ + if (!fm->def_policy && (!has_modify || !fm->drop_cnt)) + set_mtr_reg = false; + /* Prefix actions: meter, decap, encap, tag, jump, end. */ + act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* Suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5215,31 +5687,56 @@ flow_create_split_meter(struct rte_eth_dev *dev, "meter flow"); sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); - pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, - actions, sfx_actions, - pre_actions); - if (!mtr_tag_id) { + /* There's no suffix flow for meter of non-default policy. */ + if (!fm->def_policy) + pre_actions = sfx_actions + 1; + else + pre_actions = sfx_actions + actions_n; + ret = flow_meter_split_prep(dev, flow, fm, &sfx_attr, + items, sfx_items, actions, + sfx_actions, pre_actions, + (set_mtr_reg ? &mtr_flow_id : NULL), + error); + if (ret) { ret = -rte_errno; goto exit; } /* Add the prefix subflow. */ flow_split_info->prefix_mark = 0; + skip_scale_restore = flow_split_info->skip_scale; + flow_split_info->skip_scale |= + 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; ret = flow_create_split_inner(dev, flow, &dev_flow, attr, items, pre_actions, flow_split_info, error); + flow_split_info->skip_scale = skip_scale_restore; if (ret) { + if (mtr_flow_id) + mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); ret = -rte_errno; goto exit; } - dev_flow->handle->split_flow_id = mtr_tag_id; + if (mtr_flow_id) { + dev_flow->handle->split_flow_id = mtr_flow_id; + dev_flow->handle->is_meter_flow_id = 1; + } + if (!fm->def_policy) { + if (!set_mtr_reg && fm->drop_cnt) + ret = + flow_meter_create_drop_flow_with_org_pattern(dev, flow, + &sfx_attr, items, + flow_split_info, + fm, error); + goto exit; + } /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? - (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : - MLX5_FLOW_TABLE_LEVEL_SUFFIX; + (MLX5_FLOW_TABLE_LEVEL_METER - 1) : + MLX5_FLOW_TABLE_LEVEL_METER; flow_split_info->prefix_layers = flow_get_prefix_layer_flags(dev_flow); flow_split_info->prefix_mark = dev_flow->handle->mark; + flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; } /* Add the prefix subflow. */ ret = flow_create_split_metadata(dev, flow, @@ -5383,8 +5880,7 @@ flow_create_split_sample(struct rte_eth_dev *dev, struct mlx5_flow_tbl_data_entry, tbl); sfx_attr.group = sfx_attr.transfer ? - (sfx_tbl_data->table_id - 1) : - sfx_tbl_data->table_id; + (sfx_tbl_data->level - 1) : sfx_tbl_data->level; } else { MLX5_ASSERT(attr->transfer); sfx_attr.group = jump_table; @@ -5548,9 +6044,9 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list, struct rte_flow *flow = NULL; struct mlx5_flow *dev_flow; const struct rte_flow_action_rss *rss = NULL; - struct mlx5_translated_shared_action - shared_actions[MLX5_MAX_SHARED_ACTIONS]; - int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; + struct mlx5_translated_action_handle + indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; + int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; union { struct mlx5_flow_expand_rss buf; uint8_t buffer[2048]; @@ -5584,15 +6080,16 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list, .skip_scale = 0, .flow_idx = 0, .prefix_mark = 0, - .prefix_layers = 0 + .prefix_layers = 0, + .table_id = 0 }; int ret; MLX5_ASSERT(wks); rss_desc = &wks->rss_desc; - ret = flow_shared_actions_translate(dev, original_actions, - shared_actions, - &shared_actions_n, + ret = flow_action_handles_translate(dev, original_actions, + indir_actions, + &indir_actions_n, &translated_actions, error); if (ret < 0) { MLX5_ASSERT(translated_actions == NULL); @@ -5627,7 +6124,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list, memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); /* RSS Action only works on NIC RX domain */ if (attr->ingress && !attr->transfer) - rss = flow_get_rss_action(p_actions_rx); + rss = flow_get_rss_action(dev, p_actions_rx); if (rss) { if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) return 0; @@ -5653,8 +6150,8 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list, buf->entries = 1; buf->entry[0].pattern = (void *)(uintptr_t)items; } - rss_desc->shared_rss = flow_get_shared_rss_action(dev, shared_actions, - shared_actions_n); + rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, + indir_actions_n); for (i = 0; i < buf->entries; ++i) { /* Initialize flow split data. */ flow_split_info.prefix_layers = 0; @@ -5833,14 +6330,14 @@ mlx5_flow_validate(struct rte_eth_dev *dev, struct rte_flow_error *error) { int hairpin_flow; - struct mlx5_translated_shared_action - shared_actions[MLX5_MAX_SHARED_ACTIONS]; - int shared_actions_n = MLX5_MAX_SHARED_ACTIONS; + struct mlx5_translated_action_handle + indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; + int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; const struct rte_flow_action *actions; struct rte_flow_action *translated_actions = NULL; - int ret = flow_shared_actions_translate(dev, original_actions, - shared_actions, - &shared_actions_n, + int ret = flow_action_handles_translate(dev, original_actions, + indir_actions, + &indir_actions_n, &translated_actions, error); if (ret) @@ -6500,24 +6997,42 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, } /** - * Create the needed meter and suffix tables. + * Validate meter policy actions. + * Dispatcher for action type specific validation. * * @param[in] dev - * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. + * Pointer to the Ethernet device structure. + * @param[in] action + * The meter policy action object to validate. + * @param[in] attr + * Attributes of flow to determine steering domain. + * @param[out] is_rss + * Is RSS or not. + * @param[out] domain_bitmap + * Domain bitmap. + * @param[out] is_def_policy + * Is default policy or not. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. * * @return - * Pointer to table set on success, NULL otherwise. + * 0 on success, otherwise negative errno value. */ -struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +int +mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, + const struct rte_flow_action *actions[RTE_COLORS], + struct rte_flow_attr *attr, + bool *is_rss, + uint8_t *domain_bitmap, + bool *is_def_policy, + struct rte_mtr_error *error) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->validate_mtr_acts(dev, actions, attr, + is_rss, domain_bitmap, is_def_policy, error); } /** @@ -6525,66 +7040,215 @@ mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] tbl - * Pointer to the meter table set. + * @param[in] mtr_policy + * Meter policy struct. + */ +void +mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->destroy_mtr_acts(dev, mtr_policy); +} + +/** + * Create policy action, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. + * @param[in] action + * Action specification used to create meter actions. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. * * @return - * 0 on success. + * 0 on success, otherwise negative errno value. */ int -mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, - struct mlx5_meter_domains_infos *tbls) +mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy, + const struct rte_flow_action *actions[RTE_COLORS], + struct rte_mtr_error *error) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_mtr_acts(dev, mtr_policy, actions, error); +} + +/** + * Create policy rules, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. + * + * @return + * 0 on success, -1 otherwise. + */ +int +mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_policy_rules(dev, mtr_policy); +} + +/** + * Destroy policy rules, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. + */ +void +mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->destroy_mtr_tbls(dev, tbls); + fops->destroy_policy_rules(dev, mtr_policy); } /** - * Create policer rules. + * Destroy the default policy table set. + * + * @param[in] dev + * Pointer to Ethernet device. + */ +void +mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->destroy_def_policy(dev); +} + +/** + * Destroy the default policy table set. * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to flow meter structure. - * @param[in] attr - * Pointer to flow attributes. * * @return * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +mlx5_flow_create_def_policy(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->create_def_policy(dev); } /** - * Destroy policer rules. + * Create the needed meter and suffix tables. * - * @param[in] fm - * Pointer to flow meter structure. - * @param[in] attr - * Pointer to flow attributes. + * @param[in] dev + * Pointer to Ethernet device. * * @return * 0 on success, -1 otherwise. */ int -mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm, + uint32_t mtr_idx, + uint8_t domain_bitmap) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); +} + +/** + * Destroy the meter table set. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] tbl + * Pointer to the meter table set. + */ +void +mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->destroy_mtr_tbls(dev, fm); +} + +/** + * Destroy the global meter drop table. + * + * @param[in] dev + * Pointer to Ethernet device. + */ +void +mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->destroy_policer_rules(dev, fm, attr); + fops->destroy_mtr_drop_tbls(dev); +} + +/** + * Allocate the needed aso flow meter id. + * + * @param[in] dev + * Pointer to Ethernet device. + * + * @return + * Index to aso flow meter on success, NULL otherwise. + */ +uint32_t +mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_meter(dev); +} + +/** + * Free the aso flow meter id. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] mtr_idx + * Index to aso flow meter to be free. + * + * @return + * 0 on success. + */ +void +mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->free_meter(dev, mtr_idx); } /** @@ -7160,6 +7824,10 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; + uint32_t handle_idx; + int ret; + struct mlx5_flow_handle *dh; + struct rte_flow *flow; if (!priv->config.dv_flow_en) { if (fputs("device dv flow disabled\n", file) <= 0) @@ -7167,10 +7835,32 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, return -ENOTSUP; } + /* dump all */ if (!flow_idx) return mlx5_devx_cmd_flow_dump(sh->fdb_domain, - sh->rx_domain, sh->tx_domain, file); - return -ENOTSUP; + sh->rx_domain, + sh->tx_domain, file); + /* dump one */ + flow = mlx5_ipool_get(priv->sh->ipool + [MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx); + if (!flow) + return -ENOENT; + + handle_idx = flow->dev_handles; + while (handle_idx) { + dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], + handle_idx); + if (!dh) + return -ENOENT; + if (dh->drv_flow) { + ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, + file); + if (ret) + return -ENOENT; + } + handle_idx = dh->next.next; + } + return 0; } /** @@ -7213,12 +7903,12 @@ mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, /* Wrapper for driver action_validate op callback */ static int flow_drv_action_validate(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action *action, const struct mlx5_flow_driver_ops *fops, struct rte_flow_error *error) { - static const char err_msg[] = "shared action validation unsupported"; + static const char err_msg[] = "indirect action validation unsupported"; if (!fops->action_validate) { DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); @@ -7234,8 +7924,8 @@ flow_drv_action_validate(struct rte_eth_dev *dev, * * @param dev * Pointer to Ethernet device structure. - * @param[in] action - * Handle for the shared action to be destroyed. + * @param[in] handle + * Handle for the indirect action object to be destroyed. * @param[out] error * Perform verbose error reporting if not NULL. PMDs initialize this * structure in case of error only. @@ -7246,11 +7936,11 @@ flow_drv_action_validate(struct rte_eth_dev *dev, * @note: wrapper for driver action_create op callback. */ static int -mlx5_shared_action_destroy(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, +mlx5_action_handle_destroy(struct rte_eth_dev *dev, + struct rte_flow_action_handle *handle, struct rte_flow_error *error) { - static const char err_msg[] = "shared action destruction unsupported"; + static const char err_msg[] = "indirect action destruction unsupported"; struct rte_flow_attr attr = { .transfer = 0 }; const struct mlx5_flow_driver_ops *fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); @@ -7261,18 +7951,18 @@ mlx5_shared_action_destroy(struct rte_eth_dev *dev, NULL, err_msg); return -rte_errno; } - return fops->action_destroy(dev, action, error); + return fops->action_destroy(dev, handle, error); } /* Wrapper for driver action_destroy op callback */ static int flow_drv_action_update(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - const void *action_conf, + struct rte_flow_action_handle *handle, + const void *update, const struct mlx5_flow_driver_ops *fops, struct rte_flow_error *error) { - static const char err_msg[] = "shared action update unsupported"; + static const char err_msg[] = "indirect action update unsupported"; if (!fops->action_update) { DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); @@ -7280,18 +7970,18 @@ flow_drv_action_update(struct rte_eth_dev *dev, NULL, err_msg); return -rte_errno; } - return fops->action_update(dev, action, action_conf, error); + return fops->action_update(dev, handle, update, error); } /* Wrapper for driver action_destroy op callback */ static int flow_drv_action_query(struct rte_eth_dev *dev, - const struct rte_flow_shared_action *action, + const struct rte_flow_action_handle *handle, void *data, const struct mlx5_flow_driver_ops *fops, struct rte_flow_error *error) { - static const char err_msg[] = "shared action query unsupported"; + static const char err_msg[] = "indirect action query unsupported"; if (!fops->action_query) { DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); @@ -7299,29 +7989,31 @@ flow_drv_action_query(struct rte_eth_dev *dev, NULL, err_msg); return -rte_errno; } - return fops->action_query(dev, action, data, error); + return fops->action_query(dev, handle, data, error); } /** - * Create shared action for reuse in multiple flow rules. + * Create indirect action for reuse in multiple flow rules. * * @param dev * Pointer to Ethernet device structure. + * @param conf + * Pointer to indirect action object configuration. * @param[in] action - * Action configuration for shared action creation. + * Action configuration for indirect action object creation. * @param[out] error * Perform verbose error reporting if not NULL. PMDs initialize this * structure in case of error only. * @return * A valid handle in case of success, NULL otherwise and rte_errno is set. */ -static struct rte_flow_shared_action * -mlx5_shared_action_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, +static struct rte_flow_action_handle * +mlx5_action_handle_create(struct rte_eth_dev *dev, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action *action, struct rte_flow_error *error) { - static const char err_msg[] = "shared action creation unsupported"; + static const char err_msg[] = "indirect action creation unsupported"; struct rte_flow_attr attr = { .transfer = 0 }; const struct mlx5_flow_driver_ops *fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); @@ -7338,19 +8030,20 @@ mlx5_shared_action_create(struct rte_eth_dev *dev, } /** - * Updates inplace the shared action configuration pointed by *action* handle - * with the configuration provided as *action* argument. - * The update of the shared action configuration effects all flow rules reusing - * the action via handle. + * Updates inplace the indirect action configuration pointed by *handle* + * with the configuration provided as *update* argument. + * The update of the indirect action configuration effects all flow rules + * reusing the action via handle. * * @param dev * Pointer to Ethernet device structure. - * @param[in] shared_action - * Handle for the shared action to be updated. - * @param[in] action + * @param[in] handle + * Handle for the indirect action to be updated. + * @param[in] update * Action specification used to modify the action pointed by handle. - * *action* should be of same type with the action pointed by the *action* - * handle argument, otherwise considered as invalid. + * *update* could be of same type with the action pointed by the *handle* + * handle argument, or some other structures like a wrapper, depending on + * the indirect action type. * @param[out] error * Perform verbose error reporting if not NULL. PMDs initialize this * structure in case of error only. @@ -7359,9 +8052,9 @@ mlx5_shared_action_create(struct rte_eth_dev *dev, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -mlx5_shared_action_update(struct rte_eth_dev *dev, - struct rte_flow_shared_action *shared_action, - const struct rte_flow_action *action, +mlx5_action_handle_update(struct rte_eth_dev *dev, + struct rte_flow_action_handle *handle, + const void *update, struct rte_flow_error *error) { struct rte_flow_attr attr = { .transfer = 0 }; @@ -7369,26 +8062,27 @@ mlx5_shared_action_update(struct rte_eth_dev *dev, flow_get_drv_ops(flow_get_drv_type(dev, &attr)); int ret; - ret = flow_drv_action_validate(dev, NULL, action, fops, error); + ret = flow_drv_action_validate(dev, NULL, + (const struct rte_flow_action *)update, fops, error); if (ret) return ret; - return flow_drv_action_update(dev, shared_action, action->conf, fops, + return flow_drv_action_update(dev, handle, update, fops, error); } /** - * Query the shared action by handle. + * Query the indirect action by handle. * * This function allows retrieving action-specific data such as counters. * Data is gathered by special action which may be present/referenced in * more than one flow rule definition. * - * \see RTE_FLOW_ACTION_TYPE_COUNT + * see @RTE_FLOW_ACTION_TYPE_COUNT * * @param dev * Pointer to Ethernet device structure. - * @param[in] action - * Handle for the shared action to query. + * @param[in] handle + * Handle for the indirect action to query. * @param[in, out] data * Pointer to storage for the associated query data type. * @param[out] error @@ -7399,8 +8093,8 @@ mlx5_shared_action_update(struct rte_eth_dev *dev, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -mlx5_shared_action_query(struct rte_eth_dev *dev, - const struct rte_flow_shared_action *action, +mlx5_action_handle_query(struct rte_eth_dev *dev, + const struct rte_flow_action_handle *handle, void *data, struct rte_flow_error *error) { @@ -7408,11 +8102,11 @@ mlx5_shared_action_query(struct rte_eth_dev *dev, const struct mlx5_flow_driver_ops *fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); - return flow_drv_action_query(dev, action, data, fops, error); + return flow_drv_action_query(dev, handle, data, fops, error); } /** - * Destroy all shared actions. + * Destroy all indirect actions (shared RSS). * * @param dev * Pointer to Ethernet device. @@ -7421,7 +8115,7 @@ mlx5_shared_action_query(struct rte_eth_dev *dev, * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_shared_action_flush(struct rte_eth_dev *dev) +mlx5_action_handle_flush(struct rte_eth_dev *dev) { struct rte_flow_error error; struct mlx5_priv *priv = dev->data->dev_private; @@ -7431,8 +8125,8 @@ mlx5_shared_action_flush(struct rte_eth_dev *dev) ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], priv->rss_shared_actions, idx, shared_rss, next) { - ret |= mlx5_shared_action_destroy(dev, - (struct rte_flow_shared_action *)(uintptr_t)idx, &error); + ret |= mlx5_action_handle_destroy(dev, + (struct rte_flow_action_handle *)(uintptr_t)idx, &error); } return ret; } @@ -7605,10 +8299,12 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) union tunnel_offload_mark mbits = { .val = mark }; union mlx5_flow_tbl_key table_key = { { - .table_id = tunnel_id_to_flow_tbl(mbits.table_id), + .level = tunnel_id_to_flow_tbl(mbits.table_id), + .id = 0, + .reserved = 0, .dummy = 0, - .domain = !!mbits.transfer, - .direction = 0, + .is_fdb = !!mbits.transfer, + .is_egress = 0, } }; he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);