From: Dekel Peled Date: Wed, 11 Sep 2019 11:03:36 +0000 (+0300) Subject: net/mlx5: skip table zero to improve insertion rate X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=b67b4ecbde22;p=dpdk.git net/mlx5: skip table zero to improve insertion rate E-switch tables one and above provide higher insertion rate than table zero, as well as enhanced functionality. This patch adds a mechanism to utilize these advantages, by creating a default rule on port start, which directs all packets from e-switch table zero to table one. Other flow rules, requested for group n, will be created in e-switch table n+1. Jump action to e-switch group n will be created to group n+1. Utility function mlx5_flow_group_to_table() is added to translate the rte_flow group value to HW table value, and is called by PMD flow engine on flow rule validation and creation. Signed-off-by: Dekel Peled Acked-by: Matan Azrad Acked-by: Viacheslav Ovsiienko --- diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 239b56c9e0..7982490180 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -513,8 +513,8 @@ struct mlx5_flow_tbl_resource { rte_atomic32_t refcnt; /**< Reference counter. */ }; -#define MLX5_MAX_TABLES 0xffff -#define MLX5_MAX_TABLES_FDB 0xffff +#define MLX5_MAX_TABLES UINT16_MAX +#define MLX5_MAX_TABLES_FDB UINT16_MAX #define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */ #define MLX5_DBR_SIZE 8 @@ -829,6 +829,7 @@ int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, int mlx5_ctrl_flow(struct rte_eth_dev *dev, struct rte_flow_item_eth *eth_spec, struct rte_flow_item_eth *eth_mask); +struct rte_flow *mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev); int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev); void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev); void mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index eb360525da..482f65b5dd 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -2013,6 +2013,7 @@ flow_null_validate(struct rte_eth_dev *dev __rte_unused, const struct rte_flow_attr *attr __rte_unused, const struct rte_flow_item items[] __rte_unused, const struct rte_flow_action actions[] __rte_unused, + bool external __rte_unused, struct rte_flow_error *error) { return rte_flow_error_set(error, ENOTSUP, @@ -2125,6 +2126,8 @@ flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) * Pointer to the list of items. * @param[in] actions * Pointer to the list of actions. + * @param[in] external + * This flow rule is created by request external to PMD. * @param[out] error * Pointer to the error structure. * @@ -2136,13 +2139,13 @@ flow_drv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], - struct rte_flow_error *error) + bool external, struct rte_flow_error *error) { const struct mlx5_flow_driver_ops *fops; enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); fops = flow_get_drv_ops(type); - return fops->validate(dev, attr, items, actions, error); + return fops->validate(dev, attr, items, actions, external, error); } /** @@ -2314,7 +2317,7 @@ mlx5_flow_validate(struct rte_eth_dev *dev, { int ret; - ret = flow_drv_validate(dev, attr, items, actions, error); + ret = flow_drv_validate(dev, attr, items, actions, true, error); if (ret < 0) return ret; return 0; @@ -2376,6 +2379,8 @@ find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level) * Pattern specification (list terminated by the END pattern item). * @param[in] actions * Associated actions (list terminated by the END action). + * @param[in] external + * This flow rule is created by request external to PMD. * @param[out] error * Perform verbose error reporting if not NULL. * @@ -2387,7 +2392,7 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], - struct rte_flow_error *error) + bool external, struct rte_flow_error *error) { struct rte_flow *flow = NULL; struct mlx5_flow *dev_flow; @@ -2401,7 +2406,7 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, uint32_t i; uint32_t flow_size; - ret = flow_drv_validate(dev, attr, items, actions, error); + ret = flow_drv_validate(dev, attr, items, actions, external, error); if (ret < 0) return NULL; flow_size = sizeof(struct rte_flow); @@ -2443,6 +2448,7 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, if (!dev_flow) goto error; dev_flow->flow = flow; + dev_flow->external = external; LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); ret = flow_drv_translate(dev, dev_flow, attr, buf->entry[i].pattern, @@ -2467,6 +2473,55 @@ error: return NULL; } +/** + * Create a dedicated flow rule on e-switch table 0 (root table), to direct all + * incoming packets to table 1. + * + * Other flow rules, requested for group n, will be created in + * e-switch table n+1. + * Jump action to e-switch group n will be created to group n+1. + * + * Used when working in switchdev mode, to utilise advantages of table 1 + * and above. + * + * @param dev + * Pointer to Ethernet device. + * + * @return + * Pointer to flow on success, NULL otherwise and rte_errno is set. + */ +struct rte_flow * +mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) +{ + const struct rte_flow_attr attr = { + .group = 0, + .priority = 0, + .ingress = 1, + .egress = 0, + .transfer = 1, + }; + const struct rte_flow_item pattern = { + .type = RTE_FLOW_ITEM_TYPE_END, + }; + struct rte_flow_action_jump jump = { + .group = 1, + }; + const struct rte_flow_action actions[] = { + { + .type = RTE_FLOW_ACTION_TYPE_JUMP, + .conf = &jump, + }, + { + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; + struct mlx5_priv *priv = dev->data->dev_private; + struct rte_flow_error error; + + return flow_list_create(dev, &priv->ctrl_flows, &attr, &pattern, + actions, false, &error); +} + /** * Create a flow. * @@ -2483,7 +2538,7 @@ mlx5_flow_create(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; return flow_list_create(dev, &priv->flows, - attr, items, actions, error); + attr, items, actions, true, error); } /** @@ -2680,7 +2735,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, for (i = 0; i != priv->reta_idx_n; ++i) queue[i] = (*priv->reta_idx)[i]; flow = flow_list_create(dev, &priv->ctrl_flows, - &attr, items, actions, &error); + &attr, items, actions, false, &error); if (!flow) return -rte_errno; return 0; @@ -3094,7 +3149,8 @@ flow_fdir_filter_add(struct rte_eth_dev *dev, goto error; } flow = flow_list_create(dev, &priv->flows, &fdir_flow->attr, - fdir_flow->items, fdir_flow->actions, NULL); + fdir_flow->items, fdir_flow->actions, true, + NULL); if (!flow) goto error; assert(!flow->fdir); @@ -3442,3 +3498,39 @@ mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, pool->raw_hw = NULL; sh->cmng.pending_queries--; } + +/** + * Translate the rte_flow group index to HW table value. + * + * @param[in] attributes + * Pointer to flow attributes + * @param[in] external + * Value is part of flow rule created by request external to PMD. + * @param[in] group + * rte_flow group index value. + * @param[out] table + * HW table value. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, bool external, + uint32_t group, uint32_t *table, + struct rte_flow_error *error) +{ + if (attributes->transfer && external) { + if (group == UINT32_MAX) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, + "group index not supported"); + *table = group + 1; + } else { + *table = group; + } + return 0; +} diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 95abbb824d..235bccd143 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -391,6 +391,7 @@ struct mlx5_flow { #endif struct mlx5_flow_verbs verbs; }; + bool external; /**< true if the flow is created external to PMD. */ }; /* Flow structure. */ @@ -417,6 +418,7 @@ typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], + bool external, struct rte_flow_error *error); typedef struct mlx5_flow *(*mlx5_flow_prepare_t) (const struct rte_flow_attr *attr, const struct rte_flow_item items[], @@ -455,6 +457,9 @@ struct mlx5_flow_driver_ops { /* mlx5_flow.c */ +int mlx5_flow_group_to_table(const struct rte_flow_attr *attributes, + bool external, uint32_t group, uint32_t *table, + struct rte_flow_error *error); uint64_t mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, int tunnel, uint64_t layer_types, uint64_t hash_fields); diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 330a86812b..c234d13018 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -2366,8 +2366,10 @@ flow_dv_validate_action_modify_ttl(const uint64_t action_flags, * Pointer to the jump action. * @param[in] action_flags * Holds the actions detected until now. - * @param[in] group - * The group of the current flow. + * @param[in] attributes + * Pointer to flow attributes + * @param[in] external + * Action belongs to flow rule created by request external to PMD. * @param[out] error * Pointer to error structure. * @@ -2377,9 +2379,14 @@ flow_dv_validate_action_modify_ttl(const uint64_t action_flags, static int flow_dv_validate_action_jump(const struct rte_flow_action *action, uint64_t action_flags, - uint32_t group, - struct rte_flow_error *error) + const struct rte_flow_attr *attributes, + bool external, struct rte_flow_error *error) { + uint32_t max_group = attributes->transfer ? MLX5_MAX_TABLES_FDB : + MLX5_MAX_TABLES; + uint32_t target_group, table; + int ret = 0; + if (action_flags & (MLX5_FLOW_FATE_ACTIONS | MLX5_FLOW_FATE_ESWITCH_ACTIONS)) return rte_flow_error_set(error, EINVAL, @@ -2390,10 +2397,20 @@ flow_dv_validate_action_jump(const struct rte_flow_action *action, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, "action configuration not set"); - if (group >= ((const struct rte_flow_action_jump *)action->conf)->group) + target_group = + ((const struct rte_flow_action_jump *)action->conf)->group; + ret = mlx5_flow_group_to_table(attributes, external, target_group, + &table, error); + if (ret) + return ret; + if (table >= max_group) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, + "target group index out of range"); + if (attributes->group >= target_group) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "target group must be higher then" + "target group must be higher than" " the current flow group"); return 0; } @@ -3174,6 +3191,8 @@ flow_dv_counter_release(struct rte_eth_dev *dev, * Pointer to dev struct. * @param[in] attributes * Pointer to flow attributes + * @param[in] external + * This flow rule is created by request external to PMD. * @param[out] error * Pointer to error structure. * @@ -3183,6 +3202,7 @@ flow_dv_counter_release(struct rte_eth_dev *dev, static int flow_dv_validate_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attributes, + bool external __rte_unused, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; @@ -3193,7 +3213,22 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, - "groups is not supported"); + "groups are not supported"); +#else + uint32_t max_group = attributes->transfer ? MLX5_MAX_TABLES_FDB : + MLX5_MAX_TABLES; + uint32_t table; + int ret; + + ret = mlx5_flow_group_to_table(attributes, external, + attributes->group, + &table, error); + if (ret) + return ret; + if (table >= max_group) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, + "group index out of range"); #endif if (attributes->priority != MLX5_FLOW_PRIO_RSVD && attributes->priority >= priority_max) @@ -3217,12 +3252,6 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes, "egress is not supported"); - if (attributes->group >= MLX5_MAX_TABLES_FDB) - return rte_flow_error_set - (error, EINVAL, - RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, - NULL, "group must be smaller than " - RTE_STR(MLX5_MAX_TABLES_FDB)); } if (!(attributes->egress ^ attributes->ingress)) return rte_flow_error_set(error, ENOTSUP, @@ -3243,6 +3272,8 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, * Pointer to the list of items. * @param[in] actions * Pointer to the list of actions. + * @param[in] external + * This flow rule is created by request external to PMD. * @param[out] error * Pointer to the error structure. * @@ -3253,7 +3284,7 @@ static int flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], - struct rte_flow_error *error) + bool external, struct rte_flow_error *error) { int ret; uint64_t action_flags = 0; @@ -3272,7 +3303,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (items == NULL) return -1; - ret = flow_dv_validate_attributes(dev, attr, error); + ret = flow_dv_validate_attributes(dev, attr, external, error); if (ret < 0) return ret; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -3675,7 +3706,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case RTE_FLOW_ACTION_TYPE_JUMP: ret = flow_dv_validate_action_jump(actions, action_flags, - attr->group, error); + attr, external, + error); if (ret) return ret; ++actions_n; @@ -5142,8 +5174,14 @@ flow_dv_translate(struct rte_eth_dev *dev, struct rte_vlan_hdr vlan = { 0 }; bool vlan_inherited = false; uint16_t vlan_tci; + uint32_t table; + int ret = 0; - flow->group = attr->group; + ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group, + &table, error); + if (ret) + return ret; + flow->group = table; if (attr->transfer) res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; if (priority == MLX5_FLOW_PRIO_RSVD) @@ -5229,7 +5267,7 @@ flow_dv_translate(struct rte_eth_dev *dev, flow->counter = flow_dv_counter_alloc(dev, count->shared, count->id, - attr->group); + flow->group); if (flow->counter == NULL) goto cnt_err; dev_flow->dv.actions[actions_n++] = @@ -5371,7 +5409,12 @@ cnt_err: break; case RTE_FLOW_ACTION_TYPE_JUMP: jump_data = action->conf; - tbl = flow_dv_tbl_resource_get(dev, jump_data->group, + ret = mlx5_flow_group_to_table(attr, dev_flow->external, + jump_data->group, &table, + error); + if (ret) + return ret; + tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, attr->transfer, error); if (!tbl) @@ -5522,7 +5565,7 @@ cnt_err: mlx5_flow_tunnel_ip_check(items, next_protocol, &item_flags, &tunnel); flow_dv_translate_item_ipv4(match_mask, match_value, - items, tunnel, attr->group); + items, tunnel, flow->group); matcher.priority = MLX5_PRIORITY_MAP_L3; dev_flow->dv.hash_fields |= mlx5_flow_hashfields_adjust @@ -5549,7 +5592,7 @@ cnt_err: mlx5_flow_tunnel_ip_check(items, next_protocol, &item_flags, &tunnel); flow_dv_translate_item_ipv6(match_mask, match_value, - items, tunnel, attr->group); + items, tunnel, flow->group); matcher.priority = MLX5_PRIORITY_MAP_L3; dev_flow->dv.hash_fields |= mlx5_flow_hashfields_adjust @@ -5668,7 +5711,7 @@ cnt_err: matcher.priority = mlx5_flow_adjust_priority(dev, priority, matcher.priority); matcher.egress = attr->egress; - matcher.group = attr->group; + matcher.group = flow->group; matcher.transfer = attr->transfer; if (flow_dv_matcher_register(dev, &matcher, dev_flow, error)) return -rte_errno; diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index c5b28e36cb..23110f221f 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -1016,6 +1016,8 @@ flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, * Pointer to the list of items. * @param[in] actions * Pointer to the list of actions. + * @param[in] external + * This flow rule is created by request external to PMD. * @param[out] error * Pointer to the error structure. * @@ -1027,6 +1029,7 @@ flow_verbs_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], + bool external __rte_unused, struct rte_flow_error *error) { int ret; diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index aa323ad5ce..122f31c552 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -286,6 +286,9 @@ mlx5_traffic_enable(struct rte_eth_dev *dev) unsigned int j; int ret; + if (priv->config.dv_esw_en && !priv->config.vf) + if (!mlx5_flow_create_esw_table_zero_flow(dev)) + goto error; if (priv->isolated) return 0; if (dev->data->promiscuous) {