From: Bing Zhao Date: Tue, 24 Mar 2020 15:33:59 +0000 (+0000) Subject: net/mlx5: separate the flow handle resource X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=e7bfa3596a0a;p=dpdk.git net/mlx5: separate the flow handle resource Only the members of flow handle structure will be used when trying to destroy a flow. Other members of mlx5 device flow resource will only be used for flow creating, and they could be reused for different flows. So only the device flow handle structure needs to be saved for further usage. This could be separated from the whole mlx5 device flow and stored with a list for each rte flow. Other members will be pre-allocated with an array, and an index will be used to help to apply each device flow to the hardware. The flow handle sizes of Verbs and DV mode will be different, and some calculation could be done before allocating a verbs handle. Then the total memory consumption will less for Verbs when there is no inbox driver being used. Signed-off-by: Bing Zhao Acked-by: Matan Azrad --- diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 0613f7083a..8dda0c313f 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -1246,6 +1246,8 @@ mlx5_dev_close(struct rte_eth_dev *dev) */ mlx5_flow_list_flush(dev, &priv->flows, true); mlx5_flow_meter_flush(dev, NULL); + /* Free the intermediate buffers for flow creation. */ + mlx5_flow_free_intermediate(dev); /* Prevent crashes when queues are still in use. */ dev->rx_pkt_burst = removed_rx_burst; dev->tx_pkt_burst = removed_tx_burst; @@ -2768,6 +2770,11 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, err = ENOTSUP; goto error; } + /* + * Allocate the buffer for flow creating, just once. + * The allocation must be done before any flow creating. + */ + mlx5_flow_alloc_intermediate(eth_dev); /* Query availibility of metadata reg_c's. */ err = mlx5_flow_discover_mreg_c(eth_dev); if (err < 0) { diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index b24851d3c3..63d23cb1e6 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -517,6 +517,8 @@ struct mlx5_priv { struct mlx5_drop drop_queue; /* Flow drop queues. */ struct mlx5_flows flows; /* RTE Flow rules. */ struct mlx5_flows ctrl_flows; /* Control flow rules. */ + void *inter_flows; /* Intermediate resources for flow creation. */ + int flow_idx; /* Intermediate device flow index. */ LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */ LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */ LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */ @@ -728,6 +730,8 @@ int mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list); void mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list); int mlx5_flow_start_default(struct rte_eth_dev *dev); void mlx5_flow_stop_default(struct rte_eth_dev *dev); +void mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev); +void mlx5_flow_free_intermediate(struct rte_eth_dev *dev); int mlx5_flow_verify(struct rte_eth_dev *dev); int mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, uint32_t queue); int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index b2de4e6c32..f2d3730d6b 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -712,17 +712,19 @@ flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] dev_flow - * Pointer to device flow structure. + * @param[in] flow + * Pointer to flow structure. + * @param[in] dev_handle + * Pointer to device flow handle structure. */ static void -flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) +flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow, + struct mlx5_flow_handle *dev_handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct rte_flow *flow = dev_flow->flow; - const int mark = !!(dev_flow->handle.act_flags & + const int mark = !!(dev_handle->act_flags & (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); - const int tunnel = !!(dev_flow->handle.layers & MLX5_FLOW_LAYER_TUNNEL); + const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); unsigned int i; for (i = 0; i != flow->rss.queue_num; ++i) { @@ -751,7 +753,7 @@ flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) /* Increase the counter matching the flow. */ for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { if ((tunnels_info[j].tunnel & - dev_flow->handle.layers) == + dev_handle->layers) == tunnels_info[j].tunnel) { rxq_ctrl->flow_tunnels_n[j]++; break; @@ -773,10 +775,10 @@ flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) static void flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) - flow_drv_rxq_flags_set(dev, dev_flow); + LIST_FOREACH(dev_handle, &flow->dev_handles, next) + flow_drv_rxq_flags_set(dev, flow, dev_handle); } /** @@ -785,17 +787,19 @@ flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) * * @param dev * Pointer to Ethernet device. - * @param[in] dev_flow - * Pointer to the device flow. + * @param[in] flow + * Pointer to flow structure. + * @param[in] dev_handle + * Pointer to the device flow handle structure. */ static void -flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) +flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow, + struct mlx5_flow_handle *dev_handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct rte_flow *flow = dev_flow->flow; - const int mark = !!(dev_flow->handle.act_flags & + const int mark = !!(dev_handle->act_flags & (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); - const int tunnel = !!(dev_flow->handle.layers & MLX5_FLOW_LAYER_TUNNEL); + const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); unsigned int i; MLX5_ASSERT(dev->data->dev_started); @@ -820,7 +824,7 @@ flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) /* Decrease the counter matching the flow. */ for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { if ((tunnels_info[j].tunnel & - dev_flow->handle.layers) == + dev_handle->layers) == tunnels_info[j].tunnel) { rxq_ctrl->flow_tunnels_n[j]--; break; @@ -843,10 +847,10 @@ flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) static void flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) - flow_drv_rxq_flags_trim(dev, dev_flow); + LIST_FOREACH(dev_handle, &flow->dev_handles, next) + flow_drv_rxq_flags_trim(dev, flow, dev_handle); } /** @@ -2309,11 +2313,11 @@ static void flow_mreg_split_qrss_release(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) - if (dev_flow->handle.qrss_id) - flow_qrss_free_id(dev, dev_flow->handle.qrss_id); + LIST_FOREACH(dev_handle, &flow->dev_handles, next) + if (dev_handle->qrss_id) + flow_qrss_free_id(dev, dev_handle->qrss_id); } static int @@ -2329,7 +2333,8 @@ flow_null_validate(struct rte_eth_dev *dev __rte_unused, } static struct mlx5_flow * -flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, +flow_null_prepare(struct rte_eth_dev *dev __rte_unused, + const struct rte_flow_attr *attr __rte_unused, const struct rte_flow_item items[] __rte_unused, const struct rte_flow_action actions[] __rte_unused, struct rte_flow_error *error) @@ -2469,6 +2474,8 @@ flow_drv_validate(struct rte_eth_dev *dev, * setting backward reference to the flow should be done out of this function. * layers field is not filled either. * + * @param[in] dev + * Pointer to the dev structure. * @param[in] attr * Pointer to the flow attributes. * @param[in] items @@ -2482,7 +2489,8 @@ flow_drv_validate(struct rte_eth_dev *dev, * Pointer to device flow on success, otherwise NULL and rte_errno is set. */ static inline struct mlx5_flow * -flow_drv_prepare(const struct rte_flow *flow, +flow_drv_prepare(struct rte_eth_dev *dev, + const struct rte_flow *flow, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], @@ -2493,7 +2501,7 @@ flow_drv_prepare(const struct rte_flow *flow, MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); fops = flow_get_drv_ops(type); - return fops->prepare(attr, items, actions, error); + return fops->prepare(dev, attr, items, actions, error); } /** @@ -2701,17 +2709,17 @@ flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) * help to do the optimization work for source code. * If no decap actions, use the layers directly. */ - if (!(dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DECAP)) - return dev_flow->handle.layers; + if (!(dev_flow->handle->act_flags & MLX5_FLOW_ACTION_DECAP)) + return dev_flow->handle->layers; /* Convert L3 layers with decap action. */ - if (dev_flow->handle.layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) + if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; - else if (dev_flow->handle.layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) + else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; /* Convert L4 layers with decap action. */ - if (dev_flow->handle.layers & MLX5_FLOW_LAYER_INNER_L4_TCP) + if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; - else if (dev_flow->handle.layers & MLX5_FLOW_LAYER_INNER_L4_UDP) + else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; return layers; } @@ -3412,7 +3420,7 @@ flow_hairpin_split(struct rte_eth_dev *dev, * The last stage of splitting chain, just creates the subflow * without any modification. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. * @param[in] flow * Parent flow structure pointer. @@ -3445,19 +3453,19 @@ flow_create_split_inner(struct rte_eth_dev *dev, { struct mlx5_flow *dev_flow; - dev_flow = flow_drv_prepare(flow, attr, items, actions, error); + dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, error); if (!dev_flow) return -rte_errno; dev_flow->flow = flow; dev_flow->external = external; /* Subflow object was created, we must include one in the list. */ - LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); + LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next); /* * If dev_flow is as one of the suffix flow, some actions in suffix * flow may need some user defined item layer flags. */ if (prefix_layers) - dev_flow->handle.layers = prefix_layers; + dev_flow->handle->layers = prefix_layers; if (sub_flow) *sub_flow = dev_flow; return flow_drv_translate(dev, dev_flow, attr, items, actions, error); @@ -3972,7 +3980,7 @@ flow_create_split_metadata(struct rte_eth_dev *dev, * reallocation becomes possible (for example, for * other flows in other threads). */ - dev_flow->handle.qrss_id = qrss_id; + dev_flow->handle->qrss_id = qrss_id; ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); if (ret < 0) @@ -4085,7 +4093,7 @@ flow_create_split_meter(struct rte_eth_dev *dev, ret = -rte_errno; goto exit; } - dev_flow->handle.mtr_flow_id = mtr_tag_id; + dev_flow->handle->mtr_flow_id = mtr_tag_id; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -4256,7 +4264,7 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; } - LIST_INIT(&flow->dev_flows); + LIST_INIT(&flow->dev_handles); if (rss && rss->types) { unsigned int graph_root; @@ -4271,6 +4279,8 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, buf->entries = 1; buf->entry[0].pattern = (void *)(uintptr_t)items; } + /* Reset device flow index to 0. */ + priv->flow_idx = 0; for (i = 0; i < buf->entries; ++i) { /* * The splitter may create multiple dev_flows, @@ -4289,13 +4299,13 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, attr_tx.group = MLX5_HAIRPIN_TX_TABLE; attr_tx.ingress = 0; attr_tx.egress = 1; - dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items, + dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, actions_hairpin_tx.actions, error); if (!dev_flow) goto error; dev_flow->flow = flow; dev_flow->external = 0; - LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); + LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next); ret = flow_drv_translate(dev, dev_flow, &attr_tx, items_tx.items, actions_hairpin_tx.actions, error); @@ -4543,8 +4553,6 @@ error: * * @param dev * Pointer to Ethernet device. - * @param list - * Pointer to a TAILQ flow list. */ void mlx5_flow_stop_default(struct rte_eth_dev *dev) @@ -4569,6 +4577,37 @@ mlx5_flow_start_default(struct rte_eth_dev *dev) return flow_mreg_add_default_copy_action(dev, &error); } +/** + * Allocate intermediate resources for flow creation. + * + * @param dev + * Pointer to Ethernet device. + */ +void +mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + if (!priv->inter_flows) + priv->inter_flows = rte_calloc(__func__, MLX5_NUM_MAX_DEV_FLOWS, + sizeof(struct mlx5_flow), 0); +} + +/** + * Free intermediate resources for flows. + * + * @param dev + * Pointer to Ethernet device. + */ +void +mlx5_flow_free_intermediate(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + rte_free(priv->inter_flows); + priv->inter_flows = NULL; +} + /** * Verify the flow list is empty * diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index f3aea53e1d..0f0e59dd23 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -500,6 +500,8 @@ struct mlx5_flow_handle_dv { /** Device flow handle structure: used both for creating & destroying. */ struct mlx5_flow_handle { + LIST_ENTRY(mlx5_flow_handle) next; + /**< Pointer to next device flow handle. */ uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ uint64_t act_flags; @@ -516,6 +518,18 @@ struct mlx5_flow_handle { #endif }; +/* + * Size for Verbs device flow handle structure only. Do not use the DV only + * structure in Verbs. No DV flows attributes will be accessed. + * Macro offsetof() could also be used here. + */ +#ifdef HAVE_IBV_FLOW_DV_SUPPORT +#define MLX5_FLOW_HANDLE_VERBS_SIZE \ + (sizeof(struct mlx5_flow_handle) - sizeof(struct mlx5_flow_handle_dv)) +#else +#define MLX5_FLOW_HANDLE_VERBS_SIZE (sizeof(struct mlx5_flow_handle)) +#endif + /* * Max number of actions per DV flow. * See CREATE_FLOW_MAX_FLOW_ACTIONS_SUPPORTED @@ -524,7 +538,7 @@ struct mlx5_flow_handle { #define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8 /** Device flow structure only for DV flow creation. */ -struct mlx5_flow_resource_dv { +struct mlx5_flow_dv_workspace { uint32_t group; /**< The group index. */ uint8_t transfer; /**< 1 if the flow is E-Switch flow. */ int actions_n; /**< number of actions. */ @@ -533,27 +547,79 @@ struct mlx5_flow_resource_dv { /**< Holds the value that the packet is compared to. */ }; +/* + * Maximal Verbs flow specifications & actions size. + * Some elements are mutually exclusive, but enough space should be allocated. + * Tunnel cases: 1. Max 2 Ethernet + IP(v6 len > v4 len) + TCP/UDP headers. + * 2. One tunnel header (exception: GRE + MPLS), + * SPEC length: GRE == tunnel. + * Actions: 1. 1 Mark OR Flag. + * 2. 1 Drop (if any). + * 3. No limitation for counters, but it makes no sense to support too + * many counters in a single device flow. + */ +#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT +#define MLX5_VERBS_MAX_SPEC_SIZE \ + ( \ + (2 * (sizeof(struct ibv_flow_spec_eth) + \ + sizeof(struct ibv_flow_spec_ipv6) + \ + sizeof(struct ibv_flow_spec_tcp_udp)) + \ + sizeof(struct ibv_flow_spec_gre) + \ + sizeof(struct ibv_flow_spec_mpls)) \ + ) +#else +#define MLX5_VERBS_MAX_SPEC_SIZE \ + ( \ + (2 * (sizeof(struct ibv_flow_spec_eth) + \ + sizeof(struct ibv_flow_spec_ipv6) + \ + sizeof(struct ibv_flow_spec_tcp_udp)) + \ + sizeof(struct ibv_flow_spec_tunnel)) \ + ) +#endif + +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \ + defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) +#define MLX5_VERBS_MAX_ACT_SIZE \ + ( \ + sizeof(struct ibv_flow_spec_action_tag) + \ + sizeof(struct ibv_flow_spec_action_drop) + \ + sizeof(struct ibv_flow_spec_counter_action) * 4 \ + ) +#else +#define MLX5_VERBS_MAX_ACT_SIZE \ + ( \ + sizeof(struct ibv_flow_spec_action_tag) + \ + sizeof(struct ibv_flow_spec_action_drop) \ + ) +#endif + +#define MLX5_VERBS_MAX_SPEC_ACT_SIZE \ + (MLX5_VERBS_MAX_SPEC_SIZE + MLX5_VERBS_MAX_ACT_SIZE) + /** Device flow structure only for Verbs flow creation. */ -struct mlx5_flow_resource_verbs { +struct mlx5_flow_verbs_workspace { unsigned int size; /**< Size of the attribute. */ - struct ibv_flow_attr *attr; /**< Pointer to the Specification buffer. */ - uint8_t *specs; /**< Pointer to the specifications. */ + struct ibv_flow_attr attr; /**< Verbs flow attribute buffer. */ + uint8_t specs[MLX5_VERBS_MAX_SPEC_ACT_SIZE]; + /**< Specifications & actions buffer of verbs flow. */ }; +/** Maximal number of device sub-flows supported. */ +#define MLX5_NUM_MAX_DEV_FLOWS 32 + /** Device flow structure. */ struct mlx5_flow { - LIST_ENTRY(mlx5_flow) next; /**< Pointer to next device flow. */ struct rte_flow *flow; /**< Pointer to the main flow. */ uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */ bool external; /**< true if the flow is created external to PMD. */ uint8_t ingress; /**< 1 if the flow is ingress. */ union { #ifdef HAVE_IBV_FLOW_DV_SUPPORT - struct mlx5_flow_resource_dv dv; + struct mlx5_flow_dv_workspace dv; #endif - struct mlx5_flow_resource_verbs verbs; + struct mlx5_flow_verbs_workspace verbs; }; - struct mlx5_flow_handle handle; + struct mlx5_flow_handle *handle; }; /* Flow meter state. */ @@ -667,8 +733,8 @@ struct rte_flow { struct mlx5_flow_mreg_copy_resource *mreg_copy; /**< pointer to metadata register copy table resource. */ struct mlx5_flow_meter *meter; /**< Holds flow meter. */ - LIST_HEAD(dev_flows, mlx5_flow) dev_flows; - /**< Device flows that are part of the flow. */ + LIST_HEAD(dev_handles, mlx5_flow_handle) dev_handles; + /**< Device flow handles that are part of the flow. */ struct mlx5_fdir *fdir; /**< Pointer to associated FDIR if any. */ uint32_t hairpin_flow_id; /**< The flow id used for hairpin. */ uint32_t copy_applied:1; /**< The MARK copy Flow os applied. */ @@ -681,7 +747,8 @@ typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev, bool external, struct rte_flow_error *error); typedef struct mlx5_flow *(*mlx5_flow_prepare_t) - (const struct rte_flow_attr *attr, const struct rte_flow_item items[], + (struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + const struct rte_flow_item items[], const struct rte_flow_action actions[], struct rte_flow_error *error); typedef int (*mlx5_flow_translate_t)(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index f7dfeff724..f5d98d267b 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -92,7 +92,7 @@ static void flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr, struct mlx5_flow *dev_flow, bool tunnel_decap) { - uint64_t layers = dev_flow->handle.layers; + uint64_t layers = dev_flow->handle->layers; /* * If layers is already initialized, it means this dev_flow is the @@ -2458,7 +2458,7 @@ flow_dv_encap_decap_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle.dvh.encap_decap = cache_resource; + dev_flow->handle->dvh.encap_decap = cache_resource; return 0; } } @@ -2484,7 +2484,7 @@ flow_dv_encap_decap_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next); - dev_flow->handle.dvh.encap_decap = cache_resource; + dev_flow->handle->dvh.encap_decap = cache_resource; DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -2535,7 +2535,7 @@ flow_dv_jump_tbl_resource_register (void *)&tbl_data->jump, cnt); } rte_atomic32_inc(&tbl_data->jump.refcnt); - dev_flow->handle.dvh.jump = &tbl_data->jump; + dev_flow->handle->dvh.jump = &tbl_data->jump; return 0; } @@ -2573,7 +2573,7 @@ flow_dv_port_id_action_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle.dvh.port_id_action = cache_resource; + dev_flow->handle->dvh.port_id_action = cache_resource; return 0; } } @@ -2601,7 +2601,7 @@ flow_dv_port_id_action_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next); - dev_flow->handle.dvh.port_id_action = cache_resource; + dev_flow->handle->dvh.port_id_action = cache_resource; DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -2644,7 +2644,7 @@ flow_dv_push_vlan_action_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle.dvh.push_vlan_res = cache_resource; + dev_flow->handle->dvh.push_vlan_res = cache_resource; return 0; } } @@ -2673,7 +2673,7 @@ flow_dv_push_vlan_action_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next); - dev_flow->handle.dvh.push_vlan_res = cache_resource; + dev_flow->handle->dvh.push_vlan_res = cache_resource; DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -3786,7 +3786,7 @@ flow_dv_modify_hdr_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle.dvh.modify_hdr = cache_resource; + dev_flow->handle->dvh.modify_hdr = cache_resource; return 0; } } @@ -3813,7 +3813,7 @@ flow_dv_modify_hdr_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next); - dev_flow->handle.dvh.modify_hdr = cache_resource; + dev_flow->handle->dvh.modify_hdr = cache_resource; DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -5271,6 +5271,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, * Internal preparation function. Allocates the DV flow size, * this size is constant. * + * @param[in] dev + * Pointer to the rte_eth_dev structure. * @param[in] attr * Pointer to the flow attributes. * @param[in] items @@ -5285,22 +5287,41 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, * otherwise NULL and rte_errno is set. */ static struct mlx5_flow * -flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused, +flow_dv_prepare(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr __rte_unused, const struct rte_flow_item items[] __rte_unused, const struct rte_flow_action actions[] __rte_unused, struct rte_flow_error *error) { - size_t size = sizeof(struct mlx5_flow); + size_t size = sizeof(struct mlx5_flow_handle); struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; + struct mlx5_priv *priv = dev->data->dev_private; - dev_flow = rte_calloc(__func__, 1, size, 0); - if (!dev_flow) { + /* In case of corrupting the memory. */ + if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { + rte_flow_error_set(error, ENOSPC, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "not free temporary device flow"); + return NULL; + } + dev_handle = rte_calloc(__func__, 1, size, 0); + if (!dev_handle) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "not enough memory to create flow"); + "not enough memory to create flow handle"); return NULL; } + /* No multi-thread supporting. */ + dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; + dev_flow->handle = dev_handle; dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); + /* + * The matching value needs to be cleared to 0 before using. In the + * past, it will be automatically cleared when using rte_*alloc + * API. The time consumption will be almost the same as before. + */ + memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param)); dev_flow->ingress = attr->ingress; dev_flow->dv.transfer = attr->transfer; return dev_flow; @@ -5458,7 +5479,7 @@ flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, * This is workaround, masks are not supported, * and pre-validated. */ - dev_flow->handle.vf_vlan.tag = + dev_flow->handle->vf_vlan.tag = rte_be_to_cpu_16(vlan_v->tci) & 0x0fff; } tci_m = rte_be_to_cpu_16(vlan_m->tci); @@ -6959,7 +6980,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, (void *)cache_matcher, rte_atomic32_read(&cache_matcher->refcnt)); rte_atomic32_inc(&cache_matcher->refcnt); - dev_flow->handle.dvh.matcher = cache_matcher; + dev_flow->handle->dvh.matcher = cache_matcher; /* old matcher should not make the table ref++. */ flow_dv_tbl_resource_release(dev, tbl); return 0; @@ -6996,7 +7017,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, /* only matcher ref++, table ref++ already done above in get API. */ rte_atomic32_inc(&cache_matcher->refcnt); LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); - dev_flow->handle.dvh.matcher = cache_matcher; + dev_flow->handle->dvh.matcher = cache_matcher; DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", key->domain ? "FDB" : "NIC", key->table_id, cache_matcher->priority, @@ -7038,7 +7059,7 @@ flow_dv_tag_resource_register cache_resource = container_of (entry, struct mlx5_flow_dv_tag_resource, entry); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle.dvh.tag_resource = cache_resource; + dev_flow->handle->dvh.tag_resource = cache_resource; DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -7067,7 +7088,7 @@ flow_dv_tag_resource_register RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot insert tag"); } - dev_flow->handle.dvh.tag_resource = cache_resource; + dev_flow->handle->dvh.tag_resource = cache_resource; DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -7212,7 +7233,7 @@ static void flow_dv_hashfields_set(struct mlx5_flow *dev_flow) { struct rte_flow *flow = dev_flow->flow; - uint64_t items = dev_flow->handle.layers; + uint64_t items = dev_flow->handle->layers; int rss_inner = 0; uint64_t rss_types = rte_eth_rss_hf_refine(flow->rss.types); @@ -7302,6 +7323,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_config *dev_conf = &priv->config; struct rte_flow *flow = dev_flow->flow; + struct mlx5_flow_handle *handle = dev_flow->handle; uint64_t item_flags = 0; uint64_t last_item = 0; uint64_t action_flags = 0; @@ -7370,7 +7392,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, (dev, &port_id_resource, dev_flow, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.port_id_action->action; + handle->dvh.port_id_action->action; action_flags |= MLX5_FLOW_ACTION_PORT_ID; break; case RTE_FLOW_ACTION_TYPE_FLAG: @@ -7388,12 +7410,17 @@ __flow_dv_translate(struct rte_eth_dev *dev, break; } tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); - if (!dev_flow->handle.dvh.tag_resource) - if (flow_dv_tag_resource_register - (dev, tag_be, dev_flow, error)) - return -rte_errno; + /* + * Only one FLAG or MARK is supported per device flow + * right now. So the pointer to the tag resource must be + * zero before the register process. + */ + MLX5_ASSERT(!handle->dvh.tag_resource); + if (flow_dv_tag_resource_register(dev, tag_be, + dev_flow, error)) + return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.tag_resource->action; + handle->dvh.tag_resource->action; break; case RTE_FLOW_ACTION_TYPE_MARK: action_flags |= MLX5_FLOW_ACTION_MARK; @@ -7415,12 +7442,12 @@ __flow_dv_translate(struct rte_eth_dev *dev, tag_be = mlx5_flow_mark_set (((const struct rte_flow_action_mark *) (actions->conf))->id); - if (!dev_flow->handle.dvh.tag_resource) - if (flow_dv_tag_resource_register - (dev, tag_be, dev_flow, error)) - return -rte_errno; + MLX5_ASSERT(!handle->dvh.tag_resource); + if (flow_dv_tag_resource_register(dev, tag_be, + dev_flow, error)) + return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.tag_resource->action; + handle->dvh.tag_resource->action; break; case RTE_FLOW_ACTION_TYPE_SET_META: if (flow_dv_convert_action_set_meta @@ -7518,7 +7545,7 @@ cnt_err: (dev, attr, &vlan, dev_flow, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.push_vlan_res->action; + handle->dvh.push_vlan_res->action; action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; break; case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: @@ -7545,7 +7572,7 @@ cnt_err: error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.encap_decap->verbs_action; + handle->dvh.encap_decap->verbs_action; action_flags |= MLX5_FLOW_ACTION_ENCAP; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -7555,7 +7582,7 @@ cnt_err: error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.encap_decap->verbs_action; + handle->dvh.encap_decap->verbs_action; action_flags |= MLX5_FLOW_ACTION_DECAP; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: @@ -7565,7 +7592,7 @@ cnt_err: (dev, actions, dev_flow, attr, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.encap_decap->verbs_action; + handle->dvh.encap_decap->verbs_action; } else { /* Handle encap without preceding decap. */ if (flow_dv_create_action_l2_encap @@ -7573,7 +7600,7 @@ cnt_err: error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.encap_decap->verbs_action; + handle->dvh.encap_decap->verbs_action; } action_flags |= MLX5_FLOW_ACTION_ENCAP; break; @@ -7585,7 +7612,7 @@ cnt_err: (dev, dev_flow, attr->transfer, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.encap_decap->verbs_action; + handle->dvh.encap_decap->verbs_action; } /* If decap is followed by encap, handle it at encap. */ action_flags |= MLX5_FLOW_ACTION_DECAP; @@ -7617,7 +7644,7 @@ cnt_err: "cannot create jump action."); } dev_flow->dv.actions[actions_n++] = - dev_flow->handle.dvh.jump->action; + handle->dvh.jump->action; action_flags |= MLX5_FLOW_ACTION_JUMP; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: @@ -7750,7 +7777,7 @@ cnt_err: (dev, mhdr_res, dev_flow, error)) return -rte_errno; dev_flow->dv.actions[modify_action_position] = - dev_flow->handle.dvh.modify_hdr->verbs_action; + handle->dvh.modify_hdr->verbs_action; } break; default: @@ -7761,7 +7788,7 @@ cnt_err: modify_action_position = actions_n++; } dev_flow->dv.actions_n = actions_n; - dev_flow->handle.act_flags = action_flags; + handle->act_flags = action_flags; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); int item_type = items->type; @@ -7966,7 +7993,7 @@ cnt_err: * Layers may be already initialized from prefix flow if this dev_flow * is the suffix flow. */ - dev_flow->handle.layers |= item_flags; + handle->layers |= item_flags; if (action_flags & MLX5_FLOW_ACTION_RSS) flow_dv_hashfields_set(dev_flow); /* Register matcher. */ @@ -8001,19 +8028,21 @@ static int __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, struct rte_flow_error *error) { - struct mlx5_flow_resource_dv *dv; + struct mlx5_flow_dv_workspace *dv; struct mlx5_flow_handle *dh; struct mlx5_flow_handle_dv *dv_h; struct mlx5_flow *dev_flow; struct mlx5_priv *priv = dev->data->dev_private; int n; int err; + int idx; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { - dh = &dev_flow->handle; + for (idx = priv->flow_idx - 1; idx >= 0; idx--) { + dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; dv = &dev_flow->dv; - n = dv->actions_n; + dh = dev_flow->handle; dv_h = &dh->dvh; + n = dv->actions_n; if (dh->act_flags & MLX5_FLOW_ACTION_DROP) { if (dv->transfer) { dv->actions[n++] = priv->sh->esw_drop_action; @@ -8046,7 +8075,7 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, dev_flow->hash_fields, (*flow->rss.queue), flow->rss.queue_num, - !!(dev_flow->handle.layers & + !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL)); } if (!hrxq) { @@ -8084,17 +8113,16 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, return 0; error: err = rte_errno; /* Save rte_errno before cleanup. */ - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { - struct mlx5_flow_handle *dh_tmp = &dev_flow->handle; - if (dh_tmp->hrxq) { - if (dh_tmp->act_flags & MLX5_FLOW_ACTION_DROP) + LIST_FOREACH(dh, &flow->dev_handles, next) { + if (dh->hrxq) { + if (dh->act_flags & MLX5_FLOW_ACTION_DROP) mlx5_hrxq_drop_release(dev); else - mlx5_hrxq_release(dev, dh_tmp->hrxq); - dh_tmp->hrxq = NULL; + mlx5_hrxq_release(dev, dh->hrxq); + dh->hrxq = NULL; } - if (dh_tmp->vf_vlan.tag && dh_tmp->vf_vlan.created) - mlx5_vlan_vmwa_release(dev, &dh_tmp->vf_vlan); + if (dh->vf_vlan.tag && dh->vf_vlan.created) + mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); } rte_errno = err; /* Restore rte_errno. */ return -rte_errno; @@ -8105,17 +8133,17 @@ error: * * @param dev * Pointer to Ethernet device. - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int flow_dv_matcher_release(struct rte_eth_dev *dev, - struct mlx5_flow *flow) + struct mlx5_flow_handle *handle) { - struct mlx5_flow_dv_matcher *matcher = flow->handle.dvh.matcher; + struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher; MLX5_ASSERT(matcher->matcher_object); DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", @@ -8138,17 +8166,17 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, /** * Release an encap/decap resource. * - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) +flow_dv_encap_decap_resource_release(struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_encap_decap_resource *cache_resource = - flow->handle.dvh.encap_decap; + handle->dvh.encap_decap; MLX5_ASSERT(cache_resource->verbs_action); DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", @@ -8171,18 +8199,18 @@ flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) * * @param dev * Pointer to Ethernet device. - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, - struct mlx5_flow *flow) + struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_jump_tbl_resource *cache_resource = - flow->handle.dvh.jump; + handle->dvh.jump; struct mlx5_flow_tbl_data_entry *tbl_data = container_of(cache_resource, struct mlx5_flow_tbl_data_entry, jump); @@ -8206,17 +8234,17 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, /** * Release a modify-header resource. * - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow) +flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_modify_hdr_resource *cache_resource = - flow->handle.dvh.modify_hdr; + handle->dvh.modify_hdr; MLX5_ASSERT(cache_resource->verbs_action); DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--", @@ -8237,17 +8265,17 @@ flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow) /** * Release port ID action resource. * - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_port_id_action_resource_release(struct mlx5_flow *flow) +flow_dv_port_id_action_resource_release(struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_port_id_action_resource *cache_resource = - flow->handle.dvh.port_id_action; + handle->dvh.port_id_action; MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", @@ -8268,17 +8296,17 @@ flow_dv_port_id_action_resource_release(struct mlx5_flow *flow) /** * Release push vlan action resource. * - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow) +flow_dv_push_vlan_action_resource_release(struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_push_vlan_action_resource *cache_resource = - flow->handle.dvh.push_vlan_res; + handle->dvh.push_vlan_res; MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--", @@ -8309,18 +8337,16 @@ static void __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dh; - struct mlx5_flow *dev_flow; if (!flow) return; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { - dh = &dev_flow->handle; + LIST_FOREACH(dh, &flow->dev_handles, next) { if (dh->ib_flow) { claim_zero(mlx5_glue->dv_destroy_flow(dh->ib_flow)); dh->ib_flow = NULL; } if (dh->hrxq) { - if (dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DROP) + if (dh->act_flags & MLX5_FLOW_ACTION_DROP) mlx5_hrxq_drop_release(dev); else mlx5_hrxq_release(dev, dh->hrxq); @@ -8343,7 +8369,7 @@ __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) static void __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; if (!flow) return; @@ -8356,25 +8382,25 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) mlx5_flow_meter_detach(flow->meter); flow->meter = NULL; } - while (!LIST_EMPTY(&flow->dev_flows)) { - dev_flow = LIST_FIRST(&flow->dev_flows); - LIST_REMOVE(dev_flow, next); - if (dev_flow->handle.dvh.matcher) - flow_dv_matcher_release(dev, dev_flow); - if (dev_flow->handle.dvh.encap_decap) - flow_dv_encap_decap_resource_release(dev_flow); - if (dev_flow->handle.dvh.modify_hdr) - flow_dv_modify_hdr_resource_release(dev_flow); - if (dev_flow->handle.dvh.jump) - flow_dv_jump_tbl_resource_release(dev, dev_flow); - if (dev_flow->handle.dvh.port_id_action) - flow_dv_port_id_action_resource_release(dev_flow); - if (dev_flow->handle.dvh.push_vlan_res) - flow_dv_push_vlan_action_resource_release(dev_flow); - if (dev_flow->handle.dvh.tag_resource) + while (!LIST_EMPTY(&flow->dev_handles)) { + dev_handle = LIST_FIRST(&flow->dev_handles); + LIST_REMOVE(dev_handle, next); + if (dev_handle->dvh.matcher) + flow_dv_matcher_release(dev, dev_handle); + if (dev_handle->dvh.encap_decap) + flow_dv_encap_decap_resource_release(dev_handle); + if (dev_handle->dvh.modify_hdr) + flow_dv_modify_hdr_resource_release(dev_handle); + if (dev_handle->dvh.jump) + flow_dv_jump_tbl_resource_release(dev, dev_handle); + if (dev_handle->dvh.port_id_action) + flow_dv_port_id_action_resource_release(dev_handle); + if (dev_handle->dvh.push_vlan_res) + flow_dv_push_vlan_action_resource_release(dev_handle); + if (dev_handle->dvh.tag_resource) flow_dv_tag_release(dev, - dev_flow->handle.dvh.tag_resource); - rte_free(dev_flow); + dev_handle->dvh.tag_resource); + rte_free(dev_handle); } } diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index 08185ecb04..ccd33955c7 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -253,7 +253,7 @@ flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, * Size in bytes of the specification to copy. */ static void -flow_verbs_spec_add(struct mlx5_flow_resource_verbs *verbs, +flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs, void *src, unsigned int size) { void *dst; @@ -263,7 +263,7 @@ flow_verbs_spec_add(struct mlx5_flow_resource_verbs *verbs, MLX5_ASSERT(verbs->specs); dst = (void *)(verbs->specs + verbs->size); memcpy(dst, src, size); - ++verbs->attr->num_of_specs; + ++verbs->attr.num_of_specs; verbs->size += size; } @@ -392,9 +392,9 @@ flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow, if (!(item_flags & l2m)) flow_verbs_spec_add(&dev_flow->verbs, ð, size); else - flow_verbs_item_vlan_update(dev_flow->verbs.attr, ð); + flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð); if (!tunnel) - dev_flow->handle.vf_vlan.tag = + dev_flow->handle->vf_vlan.tag = rte_be_to_cpu_16(spec->tci) & 0x0fff; } @@ -744,7 +744,7 @@ flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow, const struct rte_flow_item *item __rte_unused, uint64_t item_flags) { - struct mlx5_flow_resource_verbs *verbs = &dev_flow->verbs; + struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs; #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT unsigned int size = sizeof(struct ibv_flow_spec_tunnel); struct ibv_flow_spec_tunnel tunnel = { @@ -774,11 +774,11 @@ flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow, } #endif if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) - flow_verbs_item_gre_ip_protocol_update(verbs->attr, + flow_verbs_item_gre_ip_protocol_update(&verbs->attr, IBV_FLOW_SPEC_IPV4_EXT, IPPROTO_GRE); else - flow_verbs_item_gre_ip_protocol_update(verbs->attr, + flow_verbs_item_gre_ip_protocol_update(&verbs->attr, IBV_FLOW_SPEC_IPV6, IPPROTO_GRE); flow_verbs_spec_add(verbs, &tunnel, size); @@ -1385,6 +1385,8 @@ flow_verbs_get_items_size(const struct rte_flow_item items[]) * The required size is calculate based on the actions and items. This function * also returns the detected actions and items for later use. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] attr * Pointer to the flow attributes. * @param[in] items @@ -1399,25 +1401,45 @@ flow_verbs_get_items_size(const struct rte_flow_item items[]) * is set. */ static struct mlx5_flow * -flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused, +flow_verbs_prepare(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr __rte_unused, const struct rte_flow_item items[], const struct rte_flow_action actions[], struct rte_flow_error *error) { - size_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr); + size_t size = 0; struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; + struct mlx5_priv *priv = dev->data->dev_private; size += flow_verbs_get_actions_size(actions); size += flow_verbs_get_items_size(items); - dev_flow = rte_calloc(__func__, 1, size, 0); - if (!dev_flow) { + if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) { + rte_flow_error_set(error, E2BIG, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Verbs spec/action size too large"); + return NULL; + } + /* In case of corrupting the memory. */ + if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { + rte_flow_error_set(error, ENOSPC, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "not free temporary device flow"); + return NULL; + } + dev_handle = rte_calloc(__func__, 1, MLX5_FLOW_HANDLE_VERBS_SIZE, 0); + if (!dev_handle) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "not enough memory to create flow"); + "not enough memory to create flow handle"); return NULL; } - dev_flow->verbs.attr = (void *)(dev_flow + 1); - dev_flow->verbs.specs = (void *)(dev_flow->verbs.attr + 1); + /* No multi-thread supporting. */ + dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; + dev_flow->handle = dev_handle; + /* Memcpy is used, only size needs to be cleared to 0. */ + dev_flow->verbs.size = 0; + dev_flow->verbs.attr.num_of_specs = 0; dev_flow->ingress = attr->ingress; /* Need to set transfer attribute: not supported in Verbs mode. */ return dev_flow; @@ -1499,7 +1521,7 @@ flow_verbs_translate(struct rte_eth_dev *dev, "action not supported"); } } - dev_flow->handle.act_flags = action_flags; + dev_flow->handle->act_flags = action_flags; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); @@ -1601,10 +1623,11 @@ flow_verbs_translate(struct rte_eth_dev *dev, "item not supported"); } } - dev_flow->handle.layers = item_flags; - dev_flow->verbs.attr->priority = + dev_flow->handle->layers = item_flags; + /* Other members of attr will be ignored. */ + dev_flow->verbs.attr.priority = mlx5_flow_adjust_priority(dev, priority, subpriority); - dev_flow->verbs.attr->port = (uint8_t)priv->ibv_port; + dev_flow->verbs.attr.port = (uint8_t)priv->ibv_port; return 0; } @@ -1619,26 +1642,24 @@ flow_verbs_translate(struct rte_eth_dev *dev, static void flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow_handle *dh; - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *handle; if (!flow) return; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { - dh = &dev_flow->handle; - if (dh->ib_flow) { - claim_zero(mlx5_glue->destroy_flow(dh->ib_flow)); - dh->ib_flow = NULL; + LIST_FOREACH(handle, &flow->dev_handles, next) { + if (handle->ib_flow) { + claim_zero(mlx5_glue->destroy_flow(handle->ib_flow)); + handle->ib_flow = NULL; } - if (dh->hrxq) { - if (dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DROP) + if (handle->hrxq) { + if (handle->act_flags & MLX5_FLOW_ACTION_DROP) mlx5_hrxq_drop_release(dev); else - mlx5_hrxq_release(dev, dh->hrxq); - dh->hrxq = NULL; + mlx5_hrxq_release(dev, handle->hrxq); + handle->hrxq = NULL; } - if (dh->vf_vlan.tag && dh->vf_vlan.created) - mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); + if (handle->vf_vlan.tag && handle->vf_vlan.created) + mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); } } @@ -1653,15 +1674,15 @@ flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) static void flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *handle; if (!flow) return; flow_verbs_remove(dev, flow); - while (!LIST_EMPTY(&flow->dev_flows)) { - dev_flow = LIST_FIRST(&flow->dev_flows); - LIST_REMOVE(dev_flow, next); - rte_free(dev_flow); + while (!LIST_EMPTY(&flow->dev_handles)) { + handle = LIST_FIRST(&flow->dev_handles); + LIST_REMOVE(handle, next); + rte_free(handle); } if (flow->counter) { flow_verbs_counter_release(dev, flow->counter); @@ -1687,15 +1708,17 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_handle *dh; + struct mlx5_flow_handle *handle; struct mlx5_flow *dev_flow; int err; - - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { - dh = &dev_flow->handle; - if (dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DROP) { - dh->hrxq = mlx5_hrxq_drop_new(dev); - if (!dh->hrxq) { + int idx; + + for (idx = priv->flow_idx - 1; idx >= 0; idx--) { + dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; + handle = dev_flow->handle; + if (handle->act_flags & MLX5_FLOW_ACTION_DROP) { + handle->hrxq = mlx5_hrxq_drop_new(dev); + if (!handle->hrxq) { rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -1717,7 +1740,7 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, dev_flow->hash_fields, (*flow->rss.queue), flow->rss.queue_num, - !!(dev_flow->handle.layers & + !!(handle->layers & MLX5_FLOW_LAYER_TUNNEL)); if (!hrxq) { rte_flow_error_set @@ -1726,11 +1749,11 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, "cannot get hash queue"); goto error; } - dh->hrxq = hrxq; + handle->hrxq = hrxq; } - dh->ib_flow = mlx5_glue->create_flow(dh->hrxq->qp, - dev_flow->verbs.attr); - if (!dh->ib_flow) { + handle->ib_flow = mlx5_glue->create_flow(handle->hrxq->qp, + &dev_flow->verbs.attr); + if (!handle->ib_flow) { rte_flow_error_set(error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -1738,31 +1761,29 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, goto error; } if (priv->vmwa_context && - dev_flow->handle.vf_vlan.tag && - !dev_flow->handle.vf_vlan.created) { + handle->vf_vlan.tag && !handle->vf_vlan.created) { /* * The rule contains the VLAN pattern. * For VF we are going to create VLAN * interface to make hypervisor set correct * e-Switch vport context. */ - mlx5_vlan_vmwa_acquire(dev, &dev_flow->handle.vf_vlan); + mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan); } } return 0; error: err = rte_errno; /* Save rte_errno before cleanup. */ - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { - dh = &dev_flow->handle; - if (dh->hrxq) { - if (dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DROP) + LIST_FOREACH(handle, &flow->dev_handles, next) { + if (handle->hrxq) { + if (handle->act_flags & MLX5_FLOW_ACTION_DROP) mlx5_hrxq_drop_release(dev); else - mlx5_hrxq_release(dev, dh->hrxq); - dh->hrxq = NULL; + mlx5_hrxq_release(dev, handle->hrxq); + handle->hrxq = NULL; } - if (dh->vf_vlan.tag && dh->vf_vlan.created) - mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); + if (handle->vf_vlan.tag && handle->vf_vlan.created) + mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); } rte_errno = err; /* Restore rte_errno. */ return -rte_errno; diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index 0801cb6f5a..438b705952 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -307,6 +307,7 @@ mlx5_dev_start(struct rte_eth_dev *dev) mlx5_txq_stop(dev); return -rte_errno; } + /* Set started flag here for the following steps like control flow. */ dev->data->dev_started = 1; ret = mlx5_rx_intr_vec_enable(dev); if (ret) {