X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow.c;h=3b358b6b8cfdfbcf3d9b3c44b1a8b13f209efa46;hb=31cdde02f0be730fc1b9b12a8b53f8a04fa423fc;hp=2ef6558495551835c653861546b14ce26e0260aa;hpb=c79ab350a8976718d477d616f06678872a29b1b2;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 2ef6558495..3b358b6b8c 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -8,6 +8,7 @@ #include #include #include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ @@ -711,17 +712,19 @@ flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] dev_flow - * Pointer to device flow structure. + * @param[in] flow + * Pointer to flow structure. + * @param[in] dev_handle + * Pointer to device flow handle structure. */ static void -flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) +flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow, + struct mlx5_flow_handle *dev_handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct rte_flow *flow = dev_flow->flow; - const int mark = !!(dev_flow->actions & + const int mark = !!(dev_handle->act_flags & (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); - const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); + const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); unsigned int i; for (i = 0; i != flow->rss.queue_num; ++i) { @@ -750,7 +753,7 @@ flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) /* Increase the counter matching the flow. */ for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { if ((tunnels_info[j].tunnel & - dev_flow->layers) == + dev_handle->layers) == tunnels_info[j].tunnel) { rxq_ctrl->flow_tunnels_n[j]++; break; @@ -772,10 +775,10 @@ flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) static void flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) - flow_drv_rxq_flags_set(dev, dev_flow); + LIST_FOREACH(dev_handle, &flow->dev_handles, next) + flow_drv_rxq_flags_set(dev, flow, dev_handle); } /** @@ -784,17 +787,19 @@ flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) * * @param dev * Pointer to Ethernet device. - * @param[in] dev_flow - * Pointer to the device flow. + * @param[in] flow + * Pointer to flow structure. + * @param[in] dev_handle + * Pointer to the device flow handle structure. */ static void -flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) +flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow, + struct mlx5_flow_handle *dev_handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct rte_flow *flow = dev_flow->flow; - const int mark = !!(dev_flow->actions & + const int mark = !!(dev_handle->act_flags & (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); - const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL); + const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); unsigned int i; MLX5_ASSERT(dev->data->dev_started); @@ -819,7 +824,7 @@ flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) /* Decrease the counter matching the flow. */ for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { if ((tunnels_info[j].tunnel & - dev_flow->layers) == + dev_handle->layers) == tunnels_info[j].tunnel) { rxq_ctrl->flow_tunnels_n[j]--; break; @@ -842,10 +847,10 @@ flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) static void flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) - flow_drv_rxq_flags_trim(dev, dev_flow); + LIST_FOREACH(dev_handle, &flow->dev_handles, next) + flow_drv_rxq_flags_trim(dev, flow, dev_handle); } /** @@ -2308,11 +2313,11 @@ static void flow_mreg_split_qrss_release(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) - if (dev_flow->qrss_id) - flow_qrss_free_id(dev, dev_flow->qrss_id); + LIST_FOREACH(dev_handle, &flow->dev_handles, next) + if (dev_handle->qrss_id) + flow_qrss_free_id(dev, dev_handle->qrss_id); } static int @@ -2328,7 +2333,8 @@ flow_null_validate(struct rte_eth_dev *dev __rte_unused, } static struct mlx5_flow * -flow_null_prepare(const struct rte_flow_attr *attr __rte_unused, +flow_null_prepare(struct rte_eth_dev *dev __rte_unused, + const struct rte_flow_attr *attr __rte_unused, const struct rte_flow_item items[] __rte_unused, const struct rte_flow_action actions[] __rte_unused, struct rte_flow_error *error) @@ -2468,6 +2474,8 @@ flow_drv_validate(struct rte_eth_dev *dev, * setting backward reference to the flow should be done out of this function. * layers field is not filled either. * + * @param[in] dev + * Pointer to the dev structure. * @param[in] attr * Pointer to the flow attributes. * @param[in] items @@ -2481,7 +2489,8 @@ flow_drv_validate(struct rte_eth_dev *dev, * Pointer to device flow on success, otherwise NULL and rte_errno is set. */ static inline struct mlx5_flow * -flow_drv_prepare(const struct rte_flow *flow, +flow_drv_prepare(struct rte_eth_dev *dev, + const struct rte_flow *flow, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], @@ -2492,7 +2501,7 @@ flow_drv_prepare(const struct rte_flow *flow, MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); fops = flow_get_drv_ops(type); - return fops->prepare(attr, items, actions, error); + return fops->prepare(dev, attr, items, actions, error); } /** @@ -2695,18 +2704,22 @@ flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) { uint64_t layers = 0; - /* If no decap actions, use the layers directly. */ - if (!(dev_flow->actions & MLX5_FLOW_ACTION_DECAP)) - return dev_flow->layers; + /* + * Layers bits could be localization, but usually the compiler will + * help to do the optimization work for source code. + * If no decap actions, use the layers directly. + */ + if (!(dev_flow->handle->act_flags & MLX5_FLOW_ACTION_DECAP)) + return dev_flow->handle->layers; /* Convert L3 layers with decap action. */ - if (dev_flow->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) + if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; - else if (dev_flow->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) + else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; /* Convert L4 layers with decap action. */ - if (dev_flow->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) + if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; - else if (dev_flow->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) + else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; return layers; } @@ -3407,7 +3420,7 @@ flow_hairpin_split(struct rte_eth_dev *dev, * The last stage of splitting chain, just creates the subflow * without any modification. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. * @param[in] flow * Parent flow structure pointer. @@ -3440,19 +3453,19 @@ flow_create_split_inner(struct rte_eth_dev *dev, { struct mlx5_flow *dev_flow; - dev_flow = flow_drv_prepare(flow, attr, items, actions, error); + dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, error); if (!dev_flow) return -rte_errno; dev_flow->flow = flow; dev_flow->external = external; /* Subflow object was created, we must include one in the list. */ - LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); + LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next); /* * If dev_flow is as one of the suffix flow, some actions in suffix * flow may need some user defined item layer flags. */ if (prefix_layers) - dev_flow->layers = prefix_layers; + dev_flow->handle->layers = prefix_layers; if (sub_flow) *sub_flow = dev_flow; return flow_drv_translate(dev, dev_flow, attr, items, actions, error); @@ -3967,8 +3980,7 @@ flow_create_split_metadata(struct rte_eth_dev *dev, * reallocation becomes possible (for example, for * other flows in other threads). */ - dev_flow->qrss_id = qrss_id; - qrss_id = 0; + dev_flow->handle->qrss_id = qrss_id; ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); if (ret < 0) @@ -3983,6 +3995,8 @@ flow_create_split_metadata(struct rte_eth_dev *dev, external, error); if (ret < 0) goto exit; + /* qrss ID should be freed if failed. */ + qrss_id = 0; MLX5_ASSERT(dev_flow); } @@ -4079,7 +4093,7 @@ flow_create_split_meter(struct rte_eth_dev *dev, ret = -rte_errno; goto exit; } - dev_flow->mtr_flow_id = mtr_tag_id; + dev_flow->handle->mtr_flow_id = mtr_tag_id; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -4250,7 +4264,7 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */ flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; } - LIST_INIT(&flow->dev_flows); + LIST_INIT(&flow->dev_handles); if (rss && rss->types) { unsigned int graph_root; @@ -4265,6 +4279,8 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, buf->entries = 1; buf->entry[0].pattern = (void *)(uintptr_t)items; } + /* Reset device flow index to 0. */ + priv->flow_idx = 0; for (i = 0; i < buf->entries; ++i) { /* * The splitter may create multiple dev_flows, @@ -4283,13 +4299,13 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, attr_tx.group = MLX5_HAIRPIN_TX_TABLE; attr_tx.ingress = 0; attr_tx.egress = 1; - dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items, + dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, actions_hairpin_tx.actions, error); if (!dev_flow) goto error; dev_flow->flow = flow; dev_flow->external = 0; - LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next); + LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next); ret = flow_drv_translate(dev, dev_flow, &attr_tx, items_tx.items, actions_hairpin_tx.actions, error); @@ -4312,7 +4328,11 @@ flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list, if (ret) goto error; } - if (dev->data->dev_started) { + /* + * If the flow is external (from application) OR device is started, then + * the flow will be applied immediately. + */ + if (external || dev->data->dev_started) { ret = flow_drv_apply(dev, flow, error); if (ret < 0) goto error; @@ -4404,6 +4424,17 @@ mlx5_flow_create(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; + /* + * If the device is not started yet, it is not allowed to created a + * flow from application. PMD default flows and traffic control flows + * are not affected. + */ + if (unlikely(!dev->data->dev_started)) { + rte_errno = ENODEV; + DRV_LOG(DEBUG, "port %u is not started when " + "inserting a flow", dev->data->port_id); + return NULL; + } return flow_list_create(dev, &priv->flows, attr, items, actions, true, error); } @@ -4449,15 +4480,25 @@ flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. + * @param active + * If flushing is called avtively. */ void -mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) +mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list, + bool active) { + uint32_t num_flushed = 0; + while (!TAILQ_EMPTY(list)) { struct rte_flow *flow; flow = TAILQ_FIRST(list); flow_list_destroy(dev, list, flow); + num_flushed++; + } + if (active) { + DRV_LOG(INFO, "port %u: %u flows flushed before stopping", + dev->data->port_id, num_flushed); } } @@ -4522,6 +4563,66 @@ error: return -rte_errno; } +/** + * Stop all default actions for flows. + * + * @param dev + * Pointer to Ethernet device. + */ +void +mlx5_flow_stop_default(struct rte_eth_dev *dev) +{ + flow_mreg_del_default_copy_action(dev); +} + +/** + * Start all default actions for flows. + * + * @param dev + * Pointer to Ethernet device. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_flow_start_default(struct rte_eth_dev *dev) +{ + struct rte_flow_error error; + + /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ + return flow_mreg_add_default_copy_action(dev, &error); +} + +/** + * Allocate intermediate resources for flow creation. + * + * @param dev + * Pointer to Ethernet device. + */ +void +mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + if (!priv->inter_flows) + priv->inter_flows = rte_calloc(__func__, MLX5_NUM_MAX_DEV_FLOWS, + sizeof(struct mlx5_flow), 0); +} + +/** + * Free intermediate resources for flows. + * + * @param dev + * Pointer to Ethernet device. + */ +void +mlx5_flow_free_intermediate(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + rte_free(priv->inter_flows); + priv->inter_flows = NULL; +} + /** * Verify the flow list is empty * @@ -4737,7 +4838,7 @@ mlx5_flow_flush(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; - mlx5_flow_list_flush(dev, &priv->flows); + mlx5_flow_list_flush(dev, &priv->flows, false); return 0; } @@ -5179,7 +5280,7 @@ flow_fdir_filter_flush(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; - mlx5_flow_list_flush(dev, &priv->flows); + mlx5_flow_list_flush(dev, &priv->flows, false); } /** @@ -5393,9 +5494,9 @@ mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, * Pointer to Ethernet device structure. * * @return - * Pointer to allocated counter on success, NULL otherwise. + * Index to allocated counter on success, 0 otherwise. */ -struct mlx5_flow_counter * +uint32_t mlx5_counter_alloc(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; @@ -5408,7 +5509,7 @@ mlx5_counter_alloc(struct rte_eth_dev *dev) DRV_LOG(ERR, "port %u counter allocate is not supported.", dev->data->port_id); - return NULL; + return 0; } /** @@ -5417,10 +5518,10 @@ mlx5_counter_alloc(struct rte_eth_dev *dev) * @param[in] dev * Pointer to Ethernet device structure. * @param[in] cnt - * Pointer to counter to be free. + * Index to counter to be free. */ void -mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) +mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) { const struct mlx5_flow_driver_ops *fops; struct rte_flow_attr attr = { .transfer = 0 }; @@ -5441,7 +5542,7 @@ mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) * @param[in] dev * Pointer to Ethernet device structure. * @param[in] cnt - * Pointer to counter to query. + * Index to counter to query. * @param[in] clear * Set to clear counter statistics. * @param[out] pkts @@ -5453,7 +5554,7 @@ mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) * 0 on success, a negative errno value otherwise. */ int -mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt, +mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, bool clear, uint64_t *pkts, uint64_t *bytes) { const struct mlx5_flow_driver_ops *fops; @@ -5554,6 +5655,13 @@ next_container: dcs = (struct mlx5_devx_obj *)(uintptr_t)rte_atomic64_read (&pool->a64_dcs); offset = batch ? 0 : dcs->id % MLX5_COUNTERS_PER_POOL; + /* + * Identify the counters released between query trigger and query + * handle more effiecntly. The counter released in this gap period + * should wait for a new round of query as the new arrived packets + * will not be taken into account. + */ + rte_atomic64_add(&pool->start_query_gen, 1); ret = mlx5_devx_cmd_flow_counter_query(dcs, 0, MLX5_COUNTERS_PER_POOL - offset, NULL, NULL, pool->raw_hw->mem_mng->dm->id, @@ -5562,6 +5670,7 @@ next_container: sh->devx_comp, (uint64_t)(uintptr_t)pool); if (ret) { + rte_atomic64_sub(&pool->start_query_gen, 1); DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" " %d", pool->min_dcs->id); pool->raw_hw = NULL; @@ -5601,13 +5710,17 @@ mlx5_flow_async_pool_query_handle(struct mlx5_ibv_shared *sh, struct mlx5_counter_stats_raw *raw_to_free; if (unlikely(status)) { + rte_atomic64_sub(&pool->start_query_gen, 1); raw_to_free = pool->raw_hw; } else { raw_to_free = pool->raw; rte_spinlock_lock(&pool->sl); pool->raw = pool->raw_hw; rte_spinlock_unlock(&pool->sl); - rte_atomic64_add(&pool->query_gen, 1); + MLX5_ASSERT(rte_atomic64_read(&pool->end_query_gen) + 1 == + rte_atomic64_read(&pool->start_query_gen)); + rte_atomic64_set(&pool->end_query_gen, + rte_atomic64_read(&pool->start_query_gen)); /* Be sure the new raw counters data is updated in memory. */ rte_cio_wmb(); }