X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=6263ecc7314be8ec110bcdf65ccafa3c2a8476a8;hb=b58398d9f85bb959fb7f1ec208a637ff4827cbab;hp=8688cab7fa54b24e48bb8161dda27808bc28bde7;hpb=8acf8ac9b77b3606f9b3fb1d218404c918b2c523;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 8688cab7fa..6263ecc731 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -434,6 +434,7 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, /* Fetch variable byte size mask from the array. */ mask = flow_dv_fetch_field((const uint8_t *)item->mask + field->offset, field->size); + MLX5_ASSERT(mask); if (!mask) { ++field; continue; @@ -2490,14 +2491,14 @@ flow_dv_encap_decap_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.encap_decap = idx; + dev_flow->handle->dvh.rix_encap_decap = idx; dev_flow->dv.encap_decap = cache_resource; return 0; } } /* Register new encap/decap resource. */ cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], - &dev_flow->handle->dvh.encap_decap); + &dev_flow->handle->dvh.rix_encap_decap); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -2518,7 +2519,8 @@ flow_dv_encap_decap_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps, - dev_flow->handle->dvh.encap_decap, cache_resource, next); + dev_flow->handle->dvh.rix_encap_decap, cache_resource, + next); dev_flow->dv.encap_decap = cache_resource; DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", (void *)cache_resource, @@ -2572,7 +2574,8 @@ flow_dv_jump_tbl_resource_register (void *)&tbl_data->jump, cnt); } rte_atomic32_inc(&tbl_data->jump.refcnt); - dev_flow->handle->dvh.jump = &tbl_data->jump; + dev_flow->handle->rix_jump = tbl_data->idx; + dev_flow->dv.jump = &tbl_data->jump; return 0; } @@ -2601,21 +2604,25 @@ flow_dv_port_id_action_resource_register struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_port_id_action_resource *cache_resource; + uint32_t idx = 0; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) { + ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list, + idx, cache_resource, next) { if (resource->port_id == cache_resource->port_id) { DRV_LOG(DEBUG, "port id action resource resource %p: " "refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.port_id_action = cache_resource; + dev_flow->handle->rix_port_id_action = idx; + dev_flow->dv.port_id_action = cache_resource; return 0; } } /* Register new port id action resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], + &dev_flow->handle->rix_port_id_action); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -2637,8 +2644,10 @@ flow_dv_port_id_action_resource_register } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next); - dev_flow->handle->dvh.port_id_action = cache_resource; + ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list, + dev_flow->handle->rix_port_id_action, cache_resource, + next); + dev_flow->dv.port_id_action = cache_resource; DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -2683,14 +2692,14 @@ flow_dv_push_vlan_action_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.push_vlan_res = idx; + dev_flow->handle->dvh.rix_push_vlan = idx; dev_flow->dv.push_vlan_res = cache_resource; return 0; } } /* Register new push_vlan action resource. */ cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], - &dev_flow->handle->dvh.push_vlan_res); + &dev_flow->handle->dvh.rix_push_vlan); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -2715,7 +2724,7 @@ flow_dv_push_vlan_action_resource_register rte_atomic32_inc(&cache_resource->refcnt); ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &sh->push_vlan_action_list, - dev_flow->handle->dvh.push_vlan_res, + dev_flow->handle->dvh.rix_push_vlan, cache_resource, next); dev_flow->dv.push_vlan_res = cache_resource; DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++", @@ -3699,9 +3708,9 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Meter not found"); - if (fm->ref_cnt && (!(fm->attr.transfer == attr->transfer || - (!fm->attr.ingress && !attr->ingress && attr->egress) || - (!fm->attr.egress && !attr->egress && attr->ingress)))) + if (fm->ref_cnt && (!(fm->transfer == attr->transfer || + (!fm->ingress && !attr->ingress && attr->egress) || + (!fm->egress && !attr->egress && attr->ingress)))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Flow attributes are either invalid " @@ -4482,7 +4491,9 @@ flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter) * Pointer to error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * - 0 on success and non root table. + * - 1 on success and root table. + * - a negative errno value otherwise and rte_errno is set. */ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, @@ -4492,6 +4503,7 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; uint32_t priority_max = priv->config.flow_prio - 1; + int ret = 0; #ifndef HAVE_MLX5DV_DR if (attributes->group) @@ -4500,14 +4512,15 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, NULL, "groups are not supported"); #else - uint32_t table; - int ret; + uint32_t table = 0; ret = mlx5_flow_group_to_table(attributes, external, attributes->group, !!priv->fdb_def_rule, &table, error); if (ret) return ret; + if (!table) + ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; #endif if (attributes->priority != MLX5_FLOW_PRIO_RSVD && attributes->priority >= priority_max) @@ -4537,7 +4550,7 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_ATTR, NULL, "must specify exactly one of " "ingress or egress"); - return 0; + return ret; } /** @@ -4553,6 +4566,8 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, * Pointer to the list of actions. * @param[in] external * This flow rule is created by request external to PMD. + * @param[in] hairpin + * Number of hairpin TX actions, 0 means classic flow. * @param[out] error * Pointer to the error structure. * @@ -4563,7 +4578,7 @@ static int flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], - bool external, struct rte_flow_error *error) + bool external, int hairpin, struct rte_flow_error *error) { int ret; uint64_t action_flags = 0; @@ -4610,12 +4625,15 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, struct mlx5_dev_config *dev_conf = &priv->config; uint16_t queue_index = 0xFFFF; const struct rte_flow_item_vlan *vlan_m = NULL; + int16_t rw_act_num = 0; + uint64_t is_root; if (items == NULL) return -1; ret = flow_dv_validate_attributes(dev, attr, external, error); if (ret < 0) return ret; + is_root = (uint64_t)ret; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); int type = items->type; @@ -4892,6 +4910,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, action_flags |= MLX5_FLOW_ACTION_FLAG; ++actions_n; } + rw_act_num += MLX5_ACT_NUM_SET_MARK; break; case RTE_FLOW_ACTION_TYPE_MARK: ret = flow_dv_validate_action_mark(dev, actions, @@ -4910,6 +4929,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, action_flags |= MLX5_FLOW_ACTION_MARK; ++actions_n; } + rw_act_num += MLX5_ACT_NUM_SET_MARK; break; case RTE_FLOW_ACTION_TYPE_SET_META: ret = flow_dv_validate_action_set_meta(dev, actions, @@ -4921,6 +4941,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; action_flags |= MLX5_FLOW_ACTION_SET_META; + rw_act_num += MLX5_ACT_NUM_SET_META; break; case RTE_FLOW_ACTION_TYPE_SET_TAG: ret = flow_dv_validate_action_set_tag(dev, actions, @@ -4932,6 +4953,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; action_flags |= MLX5_FLOW_ACTION_SET_TAG; + rw_act_num += MLX5_ACT_NUM_SET_TAG; break; case RTE_FLOW_ACTION_TYPE_DROP: ret = mlx5_flow_validate_action_drop(action_flags, @@ -5009,6 +5031,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, return ret; /* Count VID with push_vlan command. */ action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; + rw_act_num += MLX5_ACT_NUM_MDF_VID; break; case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: @@ -5070,8 +5093,15 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? MLX5_FLOW_ACTION_SET_MAC_SRC : MLX5_FLOW_ACTION_SET_MAC_DST; + /* + * Even if the source and destination MAC addresses have + * overlap in the header with 4B alignment, the convert + * function will handle them separately and 4 SW actions + * will be created. And 2 actions will be added each + * time no matter how many bytes of address will be set. + */ + rw_act_num += MLX5_ACT_NUM_MDF_MAC; break; - case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: ret = flow_dv_validate_action_modify_ipv4(action_flags, @@ -5087,6 +5117,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? MLX5_FLOW_ACTION_SET_IPV4_SRC : MLX5_FLOW_ACTION_SET_IPV4_DST; + rw_act_num += MLX5_ACT_NUM_MDF_IPV4; break; case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: @@ -5109,6 +5140,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? MLX5_FLOW_ACTION_SET_IPV6_SRC : MLX5_FLOW_ACTION_SET_IPV6_DST; + rw_act_num += MLX5_ACT_NUM_MDF_IPV6; break; case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: case RTE_FLOW_ACTION_TYPE_SET_TP_DST: @@ -5125,6 +5157,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? MLX5_FLOW_ACTION_SET_TP_SRC : MLX5_FLOW_ACTION_SET_TP_DST; + rw_act_num += MLX5_ACT_NUM_MDF_PORT; break; case RTE_FLOW_ACTION_TYPE_DEC_TTL: case RTE_FLOW_ACTION_TYPE_SET_TTL: @@ -5141,6 +5174,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_SET_TTL ? MLX5_FLOW_ACTION_SET_TTL : MLX5_FLOW_ACTION_DEC_TTL; + rw_act_num += MLX5_ACT_NUM_MDF_TTL; break; case RTE_FLOW_ACTION_TYPE_JUMP: ret = flow_dv_validate_action_jump(actions, @@ -5168,6 +5202,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? MLX5_FLOW_ACTION_INC_TCP_SEQ : MLX5_FLOW_ACTION_DEC_TCP_SEQ; + rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ; break; case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: @@ -5185,10 +5220,13 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? MLX5_FLOW_ACTION_INC_TCP_ACK : MLX5_FLOW_ACTION_DEC_TCP_ACK; + rw_act_num += MLX5_ACT_NUM_MDF_TCPACK; break; - case MLX5_RTE_FLOW_ACTION_TYPE_TAG: case MLX5_RTE_FLOW_ACTION_TYPE_MARK: + break; + case MLX5_RTE_FLOW_ACTION_TYPE_TAG: case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: + rw_act_num += MLX5_ACT_NUM_SET_TAG; break; case RTE_FLOW_ACTION_TYPE_METER: ret = mlx5_flow_validate_action_meter(dev, @@ -5199,6 +5237,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, return ret; action_flags |= MLX5_FLOW_ACTION_METER; ++actions_n; + /* Meter action will add one more TAG action. */ + rw_act_num += MLX5_ACT_NUM_SET_TAG; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: ret = flow_dv_validate_action_modify_ipv4_dscp @@ -5212,6 +5252,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; + rw_act_num += MLX5_ACT_NUM_SET_DSCP; break; case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: ret = flow_dv_validate_action_modify_ipv6_dscp @@ -5225,6 +5266,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; + rw_act_num += MLX5_ACT_NUM_SET_DSCP; break; default: return rte_flow_error_set(error, ENOTSUP, @@ -5297,6 +5339,21 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, NULL, "encap is not supported" " for ingress traffic"); } + /* Hairpin flow will add one more TAG action. */ + if (hairpin > 0) + rw_act_num += MLX5_ACT_NUM_SET_TAG; + /* extra metadata enabled: one more TAG action will be add. */ + if (dev_conf->dv_flow_en && + dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && + mlx5_flow_ext_mreg_supported(dev)) + rw_act_num += MLX5_ACT_NUM_SET_TAG; + if ((uint32_t)rw_act_num >= + flow_dv_modify_hdr_action_max(dev, is_root)) { + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "too many header modify" + " actions to support"); + } return 0; } @@ -5326,7 +5383,7 @@ flow_dv_prepare(struct rte_eth_dev *dev, const struct rte_flow_action actions[] __rte_unused, struct rte_flow_error *error) { - size_t size = sizeof(struct mlx5_flow_handle); + uint32_t handle_idx = 0; struct mlx5_flow *dev_flow; struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; @@ -5338,7 +5395,8 @@ flow_dv_prepare(struct rte_eth_dev *dev, "not free temporary device flow"); return NULL; } - dev_handle = rte_calloc(__func__, 1, size, 0); + dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], + &handle_idx); if (!dev_handle) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -5348,6 +5406,7 @@ flow_dv_prepare(struct rte_eth_dev *dev, /* No multi-thread supporting. */ dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; dev_flow->handle = dev_handle; + dev_flow->handle_idx = handle_idx; dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); /* * The matching value needs to be cleared to 0 before using. In the @@ -6869,6 +6928,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64); struct mlx5_flow_tbl_data_entry *tbl_data; + uint32_t idx = 0; int ret; void *domain; @@ -6879,7 +6939,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, rte_atomic32_inc(&tbl->refcnt); return tbl; } - tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0); + tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); if (!tbl_data) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -6887,6 +6947,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, "cannot allocate flow table data entry"); return NULL; } + tbl_data->idx = idx; tbl = &tbl_data->tbl; pos = &tbl_data->entry; if (transfer) @@ -6900,7 +6961,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create flow table object"); - rte_free(tbl_data); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); return NULL; } /* @@ -6917,7 +6978,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot insert flow table data entry"); mlx5_glue->dr_destroy_flow_tbl(tbl->obj); - rte_free(tbl_data); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); } rte_atomic32_inc(&tbl->refcnt); return tbl; @@ -6952,7 +7013,8 @@ flow_dv_tbl_resource_release(struct rte_eth_dev *dev, tbl->obj = NULL; /* remove the entry from the hash list and free memory. */ mlx5_hlist_remove(sh->flow_tbls, pos); - rte_free(tbl_data); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP], + tbl_data->idx); return 0; } return 1; @@ -7092,14 +7154,16 @@ flow_dv_tag_resource_register cache_resource = container_of (entry, struct mlx5_flow_dv_tag_resource, entry); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.tag_resource = cache_resource; + dev_flow->handle->dvh.rix_tag = cache_resource->idx; + dev_flow->dv.tag_resource = cache_resource; DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); return 0; } /* Register new resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], + &dev_flow->handle->dvh.rix_tag); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -7121,7 +7185,7 @@ flow_dv_tag_resource_register RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot insert tag"); } - dev_flow->handle->dvh.tag_resource = cache_resource; + dev_flow->dv.tag_resource = cache_resource; DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -7133,20 +7197,23 @@ flow_dv_tag_resource_register * * @param dev * Pointer to Ethernet device. - * @param flow - * Pointer to mlx5_flow. + * @param tag_idx + * Tag index. * * @return * 1 while a reference on it exists, 0 when freed. */ static int flow_dv_tag_release(struct rte_eth_dev *dev, - struct mlx5_flow_dv_tag_resource *tag) + uint32_t tag_idx) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_dv_tag_resource *tag; - MLX5_ASSERT(tag); + tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); + if (!tag) + return 0; DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", dev->data->port_id, (void *)tag, rte_atomic32_read(&tag->refcnt)); @@ -7155,7 +7222,7 @@ flow_dv_tag_release(struct rte_eth_dev *dev, mlx5_hlist_remove(sh->tag_table, &tag->entry); DRV_LOG(DEBUG, "port %u tag %p: removed", dev->data->port_id, (void *)tag); - rte_free(tag); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); return 0; } return 1; @@ -7261,18 +7328,20 @@ flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev, * * @param[in] dev_flow * Pointer to the mlx5_flow. + * @param[in] rss_desc + * Pointer to the mlx5_flow_rss_desc. */ static void -flow_dv_hashfields_set(struct mlx5_flow *dev_flow) +flow_dv_hashfields_set(struct mlx5_flow *dev_flow, + struct mlx5_flow_rss_desc *rss_desc) { - struct rte_flow *flow = dev_flow->flow; uint64_t items = dev_flow->handle->layers; int rss_inner = 0; - uint64_t rss_types = rte_eth_rss_hf_refine(flow->rss.types); + uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types); dev_flow->hash_fields = 0; #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT - if (flow->rss.level >= 2) { + if (rss_desc->level >= 2) { dev_flow->hash_fields |= IBV_RX_HASH_INNER; rss_inner = 1; } @@ -7357,6 +7426,9 @@ __flow_dv_translate(struct rte_eth_dev *dev, struct mlx5_dev_config *dev_conf = &priv->config; struct rte_flow *flow = dev_flow->flow; struct mlx5_flow_handle *handle = dev_flow->handle; + struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) + priv->rss_desc) + [!!priv->flow_nested_idx]; uint64_t item_flags = 0; uint64_t last_item = 0; uint64_t action_flags = 0; @@ -7412,6 +7484,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; + struct mlx5_flow_meter *fm = NULL; switch (action_type) { case RTE_FLOW_ACTION_TYPE_VOID: @@ -7420,16 +7493,20 @@ __flow_dv_translate(struct rte_eth_dev *dev, if (flow_dv_translate_action_port_id(dev, action, &port_id, error)) return -rte_errno; + memset(&port_id_resource, 0, sizeof(port_id_resource)); port_id_resource.port_id = port_id; if (flow_dv_port_id_action_resource_register (dev, &port_id_resource, dev_flow, error)) return -rte_errno; + MLX5_ASSERT(!handle->rix_port_id_action); dev_flow->dv.actions[actions_n++] = - handle->dvh.port_id_action->action; + dev_flow->dv.port_id_action->action; action_flags |= MLX5_FLOW_ACTION_PORT_ID; + dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID; break; case RTE_FLOW_ACTION_TYPE_FLAG: action_flags |= MLX5_FLOW_ACTION_FLAG; + dev_flow->handle->mark = 1; if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { struct rte_flow_action_mark mark = { .id = MLX5_FLOW_MARK_DEFAULT, @@ -7448,15 +7525,17 @@ __flow_dv_translate(struct rte_eth_dev *dev, * right now. So the pointer to the tag resource must be * zero before the register process. */ - MLX5_ASSERT(!handle->dvh.tag_resource); + MLX5_ASSERT(!handle->dvh.rix_tag); if (flow_dv_tag_resource_register(dev, tag_be, dev_flow, error)) return -rte_errno; + MLX5_ASSERT(dev_flow->dv.tag_resource); dev_flow->dv.actions[actions_n++] = - handle->dvh.tag_resource->action; + dev_flow->dv.tag_resource->action; break; case RTE_FLOW_ACTION_TYPE_MARK: action_flags |= MLX5_FLOW_ACTION_MARK; + dev_flow->handle->mark = 1; if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { const struct rte_flow_action_mark *mark = (const struct rte_flow_action_mark *) @@ -7475,12 +7554,13 @@ __flow_dv_translate(struct rte_eth_dev *dev, tag_be = mlx5_flow_mark_set (((const struct rte_flow_action_mark *) (actions->conf))->id); - MLX5_ASSERT(!handle->dvh.tag_resource); + MLX5_ASSERT(!handle->dvh.rix_tag); if (flow_dv_tag_resource_register(dev, tag_be, dev_flow, error)) return -rte_errno; + MLX5_ASSERT(dev_flow->dv.tag_resource); dev_flow->dv.actions[actions_n++] = - handle->dvh.tag_resource->action; + dev_flow->dv.tag_resource->action; break; case RTE_FLOW_ACTION_TYPE_SET_META: if (flow_dv_convert_action_set_meta @@ -7500,29 +7580,29 @@ __flow_dv_translate(struct rte_eth_dev *dev, break; case RTE_FLOW_ACTION_TYPE_DROP: action_flags |= MLX5_FLOW_ACTION_DROP; + dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; break; case RTE_FLOW_ACTION_TYPE_QUEUE: - MLX5_ASSERT(flow->rss.queue); queue = actions->conf; - flow->rss.queue_num = 1; - (*flow->rss.queue)[0] = queue->index; + rss_desc->queue_num = 1; + rss_desc->queue[0] = queue->index; action_flags |= MLX5_FLOW_ACTION_QUEUE; + dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; break; case RTE_FLOW_ACTION_TYPE_RSS: - MLX5_ASSERT(flow->rss.queue); rss = actions->conf; - if (flow->rss.queue) - memcpy((*flow->rss.queue), rss->queue, - rss->queue_num * sizeof(uint16_t)); - flow->rss.queue_num = rss->queue_num; + memcpy(rss_desc->queue, rss->queue, + rss->queue_num * sizeof(uint16_t)); + rss_desc->queue_num = rss->queue_num; /* NULL RSS key indicates default RSS key. */ rss_key = !rss->key ? rss_hash_default_key : rss->key; - memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN); + memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); /* * rss->level and rss.types should be set in advance * when expanding items for RSS. */ action_flags |= MLX5_FLOW_ACTION_RSS; + dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; break; case RTE_FLOW_ACTION_TYPE_COUNT: if (!dev_conf->devx) { @@ -7680,8 +7760,9 @@ cnt_err: "cannot create jump action."); } dev_flow->dv.actions[actions_n++] = - handle->dvh.jump->action; + dev_flow->dv.jump->action; action_flags |= MLX5_FLOW_ACTION_JUMP; + dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: @@ -7777,20 +7858,30 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_METER: mtr = actions->conf; if (!flow->meter) { - flow->meter = mlx5_flow_meter_attach(priv, - mtr->mtr_id, attr, - error); - if (!flow->meter) + fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, + attr, error); + if (!fm) return rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "meter not found " "or invalid parameters"); + flow->meter = fm->meter_id; } /* Set the meter action. */ + if (!fm) { + fm = mlx5_flow_meter_find(priv, flow->meter); + if (!fm) + return rte_flow_error_set(error, + rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "meter not found " + "or invalid parameters"); + } dev_flow->dv.actions[actions_n++] = - flow->meter->mfts->meter_action; + fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -7824,7 +7915,7 @@ cnt_err: modify_action_position = actions_n++; } dev_flow->dv.actions_n = actions_n; - handle->act_flags = action_flags; + dev_flow->act_flags = action_flags; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); int item_type = items->type; @@ -7915,7 +8006,7 @@ cnt_err: case RTE_FLOW_ITEM_TYPE_GRE: flow_dv_translate_item_gre(match_mask, match_value, items, tunnel); - matcher.priority = flow->rss.level >= 2 ? + matcher.priority = rss_desc->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_GRE; break; @@ -7927,14 +8018,14 @@ cnt_err: case RTE_FLOW_ITEM_TYPE_NVGRE: flow_dv_translate_item_nvgre(match_mask, match_value, items, tunnel); - matcher.priority = flow->rss.level >= 2 ? + matcher.priority = rss_desc->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_GRE; break; case RTE_FLOW_ITEM_TYPE_VXLAN: flow_dv_translate_item_vxlan(match_mask, match_value, items, tunnel); - matcher.priority = flow->rss.level >= 2 ? + matcher.priority = rss_desc->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_VXLAN; break; @@ -7942,21 +8033,21 @@ cnt_err: flow_dv_translate_item_vxlan_gpe(match_mask, match_value, items, tunnel); - matcher.priority = flow->rss.level >= 2 ? + matcher.priority = rss_desc->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_VXLAN_GPE; break; case RTE_FLOW_ITEM_TYPE_GENEVE: flow_dv_translate_item_geneve(match_mask, match_value, items, tunnel); - matcher.priority = flow->rss.level >= 2 ? + matcher.priority = rss_desc->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_GENEVE; break; case RTE_FLOW_ITEM_TYPE_MPLS: flow_dv_translate_item_mpls(match_mask, match_value, items, last_item, tunnel); - matcher.priority = flow->rss.level >= 2 ? + matcher.priority = rss_desc->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_MPLS; break; @@ -7999,7 +8090,7 @@ cnt_err: case RTE_FLOW_ITEM_TYPE_GTP: flow_dv_translate_item_gtp(match_mask, match_value, items, tunnel); - matcher.priority = flow->rss.level >= 2 ? + matcher.priority = rss_desc->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_GTP; break; @@ -8031,7 +8122,7 @@ cnt_err: */ handle->layers |= item_flags; if (action_flags & MLX5_FLOW_ACTION_RSS) - flow_dv_hashfields_set(dev_flow); + flow_dv_hashfields_set(dev_flow, rss_desc); /* Register matcher. */ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, matcher.mask.size); @@ -8069,6 +8160,7 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, struct mlx5_flow_handle_dv *dv_h; struct mlx5_flow *dev_flow; struct mlx5_priv *priv = dev->data->dev_private; + uint32_t handle_idx; int n; int err; int idx; @@ -8079,12 +8171,13 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, dh = dev_flow->handle; dv_h = &dh->dvh; n = dv->actions_n; - if (dh->act_flags & MLX5_FLOW_ACTION_DROP) { + if (dh->fate_action == MLX5_FLOW_FATE_DROP) { if (dv->transfer) { dv->actions[n++] = priv->sh->esw_drop_action; } else { - dh->hrxq = mlx5_hrxq_drop_new(dev); - if (!dh->hrxq) { + struct mlx5_hrxq *drop_hrxq; + drop_hrxq = mlx5_hrxq_drop_new(dev); + if (!drop_hrxq) { rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -8092,28 +8185,40 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, "cannot get drop hash queue"); goto error; } - dv->actions[n++] = dh->hrxq->action; + /* + * Drop queues will be released by the specify + * mlx5_hrxq_drop_release() function. Assign + * the special index to hrxq to mark the queue + * has been allocated. + */ + dh->rix_hrxq = UINT32_MAX; + dv->actions[n++] = drop_hrxq->action; } - } else if (dh->act_flags & - (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) { + } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { struct mlx5_hrxq *hrxq; - - MLX5_ASSERT(flow->rss.queue); - hrxq = mlx5_hrxq_get(dev, flow->rss.key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - (*flow->rss.queue), - flow->rss.queue_num); - if (!hrxq) { - hrxq = mlx5_hrxq_new - (dev, flow->rss.key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - (*flow->rss.queue), - flow->rss.queue_num, - !!(dh->layers & - MLX5_FLOW_LAYER_TUNNEL)); + uint32_t hrxq_idx; + struct mlx5_flow_rss_desc *rss_desc = + &((struct mlx5_flow_rss_desc *)priv->rss_desc) + [!!priv->flow_nested_idx]; + + MLX5_ASSERT(rss_desc->queue_num); + hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, + MLX5_RSS_HASH_KEY_LEN, + dev_flow->hash_fields, + rss_desc->queue, + rss_desc->queue_num); + if (!hrxq_idx) { + hrxq_idx = mlx5_hrxq_new + (dev, rss_desc->key, + MLX5_RSS_HASH_KEY_LEN, + dev_flow->hash_fields, + rss_desc->queue, + rss_desc->queue_num, + !!(dh->layers & + MLX5_FLOW_LAYER_TUNNEL)); } + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], + hrxq_idx); if (!hrxq) { rte_flow_error_set (error, rte_errno, @@ -8121,8 +8226,8 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, "cannot get hash queue"); goto error; } - dh->hrxq = hrxq; - dv->actions[n++] = dh->hrxq->action; + dh->rix_hrxq = hrxq_idx; + dv->actions[n++] = hrxq->action; } dh->ib_flow = mlx5_glue->dv_create_flow(dv_h->matcher->matcher_object, @@ -8149,13 +8254,17 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, return 0; error: err = rte_errno; /* Save rte_errno before cleanup. */ - LIST_FOREACH(dh, &flow->dev_handles, next) { - if (dh->hrxq) { - if (dh->act_flags & MLX5_FLOW_ACTION_DROP) + SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, + handle_idx, dh, next) { + /* hrxq is union, don't clear it if the flag is not set. */ + if (dh->rix_hrxq) { + if (dh->fate_action == MLX5_FLOW_FATE_DROP) { mlx5_hrxq_drop_release(dev); - else - mlx5_hrxq_release(dev, dh->hrxq); - dh->hrxq = NULL; + dh->rix_hrxq = 0; + } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { + mlx5_hrxq_release(dev, dh->rix_hrxq); + dh->rix_hrxq = 0; + } } if (dh->vf_vlan.tag && dh->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); @@ -8215,7 +8324,7 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - uint32_t idx = handle->dvh.encap_decap; + uint32_t idx = handle->dvh.rix_encap_decap; struct mlx5_flow_dv_encap_decap_resource *cache_resource; cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], @@ -8255,12 +8364,15 @@ static int flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { - struct mlx5_flow_dv_jump_tbl_resource *cache_resource = - handle->dvh.jump; - struct mlx5_flow_tbl_data_entry *tbl_data = - container_of(cache_resource, - struct mlx5_flow_tbl_data_entry, jump); + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_dv_jump_tbl_resource *cache_resource; + struct mlx5_flow_tbl_data_entry *tbl_data; + tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP], + handle->rix_jump); + if (!tbl_data) + return 0; + cache_resource = &tbl_data->jump; MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", (void *)cache_resource, @@ -8311,6 +8423,8 @@ flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle) /** * Release port ID action resource. * + * @param dev + * Pointer to Ethernet device. * @param handle * Pointer to mlx5_flow_handle. * @@ -8318,11 +8432,17 @@ flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle) * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_port_id_action_resource_release(struct mlx5_flow_handle *handle) +flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow_handle *handle) { - struct mlx5_flow_dv_port_id_action_resource *cache_resource = - handle->dvh.port_id_action; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_dv_port_id_action_resource *cache_resource; + uint32_t idx = handle->rix_port_id_action; + cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], + idx); + if (!cache_resource) + return 0; MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", (void *)cache_resource, @@ -8330,8 +8450,10 @@ flow_dv_port_id_action_resource_release(struct mlx5_flow_handle *handle) if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { claim_zero(mlx5_glue->destroy_flow_action (cache_resource->action)); - LIST_REMOVE(cache_resource, next); - rte_free(cache_resource); + ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID], + &priv->sh->port_id_action_list, idx, + cache_resource, next); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx); DRV_LOG(DEBUG, "port id action resource %p: removed", (void *)cache_resource); return 0; @@ -8355,7 +8477,7 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - uint32_t idx = handle->dvh.push_vlan_res; + uint32_t idx = handle->dvh.rix_push_vlan; struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], @@ -8380,6 +8502,33 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, return 1; } +/** + * Release the fate resource. + * + * @param dev + * Pointer to Ethernet device. + * @param handle + * Pointer to mlx5_flow_handle. + */ +static void +flow_dv_fate_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow_handle *handle) +{ + if (!handle->rix_fate) + return; + if (handle->fate_action == MLX5_FLOW_FATE_DROP) + mlx5_hrxq_drop_release(dev); + else if (handle->fate_action == MLX5_FLOW_FATE_QUEUE) + mlx5_hrxq_release(dev, handle->rix_hrxq); + else if (handle->fate_action == MLX5_FLOW_FATE_JUMP) + flow_dv_jump_tbl_resource_release(dev, handle); + else if (handle->fate_action == MLX5_FLOW_FATE_PORT_ID) + flow_dv_port_id_action_resource_release(dev, handle); + else + DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action); + handle->rix_fate = 0; +} + /** * Remove the flow from the NIC but keeps it in memory. * Lock free, (mutex should be acquired by caller). @@ -8393,23 +8542,27 @@ static void __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dh; + uint32_t handle_idx; + struct mlx5_priv *priv = dev->data->dev_private; if (!flow) return; - LIST_FOREACH(dh, &flow->dev_handles, next) { + handle_idx = flow->dev_handles; + while (handle_idx) { + dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], + handle_idx); + if (!dh) + return; if (dh->ib_flow) { claim_zero(mlx5_glue->dv_destroy_flow(dh->ib_flow)); dh->ib_flow = NULL; } - if (dh->hrxq) { - if (dh->act_flags & MLX5_FLOW_ACTION_DROP) - mlx5_hrxq_drop_release(dev); - else - mlx5_hrxq_release(dev, dh->hrxq); - dh->hrxq = NULL; - } + if (dh->fate_action == MLX5_FLOW_FATE_DROP || + dh->fate_action == MLX5_FLOW_FATE_QUEUE) + flow_dv_fate_resource_release(dev, dh); if (dh->vf_vlan.tag && dh->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); + handle_idx = dh->next.next; } } @@ -8426,6 +8579,7 @@ static void __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; + struct mlx5_priv *priv = dev->data->dev_private; if (!flow) return; @@ -8435,29 +8589,36 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - mlx5_flow_meter_detach(flow->meter); - flow->meter = NULL; + struct mlx5_flow_meter *fm; + + fm = mlx5_flow_meter_find(priv, flow->meter); + if (fm) + mlx5_flow_meter_detach(fm); + flow->meter = 0; } - while (!LIST_EMPTY(&flow->dev_handles)) { - dev_handle = LIST_FIRST(&flow->dev_handles); - LIST_REMOVE(dev_handle, next); + while (flow->dev_handles) { + uint32_t tmp_idx = flow->dev_handles; + + dev_handle = mlx5_ipool_get(priv->sh->ipool + [MLX5_IPOOL_MLX5_FLOW], tmp_idx); + if (!dev_handle) + return; + flow->dev_handles = dev_handle->next.next; if (dev_handle->dvh.matcher) flow_dv_matcher_release(dev, dev_handle); - if (dev_handle->dvh.encap_decap) + if (dev_handle->dvh.rix_encap_decap) flow_dv_encap_decap_resource_release(dev, dev_handle); if (dev_handle->dvh.modify_hdr) flow_dv_modify_hdr_resource_release(dev_handle); - if (dev_handle->dvh.jump) - flow_dv_jump_tbl_resource_release(dev, dev_handle); - if (dev_handle->dvh.port_id_action) - flow_dv_port_id_action_resource_release(dev_handle); - if (dev_handle->dvh.push_vlan_res) + if (dev_handle->dvh.rix_push_vlan) flow_dv_push_vlan_action_resource_release(dev, dev_handle); - if (dev_handle->dvh.tag_resource) + if (dev_handle->dvh.rix_tag) flow_dv_tag_release(dev, - dev_handle->dvh.tag_resource); - rte_free(dev_handle); + dev_handle->dvh.rix_tag); + flow_dv_fate_resource_release(dev, dev_handle); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], + tmp_idx); } } @@ -8587,11 +8748,9 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, claim_zero(mlx5_glue->dv_destroy_flow_matcher (mtd->egress.any_matcher)); if (mtd->egress.tbl) - claim_zero(flow_dv_tbl_resource_release(dev, - mtd->egress.tbl)); + flow_dv_tbl_resource_release(dev, mtd->egress.tbl); if (mtd->egress.sfx_tbl) - claim_zero(flow_dv_tbl_resource_release(dev, - mtd->egress.sfx_tbl)); + flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl); if (mtd->ingress.color_matcher) claim_zero(mlx5_glue->dv_destroy_flow_matcher (mtd->ingress.color_matcher)); @@ -8599,11 +8758,9 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, claim_zero(mlx5_glue->dv_destroy_flow_matcher (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) - claim_zero(flow_dv_tbl_resource_release(dev, - mtd->ingress.tbl)); + flow_dv_tbl_resource_release(dev, mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) - claim_zero(flow_dv_tbl_resource_release(dev, - mtd->ingress.sfx_tbl)); + flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl); if (mtd->transfer.color_matcher) claim_zero(mlx5_glue->dv_destroy_flow_matcher (mtd->transfer.color_matcher)); @@ -8611,11 +8768,9 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, claim_zero(mlx5_glue->dv_destroy_flow_matcher (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) - claim_zero(flow_dv_tbl_resource_release(dev, - mtd->transfer.tbl)); + flow_dv_tbl_resource_release(dev, mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) - claim_zero(flow_dv_tbl_resource_release(dev, - mtd->transfer.sfx_tbl)); + flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl); if (mtd->drop_actn) claim_zero(mlx5_glue->destroy_flow_action(mtd->drop_actn)); rte_free(mtd); @@ -8898,7 +9053,7 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, rte_col_2_mlx5_col(i), UINT8_MAX); if (mtb->count_actns[i]) actions[j++] = mtb->count_actns[i]; - if (fm->params.action[i] == MTR_POLICER_ACTION_DROP) + if (fm->action[i] == MTR_POLICER_ACTION_DROP) actions[j++] = mtb->drop_actn; else actions[j++] = dtb->jump_actn;