X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=b8d03d4df5a0f8b5a22ad7275825cd61f55b7f10;hb=2823b082f93c94c5c97fa572b5b84b637e088668;hp=f507358e4877b6ffa68885a4f15b40d4a575bb70;hpb=fba3213015891c7ec9634e66bec5eb8eaed0d70a;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index f507358e48..b8d03d4df5 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -27,12 +27,15 @@ #include #include #include +#include + +#include +#include +#include -#include "mlx5.h" #include "mlx5_defs.h" -#include "mlx5_glue.h" +#include "mlx5.h" #include "mlx5_flow.h" -#include "mlx5_prm.h" #include "mlx5_rxtx.h" #ifdef HAVE_IBV_FLOW_DV_SUPPORT @@ -73,27 +76,91 @@ union flow_dv_attr { /** * Initialize flow attributes structure according to flow items' types. * + * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel + * mode. For tunnel mode, the items to be modified are the outermost ones. + * * @param[in] item * Pointer to item specification. * @param[out] attr * Pointer to flow attributes structure. + * @param[in] dev_flow + * Pointer to the sub flow. + * @param[in] tunnel_decap + * Whether action is after tunnel decapsulation. */ static void -flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr) +flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr, + struct mlx5_flow *dev_flow, bool tunnel_decap) { + uint64_t layers = dev_flow->handle->layers; + + /* + * If layers is already initialized, it means this dev_flow is the + * suffix flow, the layers flags is set by the prefix flow. Need to + * use the layer flags from prefix flow as the suffix flow may not + * have the user defined items as the flow is split. + */ + if (layers) { + if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4) + attr->ipv4 = 1; + else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6) + attr->ipv6 = 1; + if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP) + attr->tcp = 1; + else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP) + attr->udp = 1; + attr->valid = 1; + return; + } for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + uint8_t next_protocol = 0xff; switch (item->type) { + case RTE_FLOW_ITEM_TYPE_GRE: + case RTE_FLOW_ITEM_TYPE_NVGRE: + case RTE_FLOW_ITEM_TYPE_VXLAN: + case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: + case RTE_FLOW_ITEM_TYPE_GENEVE: + case RTE_FLOW_ITEM_TYPE_MPLS: + if (tunnel_decap) + attr->attr = 0; + break; case RTE_FLOW_ITEM_TYPE_IPV4: - attr->ipv4 = 1; + if (!attr->ipv6) + attr->ipv4 = 1; + if (item->mask != NULL && + ((const struct rte_flow_item_ipv4 *) + item->mask)->hdr.next_proto_id) + next_protocol = + ((const struct rte_flow_item_ipv4 *) + (item->spec))->hdr.next_proto_id & + ((const struct rte_flow_item_ipv4 *) + (item->mask))->hdr.next_proto_id; + if ((next_protocol == IPPROTO_IPIP || + next_protocol == IPPROTO_IPV6) && tunnel_decap) + attr->attr = 0; break; case RTE_FLOW_ITEM_TYPE_IPV6: - attr->ipv6 = 1; + if (!attr->ipv4) + attr->ipv6 = 1; + if (item->mask != NULL && + ((const struct rte_flow_item_ipv6 *) + item->mask)->hdr.proto) + next_protocol = + ((const struct rte_flow_item_ipv6 *) + (item->spec))->hdr.proto & + ((const struct rte_flow_item_ipv6 *) + (item->mask))->hdr.proto; + if ((next_protocol == IPPROTO_IPIP || + next_protocol == IPPROTO_IPV6) && tunnel_decap) + attr->attr = 0; break; case RTE_FLOW_ITEM_TYPE_UDP: - attr->udp = 1; + if (!attr->tcp) + attr->udp = 1; break; case RTE_FLOW_ITEM_TYPE_TCP: - attr->tcp = 1; + if (!attr->udp) + attr->tcp = 1; break; default: break; @@ -102,6 +169,31 @@ flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr) attr->valid = 1; } +/** + * Convert rte_mtr_color to mlx5 color. + * + * @param[in] rcol + * rte_mtr_color. + * + * @return + * mlx5 color. + */ +static int +rte_col_2_mlx5_col(enum rte_color rcol) +{ + switch (rcol) { + case RTE_COLOR_GREEN: + return MLX5_FLOW_COLOR_GREEN; + case RTE_COLOR_YELLOW: + return MLX5_FLOW_COLOR_YELLOW; + case RTE_COLOR_RED: + return MLX5_FLOW_COLOR_RED; + default: + break; + } + return MLX5_FLOW_COLOR_UNDEFINED; +} + struct field_modify_info { uint32_t size; /* Size of field in protocol header, in bytes. */ uint32_t offset; /* Offset of field in protocol header, in bytes. */ @@ -123,6 +215,7 @@ struct field_modify_info modify_vlan_out_first_vid[] = { }; struct field_modify_info modify_ipv4[] = { + {1, 1, MLX5_MODI_OUT_IP_DSCP}, {1, 8, MLX5_MODI_OUT_IPV4_TTL}, {4, 12, MLX5_MODI_OUT_SIPV4}, {4, 16, MLX5_MODI_OUT_DIPV4}, @@ -130,6 +223,7 @@ struct field_modify_info modify_ipv4[] = { }; struct field_modify_info modify_ipv6[] = { + {1, 0, MLX5_MODI_OUT_IP_DSCP}, {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT}, {4, 8, MLX5_MODI_OUT_SIPV6_127_96}, {4, 12, MLX5_MODI_OUT_SIPV6_95_64}, @@ -161,8 +255,8 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, uint8_t next_protocol, uint64_t *item_flags, int *tunnel) { - assert(item->type == RTE_FLOW_ITEM_TYPE_IPV4 || - item->type == RTE_FLOW_ITEM_TYPE_IPV6); + MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 || + item->type == RTE_FLOW_ITEM_TYPE_IPV6); if (next_protocol == IPPROTO_IPIP) { *item_flags |= MLX5_FLOW_LAYER_IPIP; *tunnel = 1; @@ -183,7 +277,7 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, * Pointer to the rte_eth_dev structure. */ static void -flow_d_shared_lock(struct rte_eth_dev *dev) +flow_dv_shared_lock(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; @@ -192,13 +286,13 @@ flow_d_shared_lock(struct rte_eth_dev *dev) int ret; ret = pthread_mutex_lock(&sh->dv_mutex); - assert(!ret); + MLX5_ASSERT(!ret); (void)ret; } } static void -flow_d_shared_unlock(struct rte_eth_dev *dev) +flow_dv_shared_unlock(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; @@ -207,7 +301,7 @@ flow_d_shared_unlock(struct rte_eth_dev *dev) int ret; ret = pthread_mutex_unlock(&sh->dv_mutex); - assert(!ret); + MLX5_ASSERT(!ret); (void)ret; } } @@ -239,13 +333,63 @@ mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action, } } +/** + * Fetch 1, 2, 3 or 4 byte field from the byte array + * and return as unsigned integer in host-endian format. + * + * @param[in] data + * Pointer to data array. + * @param[in] size + * Size of field to extract. + * + * @return + * converted field in host endian format. + */ +static inline uint32_t +flow_dv_fetch_field(const uint8_t *data, uint32_t size) +{ + uint32_t ret; + + switch (size) { + case 1: + ret = *data; + break; + case 2: + ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); + break; + case 3: + ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); + ret = (ret << 8) | *(data + sizeof(uint16_t)); + break; + case 4: + ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data); + break; + default: + MLX5_ASSERT(false); + ret = 0; + break; + } + return ret; +} + /** * Convert modify-header action to DV specification. * + * Data length of each action is determined by provided field description + * and the item mask. Data bit offset and width of each action is determined + * by provided item mask. + * * @param[in] item * Pointer to item specification. * @param[in] field * Pointer to field modification information. + * For MLX5_MODIFICATION_TYPE_SET specifies destination field. + * For MLX5_MODIFICATION_TYPE_ADD specifies destination field. + * For MLX5_MODIFICATION_TYPE_COPY specifies source field. + * @param[in] dcopy + * Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type. + * Negative offset value sets the same offset as source offset. + * size field is ignored, value is taken from source field. * @param[in,out] resource * Pointer to the modify-header resource. * @param[in] type @@ -259,42 +403,74 @@ mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action, static int flow_dv_convert_modify_action(struct rte_flow_item *item, struct field_modify_info *field, + struct field_modify_info *dcopy, struct mlx5_flow_dv_modify_hdr_resource *resource, - uint32_t type, - struct rte_flow_error *error) + uint32_t type, struct rte_flow_error *error) { uint32_t i = resource->actions_num; struct mlx5_modification_cmd *actions = resource->actions; - const uint8_t *spec = item->spec; - const uint8_t *mask = item->mask; - uint32_t set; - - while (field->size) { - set = 0; - /* Generate modify command for each mask segment. */ - memcpy(&set, &mask[field->offset], field->size); - if (set) { - if (i >= MLX5_MODIFY_NUM) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "too many items to modify"); - actions[i].action_type = type; - actions[i].field = field->id; - actions[i].length = field->size == - 4 ? 0 : field->size * 8; - rte_memcpy(&actions[i].data[4 - field->size], - &spec[field->offset], field->size); - actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); - ++i; + + /* + * The item and mask are provided in big-endian format. + * The fields should be presented as in big-endian format either. + * Mask must be always present, it defines the actual field width. + */ + MLX5_ASSERT(item->mask); + MLX5_ASSERT(field->size); + do { + unsigned int size_b; + unsigned int off_b; + uint32_t mask; + uint32_t data; + + if (i >= MLX5_MAX_MODIFY_NUM) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "too many items to modify"); + /* Fetch variable byte size mask from the array. */ + mask = flow_dv_fetch_field((const uint8_t *)item->mask + + field->offset, field->size); + if (!mask) { + ++field; + continue; } - if (resource->actions_num != i) - resource->actions_num = i; - field++; - } - if (!resource->actions_num) + /* Deduce actual data width in bits from mask value. */ + off_b = rte_bsf32(mask); + size_b = sizeof(uint32_t) * CHAR_BIT - + off_b - __builtin_clz(mask); + MLX5_ASSERT(size_b); + size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b; + actions[i] = (struct mlx5_modification_cmd) { + .action_type = type, + .field = field->id, + .offset = off_b, + .length = size_b, + }; + /* Convert entire record to expected big-endian format. */ + actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); + if (type == MLX5_MODIFICATION_TYPE_COPY) { + MLX5_ASSERT(dcopy); + actions[i].dst_field = dcopy->id; + actions[i].dst_offset = + (int)dcopy->offset < 0 ? off_b : dcopy->offset; + /* Convert entire record to big-endian format. */ + actions[i].data1 = rte_cpu_to_be_32(actions[i].data1); + } else { + MLX5_ASSERT(item->spec); + data = flow_dv_fetch_field((const uint8_t *)item->spec + + field->offset, field->size); + /* Shift out the trailing masked bits from data. */ + data = (data & mask) >> off_b; + actions[i].data1 = rte_cpu_to_be_32(data); + } + ++i; + ++field; + } while (field->size); + if (resource->actions_num == i) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "invalid modification flow item"); + resource->actions_num = i; return 0; } @@ -334,7 +510,7 @@ flow_dv_convert_action_modify_ipv4 } item.spec = &ipv4; item.mask = &ipv4_mask; - return flow_dv_convert_modify_action(&item, modify_ipv4, resource, + return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, MLX5_MODIFICATION_TYPE_SET, error); } @@ -380,7 +556,7 @@ flow_dv_convert_action_modify_ipv6 } item.spec = &ipv6; item.mask = &ipv6_mask; - return flow_dv_convert_modify_action(&item, modify_ipv6, resource, + return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, MLX5_MODIFICATION_TYPE_SET, error); } @@ -426,7 +602,7 @@ flow_dv_convert_action_modify_mac } item.spec = ð item.mask = ð_mask; - return flow_dv_convert_modify_action(&item, modify_eth, resource, + return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource, MLX5_MODIFICATION_TYPE_SET, error); } @@ -452,17 +628,19 @@ flow_dv_convert_action_modify_vlan_vid const struct rte_flow_action_of_set_vlan_vid *conf = (const struct rte_flow_action_of_set_vlan_vid *)(action->conf); int i = resource->actions_num; - struct mlx5_modification_cmd *actions = &resource->actions[i]; + struct mlx5_modification_cmd *actions = resource->actions; struct field_modify_info *field = modify_vlan_out_first_vid; - if (i >= MLX5_MODIFY_NUM) + if (i >= MLX5_MAX_MODIFY_NUM) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many items to modify"); - actions[i].action_type = MLX5_MODIFICATION_TYPE_SET; - actions[i].field = field->id; - actions[i].length = field->size; - actions[i].offset = field->offset; + actions[i] = (struct mlx5_modification_cmd) { + .action_type = MLX5_MODIFICATION_TYPE_SET, + .field = field->id, + .length = field->size, + .offset = field->offset, + }; actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); actions[i].data1 = conf->vlan_vid; actions[i].data1 = actions[i].data1 << 16; @@ -481,6 +659,10 @@ flow_dv_convert_action_modify_vlan_vid * Pointer to rte_flow_item objects list. * @param[in] attr * Pointer to flow attributes structure. + * @param[in] dev_flow + * Pointer to the sub flow. + * @param[in] tunnel_decap + * Whether action is after tunnel decapsulation. * @param[out] error * Pointer to the error structure. * @@ -492,8 +674,8 @@ flow_dv_convert_action_modify_tp (struct mlx5_flow_dv_modify_hdr_resource *resource, const struct rte_flow_action *action, const struct rte_flow_item *items, - union flow_dv_attr *attr, - struct rte_flow_error *error) + union flow_dv_attr *attr, struct mlx5_flow *dev_flow, + bool tunnel_decap, struct rte_flow_error *error) { const struct rte_flow_action_set_tp *conf = (const struct rte_flow_action_set_tp *)(action->conf); @@ -505,7 +687,7 @@ flow_dv_convert_action_modify_tp struct field_modify_info *field; if (!attr->valid) - flow_dv_attr_init(items, attr); + flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); if (attr->udp) { memset(&udp, 0, sizeof(udp)); memset(&udp_mask, 0, sizeof(udp_mask)); @@ -522,8 +704,8 @@ flow_dv_convert_action_modify_tp item.spec = &udp; item.mask = &udp_mask; field = modify_udp; - } - if (attr->tcp) { + } else { + MLX5_ASSERT(attr->tcp); memset(&tcp, 0, sizeof(tcp)); memset(&tcp_mask, 0, sizeof(tcp_mask)); if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { @@ -540,7 +722,7 @@ flow_dv_convert_action_modify_tp item.mask = &tcp_mask; field = modify_tcp; } - return flow_dv_convert_modify_action(&item, field, resource, + return flow_dv_convert_modify_action(&item, field, NULL, resource, MLX5_MODIFICATION_TYPE_SET, error); } @@ -555,6 +737,10 @@ flow_dv_convert_action_modify_tp * Pointer to rte_flow_item objects list. * @param[in] attr * Pointer to flow attributes structure. + * @param[in] dev_flow + * Pointer to the sub flow. + * @param[in] tunnel_decap + * Whether action is after tunnel decapsulation. * @param[out] error * Pointer to the error structure. * @@ -566,8 +752,8 @@ flow_dv_convert_action_modify_ttl (struct mlx5_flow_dv_modify_hdr_resource *resource, const struct rte_flow_action *action, const struct rte_flow_item *items, - union flow_dv_attr *attr, - struct rte_flow_error *error) + union flow_dv_attr *attr, struct mlx5_flow *dev_flow, + bool tunnel_decap, struct rte_flow_error *error) { const struct rte_flow_action_set_ttl *conf = (const struct rte_flow_action_set_ttl *)(action->conf); @@ -579,7 +765,7 @@ flow_dv_convert_action_modify_ttl struct field_modify_info *field; if (!attr->valid) - flow_dv_attr_init(items, attr); + flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); if (attr->ipv4) { memset(&ipv4, 0, sizeof(ipv4)); memset(&ipv4_mask, 0, sizeof(ipv4_mask)); @@ -589,8 +775,8 @@ flow_dv_convert_action_modify_ttl item.spec = &ipv4; item.mask = &ipv4_mask; field = modify_ipv4; - } - if (attr->ipv6) { + } else { + MLX5_ASSERT(attr->ipv6); memset(&ipv6, 0, sizeof(ipv6)); memset(&ipv6_mask, 0, sizeof(ipv6_mask)); ipv6.hdr.hop_limits = conf->ttl_value; @@ -600,7 +786,7 @@ flow_dv_convert_action_modify_ttl item.mask = &ipv6_mask; field = modify_ipv6; } - return flow_dv_convert_modify_action(&item, field, resource, + return flow_dv_convert_modify_action(&item, field, NULL, resource, MLX5_MODIFICATION_TYPE_SET, error); } @@ -615,6 +801,10 @@ flow_dv_convert_action_modify_ttl * Pointer to rte_flow_item objects list. * @param[in] attr * Pointer to flow attributes structure. + * @param[in] dev_flow + * Pointer to the sub flow. + * @param[in] tunnel_decap + * Whether action is after tunnel decapsulation. * @param[out] error * Pointer to the error structure. * @@ -625,8 +815,8 @@ static int flow_dv_convert_action_modify_dec_ttl (struct mlx5_flow_dv_modify_hdr_resource *resource, const struct rte_flow_item *items, - union flow_dv_attr *attr, - struct rte_flow_error *error) + union flow_dv_attr *attr, struct mlx5_flow *dev_flow, + bool tunnel_decap, struct rte_flow_error *error) { struct rte_flow_item item; struct rte_flow_item_ipv4 ipv4; @@ -636,7 +826,7 @@ flow_dv_convert_action_modify_dec_ttl struct field_modify_info *field; if (!attr->valid) - flow_dv_attr_init(items, attr); + flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); if (attr->ipv4) { memset(&ipv4, 0, sizeof(ipv4)); memset(&ipv4_mask, 0, sizeof(ipv4_mask)); @@ -646,8 +836,8 @@ flow_dv_convert_action_modify_dec_ttl item.spec = &ipv4; item.mask = &ipv4_mask; field = modify_ipv4; - } - if (attr->ipv6) { + } else { + MLX5_ASSERT(attr->ipv6); memset(&ipv6, 0, sizeof(ipv6)); memset(&ipv6_mask, 0, sizeof(ipv6_mask)); ipv6.hdr.hop_limits = 0xFF; @@ -657,7 +847,7 @@ flow_dv_convert_action_modify_dec_ttl item.mask = &ipv6_mask; field = modify_ipv6; } - return flow_dv_convert_modify_action(&item, field, resource, + return flow_dv_convert_modify_action(&item, field, NULL, resource, MLX5_MODIFICATION_TYPE_ADD, error); } @@ -702,7 +892,7 @@ flow_dv_convert_action_modify_tcp_seq item.type = RTE_FLOW_ITEM_TYPE_TCP; item.spec = &tcp; item.mask = &tcp_mask; - return flow_dv_convert_modify_action(&item, modify_tcp, resource, + return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, MLX5_MODIFICATION_TYPE_ADD, error); } @@ -747,11 +937,12 @@ flow_dv_convert_action_modify_tcp_ack item.type = RTE_FLOW_ITEM_TYPE_TCP; item.spec = &tcp; item.mask = &tcp_mask; - return flow_dv_convert_modify_action(&item, modify_tcp, resource, + return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, MLX5_MODIFICATION_TYPE_ADD, error); } static enum mlx5_modification_field reg_to_field[] = { + [REG_NONE] = MLX5_MODI_OUT_NONE, [REG_A] = MLX5_MODI_META_DATA_REG_A, [REG_B] = MLX5_MODI_META_DATA_REG_B, [REG_C_0] = MLX5_MODI_META_REG_C_0, @@ -783,235 +974,789 @@ flow_dv_convert_action_set_reg const struct rte_flow_action *action, struct rte_flow_error *error) { - const struct mlx5_rte_flow_action_set_tag *conf = (action->conf); + const struct mlx5_rte_flow_action_set_tag *conf = action->conf; struct mlx5_modification_cmd *actions = resource->actions; uint32_t i = resource->actions_num; - if (i >= MLX5_MODIFY_NUM) + if (i >= MLX5_MAX_MODIFY_NUM) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many items to modify"); - actions[i].action_type = MLX5_MODIFICATION_TYPE_SET; - actions[i].field = reg_to_field[conf->id]; + MLX5_ASSERT(conf->id != REG_NONE); + MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field)); + actions[i] = (struct mlx5_modification_cmd) { + .action_type = MLX5_MODIFICATION_TYPE_SET, + .field = reg_to_field[conf->id], + }; actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); - actions[i].data1 = conf->data; + actions[i].data1 = rte_cpu_to_be_32(conf->data); ++i; resource->actions_num = i; - if (!resource->actions_num) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "invalid modification flow item"); return 0; } /** - * Validate META item. + * Convert SET_TAG action to DV specification. * * @param[in] dev * Pointer to the rte_eth_dev structure. - * @param[in] item - * Item specification. - * @param[in] attr - * Attributes of flow that includes this item. + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] conf + * Pointer to action specification. * @param[out] error - * Pointer to error structure. + * Pointer to the error structure. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, - const struct rte_flow_item *item, - const struct rte_flow_attr *attr, - struct rte_flow_error *error) +flow_dv_convert_action_set_tag + (struct rte_eth_dev *dev, + struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action_set_tag *conf, + struct rte_flow_error *error) { - const struct rte_flow_item_meta *spec = item->spec; - const struct rte_flow_item_meta *mask = item->mask; - const struct rte_flow_item_meta nic_mask = { - .data = UINT32_MAX + rte_be32_t data = rte_cpu_to_be_32(conf->data); + rte_be32_t mask = rte_cpu_to_be_32(conf->mask); + struct rte_flow_item item = { + .spec = &data, + .mask = &mask, + }; + struct field_modify_info reg_c_x[] = { + [1] = {0, 0, 0}, }; + enum mlx5_modification_field reg_type; int ret; - if (!spec) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM_SPEC, - item->spec, - "data cannot be empty"); - if (!spec->data) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM_SPEC, - NULL, - "data cannot be zero"); - if (!mask) - mask = &rte_flow_item_meta_mask; - ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, - (const uint8_t *)&nic_mask, - sizeof(struct rte_flow_item_meta), - error); + ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); if (ret < 0) return ret; - if (attr->ingress) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, - NULL, - "pattern not supported for ingress"); - return 0; + MLX5_ASSERT(ret != REG_NONE); + MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field)); + reg_type = reg_to_field[ret]; + MLX5_ASSERT(reg_type > 0); + reg_c_x[0] = (struct field_modify_info){4, 0, reg_type}; + return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, + MLX5_MODIFICATION_TYPE_SET, error); } /** - * Validate vport item. + * Convert internal COPY_REG action to DV specification. * * @param[in] dev * Pointer to the rte_eth_dev structure. - * @param[in] item - * Item specification. - * @param[in] attr - * Attributes of flow that includes this item. - * @param[in] item_flags - * Bit-fields that holds the items detected until now. + * @param[in,out] res + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. * @param[out] error - * Pointer to error structure. + * Pointer to the error structure. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_item_port_id(struct rte_eth_dev *dev, - const struct rte_flow_item *item, - const struct rte_flow_attr *attr, - uint64_t item_flags, - struct rte_flow_error *error) +flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev, + struct mlx5_flow_dv_modify_hdr_resource *res, + const struct rte_flow_action *action, + struct rte_flow_error *error) { - const struct rte_flow_item_port_id *spec = item->spec; - const struct rte_flow_item_port_id *mask = item->mask; - const struct rte_flow_item_port_id switch_mask = { - .id = 0xffffffff, + const struct mlx5_flow_action_copy_mreg *conf = action->conf; + rte_be32_t mask = RTE_BE32(UINT32_MAX); + struct rte_flow_item item = { + .spec = NULL, + .mask = &mask, }; - struct mlx5_priv *esw_priv; - struct mlx5_priv *dev_priv; - int ret; - - if (!attr->transfer) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - NULL, - "match on port id is valid only" - " when transfer flag is enabled"); - if (item_flags & MLX5_FLOW_ITEM_PORT_ID) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ITEM, item, - "multiple source ports are not" - " supported"); - if (!mask) - mask = &switch_mask; - if (mask->id != 0xffffffff) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ITEM_MASK, - mask, - "no support for partial mask on" - " \"id\" field"); - ret = mlx5_flow_item_acceptable - (item, (const uint8_t *)mask, - (const uint8_t *)&rte_flow_item_port_id_mask, - sizeof(struct rte_flow_item_port_id), - error); - if (ret) - return ret; - if (!spec) - return 0; - esw_priv = mlx5_port_to_eswitch_info(spec->id); - if (!esw_priv) - return rte_flow_error_set(error, rte_errno, - RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, - "failed to obtain E-Switch info for" - " port"); - dev_priv = mlx5_dev_to_eswitch_info(dev); - if (!dev_priv) - return rte_flow_error_set(error, rte_errno, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "failed to obtain E-Switch info"); - if (esw_priv->domain_id != dev_priv->domain_id) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, - "cannot match on a port from a" - " different E-Switch"); - return 0; + struct field_modify_info reg_src[] = { + {4, 0, reg_to_field[conf->src]}, + {0, 0, 0}, + }; + struct field_modify_info reg_dst = { + .offset = 0, + .id = reg_to_field[conf->dst], + }; + /* Adjust reg_c[0] usage according to reported mask. */ + if (conf->dst == REG_C_0 || conf->src == REG_C_0) { + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t reg_c0 = priv->sh->dv_regc0_mask; + + MLX5_ASSERT(reg_c0); + MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY); + if (conf->dst == REG_C_0) { + /* Copy to reg_c[0], within mask only. */ + reg_dst.offset = rte_bsf32(reg_c0); + /* + * Mask is ignoring the enianness, because + * there is no conversion in datapath. + */ +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + /* Copy from destination lower bits to reg_c[0]. */ + mask = reg_c0 >> reg_dst.offset; +#else + /* Copy from destination upper bits to reg_c[0]. */ + mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT - + rte_fls_u32(reg_c0)); +#endif + } else { + mask = rte_cpu_to_be_32(reg_c0); +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + /* Copy from reg_c[0] to destination lower bits. */ + reg_dst.offset = 0; +#else + /* Copy from reg_c[0] to destination upper bits. */ + reg_dst.offset = sizeof(reg_c0) * CHAR_BIT - + (rte_fls_u32(reg_c0) - + rte_bsf32(reg_c0)); +#endif + } + } + return flow_dv_convert_modify_action(&item, + reg_src, ®_dst, res, + MLX5_MODIFICATION_TYPE_COPY, + error); } /** - * Validate the pop VLAN action. + * Convert MARK action to DV specification. This routine is used + * in extensive metadata only and requires metadata register to be + * handled. In legacy mode hardware tag resource is engaged. * * @param[in] dev * Pointer to the rte_eth_dev structure. - * @param[in] action_flags - * Holds the actions detected until now. - * @param[in] action - * Pointer to the pop vlan action. - * @param[in] item_flags - * The items found in this flow rule. - * @param[in] attr - * Pointer to flow attributes. + * @param[in] conf + * Pointer to MARK action specification. + * @param[in,out] resource + * Pointer to the modify-header resource. * @param[out] error - * Pointer to error structure. + * Pointer to the error structure. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, - uint64_t action_flags, - const struct rte_flow_action *action, - uint64_t item_flags, - const struct rte_flow_attr *attr, - struct rte_flow_error *error) +flow_dv_convert_action_mark(struct rte_eth_dev *dev, + const struct rte_flow_action_mark *conf, + struct mlx5_flow_dv_modify_hdr_resource *resource, + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK & + priv->sh->dv_mark_mask); + rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask; + struct rte_flow_item item = { + .spec = &data, + .mask = &mask, + }; + struct field_modify_info reg_c_x[] = { + {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */ + {0, 0, 0}, + }; + int reg; - (void)action; - (void)attr; - if (!priv->sh->pop_vlan_action) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "pop vlan action is not supported"); - /* - * Check for inconsistencies: - * fail strip_vlan in a flow that matches packets without VLAN tags. - * fail strip_vlan in a flow that matches packets without explicitly a - * matching on VLAN tag ? - */ - if (action_flags & MLX5_FLOW_ACTION_OF_POP_VLAN) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "no support for multiple vlan pop " - "actions"); - if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "cannot pop vlan without a " - "match on (outer) vlan in the flow"); - if (action_flags & MLX5_FLOW_ACTION_PORT_ID) + if (!mask) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, "zero mark action mask"); + reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); + if (reg < 0) + return reg; + MLX5_ASSERT(reg > 0); + if (reg == REG_C_0) { + uint32_t msk_c0 = priv->sh->dv_regc0_mask; + uint32_t shl_c0 = rte_bsf32(msk_c0); + + data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0); + mask = rte_cpu_to_be_32(mask) & msk_c0; + mask = rte_cpu_to_be_32(mask << shl_c0); + } + reg_c_x[0].id = reg_to_field[reg]; + return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Get metadata register index for specified steering domain. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] attr + * Attributes of flow to determine steering domain. + * @param[out] error + * Pointer to the error structure. + * + * @return + * positive index on success, a negative errno value otherwise + * and rte_errno is set. + */ +static enum modify_reg +flow_dv_get_metadata_reg(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + int reg = + mlx5_flow_get_reg_id(dev, attr->transfer ? + MLX5_METADATA_FDB : + attr->egress ? + MLX5_METADATA_TX : + MLX5_METADATA_RX, 0, error); + if (reg < 0) + return rte_flow_error_set(error, + ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "unavailable " + "metadata register"); + return reg; +} + +/** + * Convert SET_META action to DV specification. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[in] conf + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_set_meta + (struct rte_eth_dev *dev, + struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_attr *attr, + const struct rte_flow_action_set_meta *conf, + struct rte_flow_error *error) +{ + uint32_t data = conf->data; + uint32_t mask = conf->mask; + struct rte_flow_item item = { + .spec = &data, + .mask = &mask, + }; + struct field_modify_info reg_c_x[] = { + [1] = {0, 0, 0}, + }; + int reg = flow_dv_get_metadata_reg(dev, attr, error); + + if (reg < 0) + return reg; + /* + * In datapath code there is no endianness + * coversions for perfromance reasons, all + * pattern conversions are done in rte_flow. + */ + if (reg == REG_C_0) { + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t msk_c0 = priv->sh->dv_regc0_mask; + uint32_t shl_c0; + + MLX5_ASSERT(msk_c0); +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + shl_c0 = rte_bsf32(msk_c0); +#else + shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0); +#endif + mask <<= shl_c0; + data <<= shl_c0; + MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask))); + } + reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; + /* The routine expects parameters in memory as big-endian ones. */ + return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Convert modify-header set IPv4 DSCP action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_ipv4_dscp + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_dscp *conf = + (const struct rte_flow_action_set_dscp *)(action->conf); + struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; + struct rte_flow_item_ipv4 ipv4; + struct rte_flow_item_ipv4 ipv4_mask; + + memset(&ipv4, 0, sizeof(ipv4)); + memset(&ipv4_mask, 0, sizeof(ipv4_mask)); + ipv4.hdr.type_of_service = conf->dscp; + ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2; + item.spec = &ipv4; + item.mask = &ipv4_mask; + return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Convert modify-header set IPv6 DSCP action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_ipv6_dscp + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_dscp *conf = + (const struct rte_flow_action_set_dscp *)(action->conf); + struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; + struct rte_flow_item_ipv6 ipv6; + struct rte_flow_item_ipv6 ipv6_mask; + + memset(&ipv6, 0, sizeof(ipv6)); + memset(&ipv6_mask, 0, sizeof(ipv6_mask)); + /* + * Even though the DSCP bits offset of IPv6 is not byte aligned, + * rdma-core only accept the DSCP bits byte aligned start from + * bit 0 to 5 as to be compatible with IPv4. No need to shift the + * bits in IPv6 case as rdma-core requires byte aligned value. + */ + ipv6.hdr.vtc_flow = conf->dscp; + ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22; + item.spec = &ipv6; + item.mask = &ipv6_mask; + return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Validate MARK item. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] item + * Item specification. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_mark(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + const struct rte_flow_attr *attr __rte_unused, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; + const struct rte_flow_item_mark *spec = item->spec; + const struct rte_flow_item_mark *mask = item->mask; + const struct rte_flow_item_mark nic_mask = { + .id = priv->sh->dv_mark_mask, + }; + int ret; + + if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "extended metadata feature" + " isn't enabled"); + if (!mlx5_flow_ext_mreg_supported(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "extended metadata register" + " isn't supported"); + if (!nic_mask.id) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "extended metadata register" + " isn't available"); + ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); + if (ret < 0) + return ret; + if (!spec) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, + item->spec, + "data cannot be empty"); + if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + &spec->id, + "mark id exceeds the limit"); + if (!mask) + mask = &nic_mask; + if (!mask->id) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, + "mask cannot be zero"); + + ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_mark), + error); + if (ret < 0) + return ret; + return 0; +} + +/** + * Validate META item. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] item + * Item specification. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, + const struct rte_flow_item *item, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; + const struct rte_flow_item_meta *spec = item->spec; + const struct rte_flow_item_meta *mask = item->mask; + struct rte_flow_item_meta nic_mask = { + .data = UINT32_MAX + }; + int reg; + int ret; + + if (!spec) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, + item->spec, + "data cannot be empty"); + if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { + if (!mlx5_flow_ext_mreg_supported(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "extended metadata register" + " isn't supported"); + reg = flow_dv_get_metadata_reg(dev, attr, error); + if (reg < 0) + return reg; + if (reg == REG_B) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "match on reg_b " + "isn't supported"); + if (reg != REG_A) + nic_mask.data = priv->sh->dv_meta_mask; + } + if (!mask) + mask = &rte_flow_item_meta_mask; + if (!mask->data) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, + "mask cannot be zero"); + + ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_meta), + error); + return ret; +} + +/** + * Validate TAG item. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] item + * Item specification. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_tag(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + const struct rte_flow_attr *attr __rte_unused, + struct rte_flow_error *error) +{ + const struct rte_flow_item_tag *spec = item->spec; + const struct rte_flow_item_tag *mask = item->mask; + const struct rte_flow_item_tag nic_mask = { + .data = RTE_BE32(UINT32_MAX), + .index = 0xff, + }; + int ret; + + if (!mlx5_flow_ext_mreg_supported(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "extensive metadata register" + " isn't supported"); + if (!spec) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, + item->spec, + "data cannot be empty"); + if (!mask) + mask = &rte_flow_item_tag_mask; + if (!mask->data) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, + "mask cannot be zero"); + + ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_tag), + error); + if (ret < 0) + return ret; + if (mask->index != 0xff) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, + "partial mask for tag index" + " is not supported"); + ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error); + if (ret < 0) + return ret; + MLX5_ASSERT(ret != REG_NONE); + return 0; +} + +/** + * Validate vport item. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] item + * Item specification. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[in] item_flags + * Bit-fields that holds the items detected until now. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_port_id(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + const struct rte_flow_attr *attr, + uint64_t item_flags, + struct rte_flow_error *error) +{ + const struct rte_flow_item_port_id *spec = item->spec; + const struct rte_flow_item_port_id *mask = item->mask; + const struct rte_flow_item_port_id switch_mask = { + .id = 0xffffffff, + }; + struct mlx5_priv *esw_priv; + struct mlx5_priv *dev_priv; + int ret; + + if (!attr->transfer) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, + "match on port id is valid only" + " when transfer flag is enabled"); + if (item_flags & MLX5_FLOW_ITEM_PORT_ID) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "multiple source ports are not" + " supported"); + if (!mask) + mask = &switch_mask; + if (mask->id != 0xffffffff) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM_MASK, + mask, + "no support for partial mask on" + " \"id\" field"); + ret = mlx5_flow_item_acceptable + (item, (const uint8_t *)mask, + (const uint8_t *)&rte_flow_item_port_id_mask, + sizeof(struct rte_flow_item_port_id), + error); + if (ret) + return ret; + if (!spec) + return 0; + esw_priv = mlx5_port_to_eswitch_info(spec->id, false); + if (!esw_priv) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, + "failed to obtain E-Switch info for" + " port"); + dev_priv = mlx5_dev_to_eswitch_info(dev); + if (!dev_priv) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "failed to obtain E-Switch info"); + if (esw_priv->domain_id != dev_priv->domain_id) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, + "cannot match on a port from a" + " different E-Switch"); + return 0; +} + +/** + * Validate GTP item. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] item + * Item specification. + * @param[in] item_flags + * Bit-fields that holds the items detected until now. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_gtp(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + uint64_t item_flags, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_item_gtp *mask = item->mask; + const struct rte_flow_item_gtp nic_mask = { + .msg_type = 0xff, + .teid = RTE_BE32(0xffffffff), + }; + + if (!priv->config.hca_attr.tunnel_stateless_gtp) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "GTP support is not enabled"); + if (item_flags & MLX5_FLOW_LAYER_TUNNEL) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "multiple tunnel layers not" + " supported"); + if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "no outer UDP layer found"); + if (!mask) + mask = &rte_flow_item_gtp_mask; + return mlx5_flow_item_acceptable + (item, (const uint8_t *)mask, + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_gtp), + error); +} + +/** + * Validate the pop VLAN action. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the pop vlan action. + * @param[in] item_flags + * The items found in this flow rule. + * @param[in] attr + * Pointer to flow attributes. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, + uint64_t action_flags, + const struct rte_flow_action *action, + uint64_t item_flags, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + const struct mlx5_priv *priv = dev->data->dev_private; + + (void)action; + (void)attr; + if (!priv->sh->pop_vlan_action) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "pop vlan action is not supported"); + if (attr->egress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "pop vlan action not supported for " + "egress"); + if (action_flags & MLX5_FLOW_VLAN_ACTIONS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "no support for multiple VLAN " + "actions"); + if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot pop vlan without a " + "match on (outer) vlan in the flow"); + if (action_flags & MLX5_FLOW_ACTION_PORT_ID) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "wrong action order, port_id should " "be after pop VLAN action"); + if (!attr->transfer && priv->representor) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "pop vlan action for VF representor " + "not supported on NIC table"); return 0; } /** * Get VLAN default info from vlan match info. * - * @param[in] dev - * Pointer to the rte_eth_dev structure. - * @param[in] item + * @param[in] items * the list of item specifications. * @param[out] vlan * pointer VLAN info to fill to. - * @param[out] error - * Pointer to error structure. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. @@ -1028,10 +1773,14 @@ flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, if (items == NULL) return; - for (; items->type != RTE_FLOW_ITEM_TYPE_END && - items->type != RTE_FLOW_ITEM_TYPE_VLAN; items++) - ; - if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) { + for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { + int type = items->type; + + if (type == RTE_FLOW_ITEM_TYPE_VLAN || + type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN) + break; + } + if (items->type != RTE_FLOW_ITEM_TYPE_END) { const struct rte_flow_item_vlan *vlan_m = items->mask; const struct rte_flow_item_vlan *vlan_v = items->spec; @@ -1040,7 +1789,7 @@ flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, /* Only full match values are accepted */ if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) == MLX5DV_FLOW_VLAN_PCP_MASK_BE) { - vlan->vlan_tci &= MLX5DV_FLOW_VLAN_PCP_MASK; + vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; vlan->vlan_tci |= rte_be_to_cpu_16(vlan_v->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE); @@ -1059,12 +1808,220 @@ flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, } /** - * Validate the push VLAN action. + * Validate the push VLAN action. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] item_flags + * The items found in this flow rule. + * @param[in] action + * Pointer to the action structure. + * @param[in] attr + * Pointer to flow attributes + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev, + uint64_t action_flags, + uint64_t item_flags __rte_unused, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + const struct rte_flow_action_of_push_vlan *push_vlan = action->conf; + const struct mlx5_priv *priv = dev->data->dev_private; + + if (!attr->transfer && attr->ingress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + NULL, + "push VLAN action not supported for " + "ingress"); + if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) && + push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "invalid vlan ethertype"); + if (action_flags & MLX5_FLOW_VLAN_ACTIONS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "no support for multiple VLAN " + "actions"); + if (action_flags & MLX5_FLOW_ACTION_PORT_ID) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "wrong action order, port_id should " + "be after push VLAN"); + if (!attr->transfer && priv->representor) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "push vlan action for VF representor " + "not supported on NIC table"); + (void)attr; + return 0; +} + +/** + * Validate the set VLAN PCP. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] actions + * Pointer to the list of actions remaining in the flow rule. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags, + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + const struct rte_flow_action *action = actions; + const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf; + + if (conf->vlan_pcp > 7) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "VLAN PCP value is too big"); + if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "set VLAN PCP action must follow " + "the push VLAN action"); + if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "Multiple VLAN PCP modification are " + "not supported"); + if (action_flags & MLX5_FLOW_ACTION_PORT_ID) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "wrong action order, port_id should " + "be after set VLAN PCP"); + return 0; +} + +/** + * Validate the set VLAN VID. + * + * @param[in] item_flags + * Holds the items detected in this rule. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] actions + * Pointer to the list of actions remaining in the flow rule. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_set_vlan_vid(uint64_t item_flags, + uint64_t action_flags, + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + const struct rte_flow_action *action = actions; + const struct rte_flow_action_of_set_vlan_vid *conf = action->conf; + + if (conf->vlan_vid > RTE_BE16(0xFFE)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "VLAN VID value is too big"); + if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) && + !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "set VLAN VID action must follow push" + " VLAN action or match on VLAN item"); + if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "Multiple VLAN VID modifications are " + "not supported"); + if (action_flags & MLX5_FLOW_ACTION_PORT_ID) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "wrong action order, port_id should " + "be after set VLAN VID"); + return 0; +} + +/* + * Validate the FLAG action. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] attr + * Pointer to flow attributes + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_flag(struct rte_eth_dev *dev, + uint64_t action_flags, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; + int ret; + + /* Fall back if no extended metadata register support. */ + if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) + return mlx5_flow_validate_action_flag(action_flags, attr, + error); + /* Extensive metadata mode requires registers. */ + if (!mlx5_flow_ext_mreg_supported(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "no metadata registers " + "to support flag action"); + if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "extended metadata register" + " isn't available"); + ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); + if (ret < 0) + return ret; + MLX5_ASSERT(ret > 0); + if (action_flags & MLX5_FLOW_ACTION_MARK) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can't mark and flag in same flow"); + if (action_flags & MLX5_FLOW_ACTION_FLAG) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can't have 2 flag" + " actions in same flow"); + return 0; +} + +/** + * Validate MARK action. * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action + * Pointer to action. * @param[in] action_flags * Holds the actions detected until now. - * @param[in] action - * Pointer to the encap action. * @param[in] attr * Pointer to flow attributes * @param[out] error @@ -1074,49 +2031,66 @@ flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_push_vlan(uint64_t action_flags, - uint64_t item_flags, - const struct rte_flow_action *action, - const struct rte_flow_attr *attr, - struct rte_flow_error *error) +flow_dv_validate_action_mark(struct rte_eth_dev *dev, + const struct rte_flow_action *action, + uint64_t action_flags, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { - const struct rte_flow_action_of_push_vlan *push_vlan = action->conf; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; + const struct rte_flow_action_mark *mark = action->conf; + int ret; - if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) && - push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ)) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "invalid vlan ethertype"); - if (action_flags & - (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN)) + /* Fall back if no extended metadata register support. */ + if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) + return mlx5_flow_validate_action_mark(action, action_flags, + attr, error); + /* Extensive metadata mode requires registers. */ + if (!mlx5_flow_ext_mreg_supported(dev)) return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "no support for multiple VLAN " - "actions"); - if (!mlx5_flow_find_action - (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) && - !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "no metadata registers " + "to support mark action"); + if (!priv->sh->dv_mark_mask) return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "push VLAN needs to match on VLAN in order to " - "get VLAN VID information because there is " - "no followed set VLAN VID action"); - if (action_flags & MLX5_FLOW_ACTION_PORT_ID) + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "extended metadata register" + " isn't available"); + ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); + if (ret < 0) + return ret; + MLX5_ASSERT(ret > 0); + if (!mark) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, - "wrong action order, port_id should " - "be after push VLAN"); - (void)attr; + "configuration cannot be null"); + if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + &mark->id, + "mark id exceeds the limit"); + if (action_flags & MLX5_FLOW_ACTION_FLAG) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can't flag and mark in same flow"); + if (action_flags & MLX5_FLOW_ACTION_MARK) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can't have 2 mark actions in same" + " flow"); return 0; } /** - * Validate the set VLAN PCP. + * Validate SET_META action. * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action + * Pointer to the action structure. * @param[in] action_flags * Holds the actions detected until now. - * @param[in] actions - * Pointer to the list of actions remaining in the flow rule. * @param[in] attr * Pointer to flow attributes * @param[out] error @@ -1126,42 +2100,54 @@ flow_dv_validate_action_push_vlan(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags, - const struct rte_flow_action actions[], - struct rte_flow_error *error) +flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, + const struct rte_flow_action *action, + uint64_t action_flags __rte_unused, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { - const struct rte_flow_action *action = actions; - const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf; + const struct rte_flow_action_set_meta *conf; + uint32_t nic_mask = UINT32_MAX; + int reg; - if (conf->vlan_pcp > 7) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "VLAN PCP value is too big"); - if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)) + if (!mlx5_flow_ext_mreg_supported(dev)) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, action, - "set VLAN PCP action must follow " - "the push VLAN action"); - if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) - return rte_flow_error_set(error, ENOTSUP, + "extended metadata register" + " isn't supported"); + reg = flow_dv_get_metadata_reg(dev, attr, error); + if (reg < 0) + return reg; + if (reg != REG_A && reg != REG_B) { + struct mlx5_priv *priv = dev->data->dev_private; + + nic_mask = priv->sh->dv_meta_mask; + } + if (!(action->conf)) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, - "Multiple VLAN PCP modification are " - "not supported"); - if (action_flags & MLX5_FLOW_ACTION_PORT_ID) + "configuration cannot be null"); + conf = (const struct rte_flow_action_set_meta *)action->conf; + if (!conf->mask) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, - "wrong action order, port_id should " - "be after set VLAN PCP"); + "zero mask doesn't have any effect"); + if (conf->mask & ~nic_mask) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "meta data must be within reg C0"); return 0; } /** - * Validate the set VLAN VID. + * Validate SET_TAG action. * - * @param[in] item_flags - * Holds the items detected in this rule. - * @param[in] actions - * Pointer to the list of actions remaining in the flow rule. + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action + * Pointer to the action structure. + * @param[in] action_flags + * Holds the actions detected until now. * @param[in] attr * Pointer to flow attributes * @param[out] error @@ -1171,50 +2157,41 @@ flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_set_vlan_vid(uint64_t item_flags, - uint64_t action_flags, - const struct rte_flow_action actions[], - struct rte_flow_error *error) +flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, + const struct rte_flow_action *action, + uint64_t action_flags, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { - const struct rte_flow_action *action = actions; - const struct rte_flow_action_of_set_vlan_vid *conf = action->conf; - - if (conf->vlan_vid > RTE_BE16(0xFFE)) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "VLAN VID value is too big"); - /* there is an of_push_vlan action before us */ - if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) { - if (mlx5_flow_find_action(actions + 1, - RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "Multiple VLAN VID modifications are " - "not supported"); - else - return 0; - } + const struct rte_flow_action_set_tag *conf; + const uint64_t terminal_action_flags = + MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE | + MLX5_FLOW_ACTION_RSS; + int ret; - /* - * Action is on an existing VLAN header: - * Need to verify this is a single modify CID action. - * Rule mast include a match on outer VLAN. - */ - if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) + if (!mlx5_flow_ext_mreg_supported(dev)) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, action, - "Multiple VLAN VID modifications are " - "not supported"); - if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) + "extensive metadata register" + " isn't supported"); + if (!(action->conf)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, - "match on VLAN is required in order " - "to set VLAN VID"); - if (action_flags & MLX5_FLOW_ACTION_PORT_ID) + "configuration cannot be null"); + conf = (const struct rte_flow_action_set_tag *)action->conf; + if (!conf->mask) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, - "wrong action order, port_id should " - "be after set VLAN VID"); + "zero mask doesn't have any effect"); + ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); + if (ret < 0) + return ret; + if (!attr->transfer && attr->ingress && + (action_flags & terminal_action_flags)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "set_tag has no effect" + " with terminal actions"); return 0; } @@ -1222,7 +2199,7 @@ flow_dv_validate_action_set_vlan_vid(uint64_t item_flags, * Validate count action. * * @param[in] dev - * device otr. + * Pointer to rte_eth_dev structure. * @param[out] error * Pointer to error structure. * @@ -1251,12 +2228,14 @@ notsup_err: /** * Validate the L2 encap action. * + * @param[in] dev + * Pointer to the rte_eth_dev structure. * @param[in] action_flags * Holds the actions detected until now. * @param[in] action - * Pointer to the encap action. + * Pointer to the action structure. * @param[in] attr - * Pointer to flow attributes + * Pointer to flow attributes. * @param[out] error * Pointer to error structure. * @@ -1264,36 +2243,36 @@ notsup_err: * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_l2_encap(uint64_t action_flags, +flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, + uint64_t action_flags, const struct rte_flow_action *action, const struct rte_flow_attr *attr, struct rte_flow_error *error) { + const struct mlx5_priv *priv = dev->data->dev_private; + if (!(action->conf)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "configuration cannot be null"); - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and encap in same flow"); - if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS)) + if (action_flags & MLX5_FLOW_ACTION_ENCAP) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can only have a single encap or" - " decap action in a flow"); - if (!attr->transfer && attr->ingress) + "can only have a single encap action " + "in a flow"); + if (!attr->transfer && priv->representor) return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, - NULL, - "encap action not supported for " - "ingress"); + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "encap action for VF representor " + "not supported on NIC table"); return 0; } /** - * Validate the L2 decap action. + * Validate a decap action. * + * @param[in] dev + * Pointer to the rte_eth_dev structure. * @param[in] action_flags * Holds the actions detected until now. * @param[in] attr @@ -1305,19 +2284,20 @@ flow_dv_validate_action_l2_encap(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_l2_decap(uint64_t action_flags, - const struct rte_flow_attr *attr, - struct rte_flow_error *error) +flow_dv_validate_action_decap(struct rte_eth_dev *dev, + uint64_t action_flags, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and decap in same flow"); - if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS)) - return rte_flow_error_set(error, EINVAL, + const struct mlx5_priv *priv = dev->data->dev_private; + + if (action_flags & MLX5_FLOW_XCAP_ACTIONS) + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can only have a single encap or" - " decap action in a flow"); + action_flags & + MLX5_FLOW_ACTION_DECAP ? "can only " + "have a single decap action" : "decap " + "after encap is not supported"); if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -1329,69 +2309,31 @@ flow_dv_validate_action_l2_decap(uint64_t action_flags, NULL, "decap action not supported for " "egress"); - return 0; -} - -/** - * Validate the raw encap action. - * - * @param[in] action_flags - * Holds the actions detected until now. - * @param[in] action - * Pointer to the encap action. - * @param[in] attr - * Pointer to flow attributes - * @param[out] error - * Pointer to error structure. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -static int -flow_dv_validate_action_raw_encap(uint64_t action_flags, - const struct rte_flow_action *action, - const struct rte_flow_attr *attr, - struct rte_flow_error *error) -{ - const struct rte_flow_action_raw_encap *raw_encap = - (const struct rte_flow_action_raw_encap *)action->conf; - if (!(action->conf)) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "configuration cannot be null"); - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and encap in same flow"); - if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can only have a single encap" - " action in a flow"); - /* encap without preceding decap is not supported for ingress */ - if (!attr->transfer && attr->ingress && - !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP)) + if (!attr->transfer && priv->representor) return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, - NULL, - "encap action not supported for " - "ingress"); - if (!raw_encap->size || !raw_encap->data) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "raw encap data cannot be empty"); + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "decap action for VF representor " + "not supported on NIC table"); return 0; } +const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,}; + /** - * Validate the raw decap action. + * Validate the raw encap and decap actions. * - * @param[in] action_flags - * Holds the actions detected until now. - * @param[in] action + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] decap + * Pointer to the decap action. + * @param[in] encap * Pointer to the encap action. * @param[in] attr * Pointer to flow attributes + * @param[in/out] action_flags + * Holds the actions detected until now. + * @param[out] actions_n + * pointer to the number of actions counter. * @param[out] error * Pointer to error structure. * @@ -1399,42 +2341,72 @@ flow_dv_validate_action_raw_encap(uint64_t action_flags, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_raw_decap(uint64_t action_flags, - const struct rte_flow_action *action, - const struct rte_flow_attr *attr, - struct rte_flow_error *error) +flow_dv_validate_action_raw_encap_decap + (struct rte_eth_dev *dev, + const struct rte_flow_action_raw_decap *decap, + const struct rte_flow_action_raw_encap *encap, + const struct rte_flow_attr *attr, uint64_t *action_flags, + int *actions_n, struct rte_flow_error *error) { - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and decap in same flow"); - if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't have encap action before" - " decap action"); - if (action_flags & MLX5_FLOW_DECAP_ACTIONS) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can only have a single decap" - " action in a flow"); - if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) + const struct mlx5_priv *priv = dev->data->dev_private; + int ret; + + if (encap && (!encap->size || !encap->data)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't have decap action after" - " modify action"); - /* decap action is valid on egress only if it is followed by encap */ - if (attr->egress) { - for (; action->type != RTE_FLOW_ACTION_TYPE_END && - action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP; - action++) { - } - if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) + "raw encap data cannot be empty"); + if (decap && encap) { + if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE && + encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) + /* L3 encap. */ + decap = NULL; + else if (encap->size <= + MLX5_ENCAPSULATION_DECISION_SIZE && + decap->size > + MLX5_ENCAPSULATION_DECISION_SIZE) + /* L3 decap. */ + encap = NULL; + else if (encap->size > + MLX5_ENCAPSULATION_DECISION_SIZE && + decap->size > + MLX5_ENCAPSULATION_DECISION_SIZE) + /* 2 L2 actions: encap and decap. */ + ; + else + return rte_flow_error_set(error, + ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "unsupported too small " + "raw decap and too small raw " + "encap combination"); + } + if (decap) { + ret = flow_dv_validate_action_decap(dev, *action_flags, attr, + error); + if (ret < 0) + return ret; + *action_flags |= MLX5_FLOW_ACTION_DECAP; + ++(*actions_n); + } + if (encap) { + if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "small raw encap size"); + if (*action_flags & MLX5_FLOW_ACTION_ENCAP) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "more than one encap action"); + if (!attr->transfer && priv->representor) return rte_flow_error_set (error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, - NULL, "decap action not supported" - " for egress"); + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "encap action for VF representor " + "not supported on NIC table"); + *action_flags |= MLX5_FLOW_ACTION_ENCAP; + ++(*actions_n); } return 0; } @@ -1442,7 +2414,7 @@ flow_dv_validate_action_raw_decap(uint64_t action_flags, /** * Find existing encap/decap resource or create and register a new one. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in, out] resource * Pointer to encap/decap resource. @@ -1464,17 +2436,15 @@ flow_dv_encap_decap_resource_register struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_encap_decap_resource *cache_resource; - struct rte_flow *flow = dev_flow->flow; struct mlx5dv_dr_domain *domain; - resource->flags = flow->group ? 0 : 1; + resource->flags = dev_flow->dv.group ? 0 : 1; if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) domain = sh->fdb_domain; else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) domain = sh->rx_domain; else domain = sh->tx_domain; - /* Lookup a matching resource from cache. */ LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) { if (resource->reformat_type == cache_resource->reformat_type && @@ -1488,7 +2458,7 @@ flow_dv_encap_decap_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->dv.encap_decap = cache_resource; + dev_flow->handle->dvh.encap_decap = cache_resource; return 0; } } @@ -1514,7 +2484,7 @@ flow_dv_encap_decap_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next); - dev_flow->dv.encap_decap = cache_resource; + dev_flow->handle->dvh.encap_decap = cache_resource; DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -1524,70 +2494,55 @@ flow_dv_encap_decap_resource_register /** * Find existing table jump resource or create and register a new one. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in, out] resource - * Pointer to jump table resource. + * @param[in, out] tbl + * Pointer to flow table resource. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error * pointer to error structure. - * - * @return - * 0 on success otherwise -errno and errno is set. - */ -static int -flow_dv_jump_tbl_resource_register - (struct rte_eth_dev *dev, - struct mlx5_flow_dv_jump_tbl_resource *resource, - struct mlx5_flow *dev_flow, - struct rte_flow_error *error) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_ibv_shared *sh = priv->sh; - struct mlx5_flow_dv_jump_tbl_resource *cache_resource; - - /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &sh->jump_tbl, next) { - if (resource->tbl == cache_resource->tbl) { - DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->dv.jump = cache_resource; - return 0; - } - } - /* Register new jump table resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate resource memory"); - *cache_resource = *resource; - cache_resource->action = - mlx5_glue->dr_create_flow_action_dest_flow_tbl - (resource->tbl->obj); - if (!cache_resource->action) { - rte_free(cache_resource); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create action"); + * + * @return + * 0 on success otherwise -errno and errno is set. + */ +static int +flow_dv_jump_tbl_resource_register + (struct rte_eth_dev *dev __rte_unused, + struct mlx5_flow_tbl_resource *tbl, + struct mlx5_flow *dev_flow, + struct rte_flow_error *error) +{ + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); + int cnt; + + MLX5_ASSERT(tbl); + cnt = rte_atomic32_read(&tbl_data->jump.refcnt); + if (!cnt) { + tbl_data->jump.action = + mlx5_glue->dr_create_flow_action_dest_flow_tbl + (tbl->obj); + if (!tbl_data->jump.action) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create jump action"); + DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", + (void *)&tbl_data->jump, cnt); + } else { + MLX5_ASSERT(tbl_data->jump.action); + DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++", + (void *)&tbl_data->jump, cnt); } - rte_atomic32_init(&cache_resource->refcnt); - rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next); - dev_flow->dv.jump = cache_resource; - DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); + rte_atomic32_inc(&tbl_data->jump.refcnt); + dev_flow->handle->dvh.jump = &tbl_data->jump; return 0; } /** * Find existing table port ID resource or create and register a new one. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in, out] resource * Pointer to port ID action resource. @@ -1618,7 +2573,7 @@ flow_dv_port_id_action_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->dv.port_id_action = cache_resource; + dev_flow->handle->dvh.port_id_action = cache_resource; return 0; } } @@ -1629,8 +2584,13 @@ flow_dv_port_id_action_resource_register RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); *cache_resource = *resource; + /* + * Depending on rdma_core version the glue routine calls + * either mlx5dv_dr_action_create_dest_ib_port(domain, ibv_port) + * or mlx5dv_dr_action_create_dest_vport(domain, vport_id). + */ cache_resource->action = - mlx5_glue->dr_create_flow_action_dest_vport + mlx5_glue->dr_create_flow_action_dest_port (priv->sh->fdb_domain, resource->port_id); if (!cache_resource->action) { rte_free(cache_resource); @@ -1641,7 +2601,7 @@ flow_dv_port_id_action_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next); - dev_flow->dv.port_id_action = cache_resource; + dev_flow->handle->dvh.port_id_action = cache_resource; DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -1651,7 +2611,7 @@ flow_dv_port_id_action_resource_register /** * Find existing push vlan resource or create and register a new one. * - * @param dev[in, out] + * @param [in, out] dev * Pointer to rte_eth_dev structure. * @param[in, out] resource * Pointer to port ID action resource. @@ -1684,7 +2644,7 @@ flow_dv_push_vlan_action_resource_register (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->dv.push_vlan_res = cache_resource; + dev_flow->handle->dvh.push_vlan_res = cache_resource; return 0; } } @@ -1713,7 +2673,7 @@ flow_dv_push_vlan_action_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next); - dev_flow->dv.push_vlan_res = cache_resource; + dev_flow->handle->dvh.push_vlan_res = cache_resource; DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -2035,8 +2995,6 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, (const struct rte_flow_action_raw_encap *)action->conf; res.size = raw_encap_data->size; memcpy(res.buf, raw_encap_data->data, res.size); - if (flow_dv_zero_encap_udp_csum(res.buf, error)) - return -rte_errno; } else { if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) encap_data = @@ -2050,6 +3008,8 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, &res.size, error)) return -rte_errno; } + if (flow_dv_zero_encap_udp_csum(res.buf, error)) + return -rte_errno; if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -2123,9 +3083,9 @@ flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, encap_data = (const struct rte_flow_action_raw_encap *)action->conf; res.size = encap_data->size; memcpy(res.buf, encap_data->data, res.size); - res.reformat_type = attr->egress ? - MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL : - MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2; + res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ? + MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 : + MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL; if (attr->transfer) res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; else @@ -2143,12 +3103,12 @@ flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to rte_eth_dev structure. - * @param[in] vlan_tag - * the vlan tag to push to the Ethernet header. - * @param[in, out] dev_flow - * Pointer to the mlx5_flow. * @param[in] attr * Pointer to the flow attributes. + * @param[in] vlan + * Pointer to the vlan to push to the Ethernet header. + * @param[in, out] dev_flow + * Pointer to the mlx5_flow. * @param[out] error * Pointer to the error structure. * @@ -2198,7 +3158,7 @@ flow_dv_validate_action_modify_hdr(const uint64_t action_flags, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, "action configuration not set"); - if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) + if (action_flags & MLX5_FLOW_ACTION_ENCAP) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can't have encap action before" @@ -2262,10 +3222,14 @@ flow_dv_validate_action_modify_ipv4(const uint64_t action_flags, struct rte_flow_error *error) { int ret = 0; + uint64_t layer; ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); if (!ret) { - if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) + layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? + MLX5_FLOW_LAYER_INNER_L3_IPV4 : + MLX5_FLOW_LAYER_OUTER_L3_IPV4; + if (!(item_flags & layer)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -2296,10 +3260,14 @@ flow_dv_validate_action_modify_ipv6(const uint64_t action_flags, struct rte_flow_error *error) { int ret = 0; + uint64_t layer; ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); if (!ret) { - if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) + layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? + MLX5_FLOW_LAYER_INNER_L3_IPV6 : + MLX5_FLOW_LAYER_OUTER_L3_IPV6; + if (!(item_flags & layer)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -2330,10 +3298,14 @@ flow_dv_validate_action_modify_tp(const uint64_t action_flags, struct rte_flow_error *error) { int ret = 0; + uint64_t layer; ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); if (!ret) { - if (!(item_flags & MLX5_FLOW_LAYER_L4)) + layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? + MLX5_FLOW_LAYER_INNER_L4 : + MLX5_FLOW_LAYER_OUTER_L4; + if (!(item_flags & layer)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "no transport layer " @@ -2365,10 +3337,14 @@ flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags, struct rte_flow_error *error) { int ret = 0; + uint64_t layer; ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); if (!ret) { - if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)) + layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? + MLX5_FLOW_LAYER_INNER_L4_TCP : + MLX5_FLOW_LAYER_OUTER_L4_TCP; + if (!(item_flags & layer)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "no TCP item in" @@ -2410,10 +3386,14 @@ flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags, struct rte_flow_error *error) { int ret = 0; + uint64_t layer; ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); if (!ret) { - if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)) + layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? + MLX5_FLOW_LAYER_INNER_L4_TCP : + MLX5_FLOW_LAYER_OUTER_L4_TCP; + if (!(item_flags & layer)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "no TCP item in" @@ -2454,10 +3434,14 @@ flow_dv_validate_action_modify_ttl(const uint64_t action_flags, struct rte_flow_error *error) { int ret = 0; + uint64_t layer; ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); if (!ret) { - if (!(item_flags & MLX5_FLOW_LAYER_L3)) + layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? + MLX5_FLOW_LAYER_INNER_L3 : + MLX5_FLOW_LAYER_OUTER_L3; + if (!(item_flags & layer)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -2489,8 +3473,6 @@ flow_dv_validate_action_jump(const struct rte_flow_action *action, const struct rte_flow_attr *attributes, bool external, struct rte_flow_error *error) { - uint32_t max_group = attributes->transfer ? MLX5_MAX_TABLES_FDB : - MLX5_MAX_TABLES; uint32_t target_group, table; int ret = 0; @@ -2500,6 +3482,10 @@ flow_dv_validate_action_jump(const struct rte_flow_action *action, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can't have 2 fate actions in" " same flow"); + if (action_flags & MLX5_FLOW_ACTION_METER) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "jump with meter not support"); if (!action->conf) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, @@ -2507,17 +3493,13 @@ flow_dv_validate_action_jump(const struct rte_flow_action *action, target_group = ((const struct rte_flow_action_jump *)action->conf)->group; ret = mlx5_flow_group_to_table(attributes, external, target_group, - &table, error); + true, &table, error); if (ret) return ret; - if (table >= max_group) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, - "target group index out of range"); - if (attributes->group >= target_group) + if (attributes->group == target_group) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "target group must be higher than" + "target group must be other than" " the current flow group"); return 0; } @@ -2577,7 +3559,7 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev, "failed to obtain E-Switch info"); port_id = action->conf; port = port_id->original ? dev->data->port_id : port_id->id; - act_priv = mlx5_port_to_eswitch_info(port); + act_priv = mlx5_port_to_eswitch_info(port, false); if (!act_priv) return rte_flow_error_set (error, rte_errno, @@ -2592,6 +3574,164 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev, return 0; } +/** + * Get the maximum number of modify header actions. + * + * @param dev + * Pointer to rte_eth_dev structure. + * @param flags + * Flags bits to check if root level. + * + * @return + * Max number of modify header actions device can support. + */ +static unsigned int +flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev, uint64_t flags) +{ + /* + * There's no way to directly query the max cap. Although it has to be + * acquried by iterative trial, it is a safe assumption that more + * actions are supported by FW if extensive metadata register is + * supported. (Only in the root table) + */ + if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL)) + return MLX5_MAX_MODIFY_NUM; + else + return mlx5_flow_ext_mreg_supported(dev) ? + MLX5_ROOT_TBL_MODIFY_NUM : + MLX5_ROOT_TBL_MODIFY_NUM_NO_MREG; +} + +/** + * Validate the meter action. + * + * @param[in] dev + * Pointer to rte_eth_dev structure. + * @param[in] action_flags + * Bit-fields that holds the actions detected until now. + * @param[in] action + * Pointer to the meter action. + * @param[in] attr + * Attributes of flow that includes this action. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_ernno is set. + */ +static int +mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, + uint64_t action_flags, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_action_meter *am = action->conf; + struct mlx5_flow_meter *fm; + + if (!am) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "meter action conf is NULL"); + + if (action_flags & MLX5_FLOW_ACTION_METER) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "meter chaining not support"); + if (action_flags & MLX5_FLOW_ACTION_JUMP) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "meter with jump not support"); + if (!priv->mtr_en) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "meter action not supported"); + fm = mlx5_flow_meter_find(priv, am->mtr_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Meter not found"); + if (fm->ref_cnt && (!(fm->attr.transfer == attr->transfer || + (!fm->attr.ingress && !attr->ingress && attr->egress) || + (!fm->attr.egress && !attr->egress && attr->ingress)))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Flow attributes are either invalid " + "or have a conflict with current " + "meter attributes"); + return 0; +} + +/** + * Validate the modify-header IPv4 DSCP actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no ipv4 item in pattern"); + } + return ret; +} + +/** + * Validate the modify-header IPv6 DSCP actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no ipv6 item in pattern"); + } + return ret; +} + /** * Find existing modify-header resource or create and register a new one. * @@ -2618,45 +3758,51 @@ flow_dv_modify_hdr_resource_register struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_modify_hdr_resource *cache_resource; struct mlx5dv_dr_domain *ns; + uint32_t actions_len; + resource->flags = dev_flow->dv.group ? 0 : + MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; + if (resource->actions_num > flow_dv_modify_hdr_action_max(dev, + resource->flags)) + return rte_flow_error_set(error, EOVERFLOW, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "too many modify header items"); if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) ns = sh->fdb_domain; else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) ns = sh->tx_domain; else ns = sh->rx_domain; - resource->flags = - dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; /* Lookup a matching resource from cache. */ + actions_len = resource->actions_num * sizeof(resource->actions[0]); LIST_FOREACH(cache_resource, &sh->modify_cmds, next) { if (resource->ft_type == cache_resource->ft_type && resource->actions_num == cache_resource->actions_num && resource->flags == cache_resource->flags && !memcmp((const void *)resource->actions, (const void *)cache_resource->actions, - (resource->actions_num * - sizeof(resource->actions[0])))) { + actions_len)) { DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->dv.modify_hdr = cache_resource; + dev_flow->handle->dvh.modify_hdr = cache_resource; return 0; } } /* Register new modify-header resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + cache_resource = rte_calloc(__func__, 1, + sizeof(*cache_resource) + actions_len, 0); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); *cache_resource = *resource; + rte_memcpy(cache_resource->actions, resource->actions, actions_len); cache_resource->verbs_action = mlx5_glue->dv_create_flow_action_modify_header - (sh->ctx, cache_resource->ft_type, - ns, cache_resource->flags, - cache_resource->actions_num * - sizeof(cache_resource->actions[0]), + (sh->ctx, cache_resource->ft_type, ns, + cache_resource->flags, actions_len, (uint64_t *)cache_resource->actions); if (!cache_resource->verbs_action) { rte_free(cache_resource); @@ -2667,139 +3813,48 @@ flow_dv_modify_hdr_resource_register rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next); - dev_flow->dv.modify_hdr = cache_resource; + dev_flow->handle->dvh.modify_hdr = cache_resource; DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); return 0; } -#define MLX5_CNT_CONTAINER_RESIZE 64 - /** - * Get or create a flow counter. + * Get DV flow counter by index. * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] shared - * Indicate if this counter is shared with other flows. - * @param[in] id - * Counter identifier. + * @param[in] idx + * mlx5 flow counter index in the container. + * @param[out] ppool + * mlx5 flow counter pool in the container, * * @return - * pointer to flow counter on success, NULL otherwise and rte_errno is set. + * Pointer to the counter, NULL otherwise. */ static struct mlx5_flow_counter * -flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared, - uint32_t id) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_counter *cnt = NULL; - struct mlx5_devx_obj *dcs = NULL; - - if (!priv->config.devx) { - rte_errno = ENOTSUP; - return NULL; - } - if (shared) { - TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) { - if (cnt->shared && cnt->id == id) { - cnt->ref_cnt++; - return cnt; - } - } - } - dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); - if (!dcs) - return NULL; - cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0); - if (!cnt) { - claim_zero(mlx5_devx_cmd_destroy(cnt->dcs)); - rte_errno = ENOMEM; - return NULL; - } - struct mlx5_flow_counter tmpl = { - .shared = shared, - .ref_cnt = 1, - .id = id, - .dcs = dcs, - }; - tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0); - if (!tmpl.action) { - claim_zero(mlx5_devx_cmd_destroy(cnt->dcs)); - rte_errno = errno; - rte_free(cnt); - return NULL; - } - *cnt = tmpl; - TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next); - return cnt; -} - -/** - * Release a flow counter. - * - * @param[in] dev - * Pointer to the Ethernet device structure. - * @param[in] counter - * Pointer to the counter handler. - */ -static void -flow_dv_counter_release_fallback(struct rte_eth_dev *dev, - struct mlx5_flow_counter *counter) +flow_dv_counter_get_by_idx(struct rte_eth_dev *dev, + uint32_t idx, + struct mlx5_flow_counter_pool **ppool) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_pools_container *cont; + struct mlx5_flow_counter_pool *pool; + uint32_t batch = 0; - if (!counter) - return; - if (--counter->ref_cnt == 0) { - TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next); - claim_zero(mlx5_devx_cmd_destroy(counter->dcs)); - rte_free(counter); - } -} - -/** - * Query a devx flow counter. - * - * @param[in] dev - * Pointer to the Ethernet device structure. - * @param[in] cnt - * Pointer to the flow counter. - * @param[out] pkts - * The statistics value of packets. - * @param[out] bytes - * The statistics value of bytes. - * - * @return - * 0 on success, otherwise a negative errno value and rte_errno is set. - */ -static inline int -_flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused, - struct mlx5_flow_counter *cnt, uint64_t *pkts, - uint64_t *bytes) -{ - return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes, - 0, NULL, NULL, 0); -} - -/** - * Get a pool by a counter. - * - * @param[in] cnt - * Pointer to the counter. - * - * @return - * The counter pool. - */ -static struct mlx5_flow_counter_pool * -flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt) -{ - if (!cnt->batch) { - cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL; - return (struct mlx5_flow_counter_pool *)cnt - 1; + idx--; + if (idx >= MLX5_CNT_BATCH_OFFSET) { + idx -= MLX5_CNT_BATCH_OFFSET; + batch = 1; } - return cnt->pool; + cont = MLX5_CNT_CONTAINER(priv->sh, batch, 0); + MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n); + pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL]; + MLX5_ASSERT(pool); + if (ppool) + *ppool = pool; + return &pool->counters_raw[idx % MLX5_COUNTERS_PER_POOL]; } /** @@ -2816,15 +3871,26 @@ flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt) static struct mlx5_flow_counter_pool * flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id) { - struct mlx5_flow_counter_pool *pool; + uint32_t i; + uint32_t n_valid = rte_atomic16_read(&cont->n_valid); - TAILQ_FOREACH(pool, &cont->pool_list, next) { + for (i = 0; i < n_valid; i++) { + struct mlx5_flow_counter_pool *pool = cont->pools[i]; int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) * - MLX5_COUNTERS_PER_POOL; + MLX5_COUNTERS_PER_POOL; - if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) + if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) { + /* + * Move the pool to the head, as counter allocate + * always gets the first pool in the container. + */ + if (pool != TAILQ_FIRST(&cont->pool_list)) { + TAILQ_REMOVE(&cont->pool_list, pool, next); + TAILQ_INSERT_HEAD(&cont->pool_list, pool, next); + } return pool; - }; + } + } return NULL; } @@ -2873,6 +3939,11 @@ flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n) mkey_attr.size = size; mkey_attr.umem_id = mem_mng->umem->umem_id; mkey_attr.pd = sh->pdn; + mkey_attr.log_entity_size = 0; + mkey_attr.pg_access = 0; + mkey_attr.klm_array = NULL; + mkey_attr.klm_num = 0; + mkey_attr.relaxed_ordering = 1; mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); if (!mem_mng->dm) { mlx5_glue->devx_umem_dereg(mem_mng->umem); @@ -2909,12 +3980,14 @@ flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch) MLX5_CNT_CONTAINER(priv->sh, batch, 0); struct mlx5_pools_container *new_cont = MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0); - struct mlx5_counter_stats_mem_mng *mem_mng; + struct mlx5_counter_stats_mem_mng *mem_mng = NULL; uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE; uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize; int i; - if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) { + /* Fallback mode has no background thread. Skip the check. */ + if (!priv->counter_fallback && + cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) { /* The last resize still hasn't detected by the host thread. */ rte_errno = EAGAIN; return NULL; @@ -2927,16 +4000,29 @@ flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch) if (cont->n) memcpy(new_cont->pools, cont->pools, cont->n * sizeof(struct mlx5_flow_counter_pool *)); - mem_mng = flow_dv_create_counter_stat_mem_mng(dev, - MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES); - if (!mem_mng) { - rte_free(new_cont->pools); - return NULL; + /* + * Fallback mode query the counter directly, no background query + * resources are needed. + */ + if (!priv->counter_fallback) { + mem_mng = flow_dv_create_counter_stat_mem_mng(dev, + MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES); + if (!mem_mng) { + rte_free(new_cont->pools); + return NULL; + } + for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) + LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws, + mem_mng->raws + + MLX5_CNT_CONTAINER_RESIZE + + i, next); + } else { + /* + * Release the old container pools directly as no background + * thread helps that. + */ + rte_free(cont->pools); } - for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) - LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws, - mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + - i, next); new_cont->n = resize; rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid)); TAILQ_INIT(&new_cont->pool_list); @@ -2954,7 +4040,7 @@ flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch) * @param[in] dev * Pointer to the Ethernet device structure. * @param[in] cnt - * Pointer to the flow counter. + * Index to the flow counter. * @param[out] pkts * The statistics value of packets. * @param[out] bytes @@ -2964,17 +4050,23 @@ flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch) * 0 on success, otherwise a negative errno value and rte_errno is set. */ static inline int -_flow_dv_query_count(struct rte_eth_dev *dev, - struct mlx5_flow_counter *cnt, uint64_t *pkts, +_flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts, uint64_t *bytes) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_counter_pool *pool = - flow_dv_counter_pool_get(cnt); - int offset = cnt - &pool->counters_raw[0]; - - if (priv->counter_fallback) - return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes); + struct mlx5_flow_counter_pool *pool = NULL; + struct mlx5_flow_counter *cnt; + struct mlx5_flow_counter_ext *cnt_ext = NULL; + int offset; + + cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); + MLX5_ASSERT(pool); + if (counter < MLX5_CNT_BATCH_OFFSET) { + cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); + if (priv->counter_fallback) + return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0, + 0, pkts, bytes, 0, NULL, NULL, 0); + } rte_spinlock_lock(&pool->sl); /* @@ -2982,10 +4074,11 @@ _flow_dv_query_count(struct rte_eth_dev *dev, * current allocated in parallel to the host reading. * In this case the new counter values must be reported as 0. */ - if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) { + if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) { *pkts = 0; *bytes = 0; } else { + offset = cnt - &pool->counters_raw[0]; *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits); *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes); } @@ -3002,11 +4095,13 @@ _flow_dv_query_count(struct rte_eth_dev *dev, * The devX counter handle. * @param[in] batch * Whether the pool is for counter that was allocated by batch command. + * @param[in/out] cont_cur + * Pointer to the container pointer, it will be update in pool resize. * * @return - * A new pool pointer on success, NULL otherwise and rte_errno is set. + * The pool container pointer on success, NULL otherwise and rte_errno is set. */ -static struct mlx5_flow_counter_pool * +static struct mlx5_pools_container * flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, uint32_t batch) { @@ -3022,30 +4117,44 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, if (!cont) return NULL; } - size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL * - sizeof(struct mlx5_flow_counter); + size = sizeof(*pool); + if (!batch) + size += MLX5_COUNTERS_PER_POOL * + sizeof(struct mlx5_flow_counter_ext); pool = rte_calloc(__func__, 1, size, 0); if (!pool) { rte_errno = ENOMEM; return NULL; } pool->min_dcs = dcs; - pool->raw = cont->init_mem_mng->raws + n_valid % + if (!priv->counter_fallback) + pool->raw = cont->init_mem_mng->raws + n_valid % MLX5_CNT_CONTAINER_RESIZE; pool->raw_hw = NULL; rte_spinlock_init(&pool->sl); /* * The generation of the new allocated counters in this pool is 0, 2 in * the pool generation makes all the counters valid for allocation. + * The start and end query generation protect the counters be released + * between the query and update gap period will not be reallocated + * without the last query finished and stats updated to the memory. */ - rte_atomic64_set(&pool->query_gen, 0x2); + rte_atomic64_set(&pool->start_query_gen, 0x2); + /* + * There's no background query thread for fallback mode, set the + * end_query_gen to the maximum value since no need to wait for + * statistics update. + */ + rte_atomic64_set(&pool->end_query_gen, priv->counter_fallback ? + INT64_MAX : 0x2); TAILQ_INIT(&pool->counters); - TAILQ_INSERT_TAIL(&cont->pool_list, pool, next); + TAILQ_INSERT_HEAD(&cont->pool_list, pool, next); + pool->index = n_valid; cont->pools[n_valid] = pool; /* Pool initialization must be updated before host thread access. */ rte_cio_wmb(); rte_atomic16_add(&cont->n_valid, 1); - return pool; + return cont; } /** @@ -3059,42 +4168,45 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, * Whether the pool is for counter that was allocated by batch command. * * @return - * The free counter pool pointer and @p cnt_free is set on success, + * The counter container pointer and @p cnt_free is set on success, * NULL otherwise and rte_errno is set. */ -static struct mlx5_flow_counter_pool * +static struct mlx5_pools_container * flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, struct mlx5_flow_counter **cnt_free, uint32_t batch) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_pools_container *cont; struct mlx5_flow_counter_pool *pool; struct mlx5_devx_obj *dcs = NULL; struct mlx5_flow_counter *cnt; uint32_t i; + cont = MLX5_CNT_CONTAINER(priv->sh, batch, 0); if (!batch) { /* bulk_bitmap must be 0 for single counter allocation. */ dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); if (!dcs) return NULL; - pool = flow_dv_find_pool_by_id - (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id); + pool = flow_dv_find_pool_by_id(cont, dcs->id); if (!pool) { - pool = flow_dv_pool_create(dev, dcs, batch); - if (!pool) { + cont = flow_dv_pool_create(dev, dcs, batch); + if (!cont) { mlx5_devx_cmd_destroy(dcs); return NULL; } + pool = TAILQ_FIRST(&cont->pool_list); } else if (dcs->id < pool->min_dcs->id) { rte_atomic64_set(&pool->a64_dcs, (int64_t)(uintptr_t)dcs); } - cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL]; + i = dcs->id % MLX5_COUNTERS_PER_POOL; + cnt = &pool->counters_raw[i]; TAILQ_INSERT_HEAD(&pool->counters, cnt, next); - cnt->dcs = dcs; + MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs; *cnt_free = cnt; - return pool; + return cont; } /* bulk_bitmap is in 128 counters units. */ if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4) @@ -3103,18 +4215,18 @@ flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, rte_errno = ENODATA; return NULL; } - pool = flow_dv_pool_create(dev, dcs, batch); - if (!pool) { + cont = flow_dv_pool_create(dev, dcs, batch); + if (!cont) { mlx5_devx_cmd_destroy(dcs); return NULL; } + pool = TAILQ_FIRST(&cont->pool_list); for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { cnt = &pool->counters_raw[i]; - cnt->pool = pool; TAILQ_INSERT_HEAD(&pool->counters, cnt, next); } *cnt_free = &pool->counters_raw[0]; - return pool; + return cont; } /** @@ -3124,23 +4236,30 @@ flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, * Pointer to the relevant counter pool container. * @param[in] id * The shared counter ID to search. + * @param[out] ppool + * mlx5 flow counter pool in the container, * * @return - * NULL if not existed, otherwise pointer to the shared counter. + * NULL if not existed, otherwise pointer to the shared extend counter. */ -static struct mlx5_flow_counter * -flow_dv_counter_shared_search(struct mlx5_pools_container *cont, - uint32_t id) +static struct mlx5_flow_counter_ext * +flow_dv_counter_shared_search(struct mlx5_pools_container *cont, uint32_t id, + struct mlx5_flow_counter_pool **ppool) { - static struct mlx5_flow_counter *cnt; + static struct mlx5_flow_counter_ext *cnt; struct mlx5_flow_counter_pool *pool; - int i; + uint32_t i; + uint32_t n_valid = rte_atomic16_read(&cont->n_valid); - TAILQ_FOREACH(pool, &cont->pool_list, next) { + for (i = 0; i < n_valid; i++) { + pool = cont->pools[i]; for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { - cnt = &pool->counters_raw[i]; - if (cnt->ref_cnt && cnt->shared && cnt->id == id) + cnt = MLX5_GET_POOL_CNT_EXT(pool, i); + if (cnt->ref_cnt && cnt->shared && cnt->id == id) { + if (ppool) + *ppool = cont->pools[i]; return cnt; + } } } return NULL; @@ -3159,15 +4278,16 @@ flow_dv_counter_shared_search(struct mlx5_pools_container *cont, * Counter flow group. * * @return - * pointer to flow counter on success, NULL otherwise and rte_errno is set. + * Index to flow counter on success, 0 otherwise and rte_errno is set. */ -static struct mlx5_flow_counter * +static uint32_t flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, uint16_t group) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_counter_pool *pool = NULL; struct mlx5_flow_counter *cnt_free = NULL; + struct mlx5_flow_counter_ext *cnt_ext = NULL; /* * Currently group 0 flow counter cannot be assigned to a flow if it is * not the first one in the batch counter allocation, so it is better @@ -3176,25 +4296,27 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, * A counter can be shared between different groups so need to take * shared counters from the single container. */ - uint32_t batch = (group && !shared) ? 1 : 0; + uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0; struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 0); + uint32_t cnt_idx; - if (priv->counter_fallback) - return flow_dv_counter_alloc_fallback(dev, shared, id); if (!priv->config.devx) { rte_errno = ENOTSUP; - return NULL; + return 0; } if (shared) { - cnt_free = flow_dv_counter_shared_search(cont, id); - if (cnt_free) { - if (cnt_free->ref_cnt + 1 == 0) { + cnt_ext = flow_dv_counter_shared_search(cont, id, &pool); + if (cnt_ext) { + if (cnt_ext->ref_cnt + 1 == 0) { rte_errno = E2BIG; - return NULL; + return 0; } - cnt_free->ref_cnt++; - return cnt_free; + cnt_ext->ref_cnt++; + cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL + + (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL) + + 1; + return cnt_idx; } } /* Pools which has a free counters are in the start. */ @@ -3209,17 +4331,19 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, * updated too. */ cnt_free = TAILQ_FIRST(&pool->counters); - if (cnt_free && cnt_free->query_gen + 1 < - rte_atomic64_read(&pool->query_gen)) + if (cnt_free && cnt_free->query_gen < + rte_atomic64_read(&pool->end_query_gen)) break; cnt_free = NULL; } if (!cnt_free) { - pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch); - if (!pool) - return NULL; + cont = flow_dv_counter_pool_prepare(dev, &cnt_free, batch); + if (!cont) + return 0; + pool = TAILQ_FIRST(&cont->pool_list); } - cnt_free->batch = batch; + if (!batch) + cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free); /* Create a DV counter action only in the first time usage. */ if (!cnt_free->action) { uint16_t offset; @@ -3230,23 +4354,28 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, dcs = pool->min_dcs; } else { offset = 0; - dcs = cnt_free->dcs; + dcs = cnt_ext->dcs; } cnt_free->action = mlx5_glue->dv_create_flow_action_counter (dcs->obj, offset); if (!cnt_free->action) { rte_errno = errno; - return NULL; + return 0; } } + cnt_idx = MLX5_MAKE_CNT_IDX(pool->index, + (cnt_free - pool->counters_raw)); + cnt_idx += batch * MLX5_CNT_BATCH_OFFSET; /* Update the counter reset values. */ - if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits, + if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits, &cnt_free->bytes)) - return NULL; - cnt_free->shared = shared; - cnt_free->ref_cnt = 1; - cnt_free->id = id; - if (!priv->sh->cmng.query_thread_on) + return 0; + if (cnt_ext) { + cnt_ext->shared = shared; + cnt_ext->ref_cnt = 1; + cnt_ext->id = id; + } + if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on) /* Start the asynchronous batch query by the host thread. */ mlx5_set_query_alarm(priv->sh); TAILQ_REMOVE(&pool->counters, cnt_free, next); @@ -3255,7 +4384,7 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, TAILQ_REMOVE(&cont->pool_list, pool, next); TAILQ_INSERT_TAIL(&cont->pool_list, pool, next); } - return cnt_free; + return cnt_idx; } /** @@ -3264,28 +4393,33 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, * @param[in] dev * Pointer to the Ethernet device structure. * @param[in] counter - * Pointer to the counter handler. + * Index to the counter handler. */ static void -flow_dv_counter_release(struct rte_eth_dev *dev, - struct mlx5_flow_counter *counter) +flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter) { - struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_counter_pool *pool = NULL; + struct mlx5_flow_counter *cnt; + struct mlx5_flow_counter_ext *cnt_ext = NULL; if (!counter) return; - if (priv->counter_fallback) { - flow_dv_counter_release_fallback(dev, counter); - return; - } - if (--counter->ref_cnt == 0) { - struct mlx5_flow_counter_pool *pool = - flow_dv_counter_pool_get(counter); - - /* Put the counter in the end - the last updated one. */ - TAILQ_INSERT_TAIL(&pool->counters, counter, next); - counter->query_gen = rte_atomic64_read(&pool->query_gen); + cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); + MLX5_ASSERT(pool); + if (counter < MLX5_CNT_BATCH_OFFSET) { + cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); + if (cnt_ext && --cnt_ext->ref_cnt) + return; } + /* Put the counter in the end - the last updated one. */ + TAILQ_INSERT_TAIL(&pool->counters, cnt, next); + /* + * Counters released between query trigger and handler need + * to wait the next round of query. Since the packets arrive + * in the gap period will not be taken into account to the + * old counter. + */ + cnt->query_gen = rte_atomic64_read(&pool->start_query_gen); } /** @@ -3320,20 +4454,14 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, NULL, "groups are not supported"); #else - uint32_t max_group = attributes->transfer ? MLX5_MAX_TABLES_FDB : - MLX5_MAX_TABLES; uint32_t table; int ret; ret = mlx5_flow_group_to_table(attributes, external, - attributes->group, + attributes->group, !!priv->fdb_def_rule, &table, error); if (ret) return ret; - if (table >= max_group) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, - "group index out of range"); #endif if (attributes->priority != MLX5_FLOW_PRIO_RSVD && attributes->priority >= priority_max) @@ -3398,14 +4526,43 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, uint8_t next_protocol = 0xff; uint16_t ether_type = 0; int actions_n = 0; + uint8_t item_ipv6_proto = 0; const struct rte_flow_item *gre_item = NULL; - struct rte_flow_item_tcp nic_tcp_mask = { + const struct rte_flow_action_raw_decap *decap; + const struct rte_flow_action_raw_encap *encap; + const struct rte_flow_action_rss *rss; + const struct rte_flow_item_tcp nic_tcp_mask = { .hdr = { .tcp_flags = 0xFF, .src_port = RTE_BE16(UINT16_MAX), .dst_port = RTE_BE16(UINT16_MAX), } }; + const struct rte_flow_item_ipv4 nic_ipv4_mask = { + .hdr = { + .src_addr = RTE_BE32(0xffffffff), + .dst_addr = RTE_BE32(0xffffffff), + .type_of_service = 0xff, + .next_proto_id = 0xff, + .time_to_live = 0xff, + }, + }; + const struct rte_flow_item_ipv6 nic_ipv6_mask = { + .hdr = { + .src_addr = + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + .dst_addr = + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + .vtc_flow = RTE_BE32(0xffffffff), + .proto = 0xff, + .hop_limits = 0xff, + }, + }; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *dev_conf = &priv->config; + uint16_t queue_index = 0xFFFF; if (items == NULL) return -1; @@ -3469,7 +4626,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, &item_flags, &tunnel); ret = mlx5_flow_validate_item_ipv4(items, item_flags, last_item, - ether_type, NULL, + ether_type, + &nic_ipv4_mask, error); if (ret < 0) return ret; @@ -3494,7 +4652,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, &item_flags, &tunnel); ret = mlx5_flow_validate_item_ipv6(items, item_flags, last_item, - ether_type, NULL, + ether_type, + &nic_ipv6_mask, error); if (ret < 0) return ret; @@ -3503,6 +4662,9 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (items->mask != NULL && ((const struct rte_flow_item_ipv6 *) items->mask)->hdr.proto) { + item_ipv6_proto = + ((const struct rte_flow_item_ipv6 *) + items->spec)->hdr.proto; next_protocol = ((const struct rte_flow_item_ipv6 *) items->spec)->hdr.proto; @@ -3578,7 +4740,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; - last_item = MLX5_FLOW_LAYER_VXLAN_GPE; + last_item = MLX5_FLOW_LAYER_GENEVE; break; case RTE_FLOW_ITEM_TYPE_MPLS: ret = mlx5_flow_validate_item_mpls(dev, items, @@ -3588,6 +4750,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, return ret; last_item = MLX5_FLOW_LAYER_MPLS; break; + + case RTE_FLOW_ITEM_TYPE_MARK: + ret = flow_dv_validate_item_mark(dev, items, attr, + error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_ITEM_MARK; + break; case RTE_FLOW_ITEM_TYPE_META: ret = flow_dv_validate_item_meta(dev, items, attr, error); @@ -3609,11 +4779,26 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + item_ipv6_proto = IPPROTO_ICMPV6; last_item = MLX5_FLOW_LAYER_ICMP6; break; + case RTE_FLOW_ITEM_TYPE_TAG: + ret = flow_dv_validate_item_tag(dev, items, + attr, error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_ITEM_TAG; + break; case MLX5_RTE_FLOW_ITEM_TYPE_TAG: case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: break; + case RTE_FLOW_ITEM_TYPE_GTP: + ret = flow_dv_validate_item_gtp(dev, items, item_flags, + error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_LAYER_GTP; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, @@ -3642,21 +4827,61 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; break; case RTE_FLOW_ACTION_TYPE_FLAG: - ret = mlx5_flow_validate_action_flag(action_flags, - attr, error); + ret = flow_dv_validate_action_flag(dev, action_flags, + attr, error); if (ret < 0) return ret; - action_flags |= MLX5_FLOW_ACTION_FLAG; - ++actions_n; + if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { + /* Count all modify-header actions as one. */ + if (!(action_flags & + MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_FLAG | + MLX5_FLOW_ACTION_MARK_EXT; + } else { + action_flags |= MLX5_FLOW_ACTION_FLAG; + ++actions_n; + } break; case RTE_FLOW_ACTION_TYPE_MARK: - ret = mlx5_flow_validate_action_mark(actions, - action_flags, - attr, error); + ret = flow_dv_validate_action_mark(dev, actions, + action_flags, + attr, error); if (ret < 0) return ret; - action_flags |= MLX5_FLOW_ACTION_MARK; - ++actions_n; + if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { + /* Count all modify-header actions as one. */ + if (!(action_flags & + MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_MARK | + MLX5_FLOW_ACTION_MARK_EXT; + } else { + action_flags |= MLX5_FLOW_ACTION_MARK; + ++actions_n; + } + break; + case RTE_FLOW_ACTION_TYPE_SET_META: + ret = flow_dv_validate_action_set_meta(dev, actions, + action_flags, + attr, error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_SET_META; + break; + case RTE_FLOW_ACTION_TYPE_SET_TAG: + ret = flow_dv_validate_action_set_tag(dev, actions, + action_flags, + attr, error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_SET_TAG; break; case RTE_FLOW_ACTION_TYPE_DROP: ret = mlx5_flow_validate_action_drop(action_flags, @@ -3672,16 +4897,21 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, attr, error); if (ret < 0) return ret; + queue_index = ((const struct rte_flow_action_queue *) + (actions->conf))->index; action_flags |= MLX5_FLOW_ACTION_QUEUE; ++actions_n; break; case RTE_FLOW_ACTION_TYPE_RSS: + rss = actions->conf; ret = mlx5_flow_validate_action_rss(actions, action_flags, dev, attr, item_flags, error); if (ret < 0) return ret; + if (rss != NULL && rss->queue_num) + queue_index = rss->queue[0]; action_flags |= MLX5_FLOW_ACTION_RSS; ++actions_n; break; @@ -3703,7 +4933,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; break; case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: - ret = flow_dv_validate_action_push_vlan(action_flags, + ret = flow_dv_validate_action_push_vlan(dev, + action_flags, item_flags, actions, attr, error); @@ -3731,46 +4962,48 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, break; case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: - ret = flow_dv_validate_action_l2_encap(action_flags, + ret = flow_dv_validate_action_l2_encap(dev, + action_flags, actions, attr, error); if (ret < 0) return ret; - action_flags |= actions->type == - RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ? - MLX5_FLOW_ACTION_VXLAN_ENCAP : - MLX5_FLOW_ACTION_NVGRE_ENCAP; + action_flags |= MLX5_FLOW_ACTION_ENCAP; ++actions_n; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: - ret = flow_dv_validate_action_l2_decap(action_flags, - attr, error); + ret = flow_dv_validate_action_decap(dev, action_flags, + attr, error); if (ret < 0) return ret; - action_flags |= actions->type == - RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ? - MLX5_FLOW_ACTION_VXLAN_DECAP : - MLX5_FLOW_ACTION_NVGRE_DECAP; + action_flags |= MLX5_FLOW_ACTION_DECAP; ++actions_n; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: - ret = flow_dv_validate_action_raw_encap(action_flags, - actions, attr, - error); + ret = flow_dv_validate_action_raw_encap_decap + (dev, NULL, actions->conf, attr, &action_flags, + &actions_n, error); if (ret < 0) return ret; - action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP; - ++actions_n; break; case RTE_FLOW_ACTION_TYPE_RAW_DECAP: - ret = flow_dv_validate_action_raw_decap(action_flags, - actions, attr, - error); + decap = actions->conf; + while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) + ; + if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { + encap = NULL; + actions--; + } else { + encap = actions->conf; + } + ret = flow_dv_validate_action_raw_encap_decap + (dev, + decap ? decap : &empty_decap, encap, + attr, &action_flags, &actions_n, + error); if (ret < 0) return ret; - action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; - ++actions_n; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: @@ -3813,6 +5046,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + if (item_ipv6_proto == IPPROTO_ICMPV6) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "Can't change header " + "with ICMPv6 proto"); /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; @@ -3898,6 +5137,44 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, MLX5_FLOW_ACTION_DEC_TCP_ACK; break; case MLX5_RTE_FLOW_ACTION_TYPE_TAG: + case MLX5_RTE_FLOW_ACTION_TYPE_MARK: + case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: + break; + case RTE_FLOW_ACTION_TYPE_METER: + ret = mlx5_flow_validate_action_meter(dev, + action_flags, + actions, attr, + error); + if (ret < 0) + return ret; + action_flags |= MLX5_FLOW_ACTION_METER; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: + ret = flow_dv_validate_action_modify_ipv4_dscp + (action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: + ret = flow_dv_validate_action_modify_ipv6_dscp + (action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; break; default: return rte_flow_error_set(error, ENOTSUP, @@ -3906,21 +5183,28 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, "action not supported"); } } - if ((action_flags & MLX5_FLOW_LAYER_TUNNEL) && - (action_flags & MLX5_FLOW_VLAN_ACTIONS)) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, - actions, - "can't have vxlan and vlan" - " actions in the same rule"); + /* + * Validate the drop action mutual exclusion with other actions. + * Drop action is mutually-exclusive with any other action, except for + * Count action. + */ + if ((action_flags & MLX5_FLOW_ACTION_DROP) && + (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Drop action is mutually-exclusive " + "with any other action, except for " + "Count action"); /* Eswitch has few restrictions on using items and actions */ if (attr->transfer) { - if (action_flags & MLX5_FLOW_ACTION_FLAG) + if (!mlx5_flow_ext_mreg_supported(dev) && + action_flags & MLX5_FLOW_ACTION_FLAG) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "unsupported action FLAG"); - if (action_flags & MLX5_FLOW_ACTION_MARK) + if (!mlx5_flow_ext_mreg_supported(dev) && + action_flags & MLX5_FLOW_ACTION_MARK) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -3947,6 +5231,22 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, actions, "no fate action is found"); } + /* Continue validation for Xcap actions.*/ + if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF || + mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) { + if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) == + MLX5_FLOW_XCAP_ACTIONS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "encap and decap " + "combination aren't supported"); + if (!attr->transfer && attr->ingress && (action_flags & + MLX5_FLOW_ACTION_ENCAP)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "encap is not supported" + " for ingress traffic"); + } return 0; } @@ -3954,6 +5254,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, * Internal preparation function. Allocates the DV flow size, * this size is constant. * + * @param[in] dev + * Pointer to the rte_eth_dev structure. * @param[in] attr * Pointer to the flow attributes. * @param[in] items @@ -3968,26 +5270,47 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, * otherwise NULL and rte_errno is set. */ static struct mlx5_flow * -flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused, +flow_dv_prepare(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr __rte_unused, const struct rte_flow_item items[] __rte_unused, const struct rte_flow_action actions[] __rte_unused, struct rte_flow_error *error) { - uint32_t size = sizeof(struct mlx5_flow); - struct mlx5_flow *flow; + size_t size = sizeof(struct mlx5_flow_handle); + struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; + struct mlx5_priv *priv = dev->data->dev_private; - flow = rte_calloc(__func__, 1, size, 0); - if (!flow) { + /* In case of corrupting the memory. */ + if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { + rte_flow_error_set(error, ENOSPC, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "not free temporary device flow"); + return NULL; + } + dev_handle = rte_calloc(__func__, 1, size, 0); + if (!dev_handle) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "not enough memory to create flow"); + "not enough memory to create flow handle"); return NULL; } - flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); - return flow; + /* No multi-thread supporting. */ + dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; + dev_flow->handle = dev_handle; + dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); + /* + * The matching value needs to be cleared to 0 before using. In the + * past, it will be automatically cleared when using rte_*alloc + * API. The time consumption will be almost the same as before. + */ + memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param)); + dev_flow->ingress = attr->ingress; + dev_flow->dv.transfer = attr->transfer; + return dev_flow; } -#ifndef NDEBUG +#ifdef RTE_LIBRTE_MLX5_DEBUG /** * Sanity check for match mask and value. Similar to check_valid_spec() in * kernel driver. If unmasked bit is present in value, it returns failure. @@ -4073,6 +5396,23 @@ flow_dv_translate_item_eth(void *matcher, void *key, /* The value must be in the range of the mask. */ for (i = 0; i < sizeof(eth_m->dst); ++i) l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i]; + if (eth_v->type) { + /* When ethertype is present set mask for tagged VLAN. */ + MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); + /* Set value for tagged VLAN if ethertype is 802.1Q. */ + if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) || + eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) { + MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, + 1); + /* Return here to avoid setting match on ethertype. */ + return; + } + } + /* + * HW supports match on one Ethertype, the Ethertype following the last + * VLAN tag of the packet (see PRM). + * Set match on ethertype only if ETH header is not followed by VLAN. + */ MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, rte_be_to_cpu_16(eth_m->type)); l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype); @@ -4122,7 +5462,7 @@ flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, * This is workaround, masks are not supported, * and pre-validated. */ - dev_flow->dv.vf_vlan.tag = + dev_flow->handle->vf_vlan.tag = rte_be_to_cpu_16(vlan_v->tci) & 0x0fff; } tci_m = rte_be_to_cpu_16(vlan_m->tci); @@ -4150,6 +5490,8 @@ flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, * Flow matcher value. * @param[in] item * Flow pattern to translate. + * @param[in] item_flags + * Bit-fields that holds the items detected until now. * @param[in] inner * Item is inner pattern. * @param[in] group @@ -4158,6 +5500,7 @@ flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, static void flow_dv_translate_item_ipv4(void *matcher, void *key, const struct rte_flow_item *item, + const uint64_t item_flags, int inner, uint32_t group) { const struct rte_flow_item_ipv4 *ipv4_m = item->mask; @@ -4168,6 +5511,7 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, .dst_addr = RTE_BE32(0xffffffff), .type_of_service = 0xff, .next_proto_id = 0xff, + .time_to_live = 0xff, }, }; void *headers_m; @@ -4190,6 +5534,13 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, else MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4); + /* + * On outer header (which must contains L2), or inner header with L2, + * set cvlan_tag mask bit to mark this packet as untagged. + * This should be done even if item->spec is empty. + */ + if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) + MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); if (!ipv4_v) return; if (!ipv4_m) @@ -4217,6 +5568,10 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, ipv4_m->hdr.next_proto_id); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id); + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, + ipv4_m->hdr.time_to_live); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, + ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live); } /** @@ -4228,6 +5583,8 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, * Flow matcher value. * @param[in] item * Flow pattern to translate. + * @param[in] item_flags + * Bit-fields that holds the items detected until now. * @param[in] inner * Item is inner pattern. * @param[in] group @@ -4236,6 +5593,7 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, static void flow_dv_translate_item_ipv6(void *matcher, void *key, const struct rte_flow_item *item, + const uint64_t item_flags, int inner, uint32_t group) { const struct rte_flow_item_ipv6 *ipv6_m = item->mask; @@ -4278,6 +5636,13 @@ flow_dv_translate_item_ipv6(void *matcher, void *key, else MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6); + /* + * On outer header (which must contains L2), or inner header with L2, + * set cvlan_tag mask bit to mark this packet as untagged. + * This should be done even if item->spec is empty. + */ + if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) + MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); if (!ipv6_v) return; if (!ipv6_m) @@ -4321,6 +5686,11 @@ flow_dv_translate_item_ipv6(void *matcher, void *key, ipv6_m->hdr.proto); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, ipv6_v->hdr.proto & ipv6_m->hdr.proto); + /* Hop limit. */ + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, + ipv6_m->hdr.hop_limits); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, + ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits); } /** @@ -4443,13 +5813,13 @@ flow_dv_translate_item_gre_key(void *matcher, void *key, void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); + /* GRE K bit must be on and should already be validated */ + MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1); + MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1); if (!key_v) return; if (!key_m) key_m = &gre_key_default_mask; - /* GRE K bit must be on and should already be validated */ - MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1); - MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1); MLX5_SET(fte_match_set_misc, misc_m, gre_key_h, rte_be_to_cpu_32(*key_m) >> 8); MLX5_SET(fte_match_set_misc, misc_v, gre_key_h, @@ -4647,6 +6017,76 @@ flow_dv_translate_item_vxlan(void *matcher, void *key, vni_v[i] = vni_m[i] & vxlan_v->vni[i]; } +/** + * Add VXLAN-GPE item to matcher and to the value. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[in] inner + * Item is inner pattern. + */ + +static void +flow_dv_translate_item_vxlan_gpe(void *matcher, void *key, + const struct rte_flow_item *item, int inner) +{ + const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask; + const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec; + void *headers_m; + void *headers_v; + void *misc_m = + MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3); + void *misc_v = + MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + char *vni_m; + char *vni_v; + uint16_t dport; + int size; + int i; + uint8_t flags_m = 0xff; + uint8_t flags_v = 0xc; + + if (inner) { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + inner_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); + } else { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + outer_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); + } + dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? + MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; + if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { + MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); + } + if (!vxlan_v) + return; + if (!vxlan_m) + vxlan_m = &rte_flow_item_vxlan_gpe_mask; + size = sizeof(vxlan_m->vni); + vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni); + vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni); + memcpy(vni_m, vxlan_m->vni, size); + for (i = 0; i < size; ++i) + vni_v[i] = vni_m[i] & vxlan_v->vni[i]; + if (vxlan_m->flags) { + flags_m = vxlan_m->flags; + flags_v = vxlan_v->flags; + } + MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m); + MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v); + MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol, + vxlan_m->protocol); + MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol, + vxlan_v->protocol); +} + /** * Add Geneve item to matcher and to the value. * @@ -4800,46 +6240,202 @@ flow_dv_translate_item_mpls(void *matcher, void *key, misc2_v, outer_first_mpls); } - break; - } - if (out_mpls_m && out_mpls_v) { - *out_mpls_m = *in_mpls_m; - *out_mpls_v = *in_mpls_v & *in_mpls_m; + break; + } + if (out_mpls_m && out_mpls_v) { + *out_mpls_m = *in_mpls_m; + *out_mpls_v = *in_mpls_v & *in_mpls_m; + } +} + +/** + * Add metadata register item to matcher + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] reg_type + * Type of device metadata register + * @param[in] value + * Register value + * @param[in] mask + * Register mask + */ +static void +flow_dv_match_meta_reg(void *matcher, void *key, + enum modify_reg reg_type, + uint32_t data, uint32_t mask) +{ + void *misc2_m = + MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); + void *misc2_v = + MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); + uint32_t temp; + + data &= mask; + switch (reg_type) { + case REG_A: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data); + break; + case REG_B: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data); + break; + case REG_C_0: + /* + * The metadata register C0 field might be divided into + * source vport index and META item value, we should set + * this field according to specified mask, not as whole one. + */ + temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0); + temp |= mask; + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp); + temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0); + temp &= ~mask; + temp |= data; + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp); + break; + case REG_C_1: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data); + break; + case REG_C_2: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data); + break; + case REG_C_3: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data); + break; + case REG_C_4: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data); + break; + case REG_C_5: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data); + break; + case REG_C_6: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data); + break; + case REG_C_7: + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask); + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data); + break; + default: + MLX5_ASSERT(false); + break; + } +} + +/** + * Add MARK item to matcher + * + * @param[in] dev + * The device to configure through. + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + */ +static void +flow_dv_translate_item_mark(struct rte_eth_dev *dev, + void *matcher, void *key, + const struct rte_flow_item *item) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_item_mark *mark; + uint32_t value; + uint32_t mask; + + mark = item->mask ? (const void *)item->mask : + &rte_flow_item_mark_mask; + mask = mark->id & priv->sh->dv_mark_mask; + mark = (const void *)item->spec; + MLX5_ASSERT(mark); + value = mark->id & priv->sh->dv_mark_mask & mask; + if (mask) { + enum modify_reg reg; + + /* Get the metadata register index for the mark. */ + reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL); + MLX5_ASSERT(reg > 0); + if (reg == REG_C_0) { + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t msk_c0 = priv->sh->dv_regc0_mask; + uint32_t shl_c0 = rte_bsf32(msk_c0); + + mask &= msk_c0; + mask <<= shl_c0; + value <<= shl_c0; + } + flow_dv_match_meta_reg(matcher, key, reg, value, mask); } } /** * Add META item to matcher * + * @param[in] dev + * The devich to configure through. * @param[in, out] matcher * Flow matcher. * @param[in, out] key * Flow matcher value. + * @param[in] attr + * Attributes of flow that includes this item. * @param[in] item * Flow pattern to translate. - * @param[in] inner - * Item is inner pattern. */ static void -flow_dv_translate_item_meta(void *matcher, void *key, +flow_dv_translate_item_meta(struct rte_eth_dev *dev, + void *matcher, void *key, + const struct rte_flow_attr *attr, const struct rte_flow_item *item) { const struct rte_flow_item_meta *meta_m; const struct rte_flow_item_meta *meta_v; - void *misc2_m = - MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); - void *misc2_v = - MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); meta_m = (const void *)item->mask; if (!meta_m) meta_m = &rte_flow_item_meta_mask; meta_v = (const void *)item->spec; if (meta_v) { - MLX5_SET(fte_match_set_misc2, misc2_m, - metadata_reg_a, meta_m->data); - MLX5_SET(fte_match_set_misc2, misc2_v, - metadata_reg_a, meta_v->data & meta_m->data); + int reg; + uint32_t value = meta_v->data; + uint32_t mask = meta_m->data; + + reg = flow_dv_get_metadata_reg(dev, attr, NULL); + if (reg < 0) + return; + /* + * In datapath code there is no endianness + * coversions for perfromance reasons, all + * pattern conversions are done in rte_flow. + */ + value = rte_cpu_to_be_32(value); + mask = rte_cpu_to_be_32(mask); + if (reg == REG_C_0) { + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t msk_c0 = priv->sh->dv_regc0_mask; + uint32_t shl_c0 = rte_bsf32(msk_c0); +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask); + + value >>= shr_c0; + mask >>= shr_c0; +#endif + value <<= shl_c0; + mask <<= shl_c0; + MLX5_ASSERT(msk_c0); + MLX5_ASSERT(!(~msk_c0 & mask)); + } + flow_dv_match_meta_reg(matcher, key, reg, value, mask); } } @@ -4857,18 +6453,14 @@ static void flow_dv_translate_item_meta_vport(void *matcher, void *key, uint32_t value, uint32_t mask) { - void *misc2_m = - MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); - void *misc2_v = - MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); - - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, mask); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, value); + flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask); } /** * Add tag item to matcher * + * @param[in] dev + * The devich to configure through. * @param[in, out] matcher * Flow matcher. * @param[in, out] key @@ -4877,81 +6469,56 @@ flow_dv_translate_item_meta_vport(void *matcher, void *key, * Flow pattern to translate. */ static void -flow_dv_translate_item_tag(void *matcher, void *key, - const struct rte_flow_item *item) +flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev, + void *matcher, void *key, + const struct rte_flow_item *item) { - void *misc2_m = - MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); - void *misc2_v = - MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); const struct mlx5_rte_flow_item_tag *tag_v = item->spec; const struct mlx5_rte_flow_item_tag *tag_m = item->mask; - enum modify_reg reg = tag_v->id; - rte_be32_t value = tag_v->data; - rte_be32_t mask = tag_m->data; - - switch (reg) { - case REG_A: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, - rte_be_to_cpu_32(value)); - break; - case REG_B: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, - rte_be_to_cpu_32(value)); - break; - case REG_C_0: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, - rte_be_to_cpu_32(value)); - break; - case REG_C_1: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, - rte_be_to_cpu_32(value)); - break; - case REG_C_2: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, - rte_be_to_cpu_32(value)); - break; - case REG_C_3: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, - rte_be_to_cpu_32(value)); - break; - case REG_C_4: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, - rte_be_to_cpu_32(value)); - break; - case REG_C_5: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, - rte_be_to_cpu_32(value)); - break; - case REG_C_6: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, - rte_be_to_cpu_32(value)); - break; - case REG_C_7: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, - rte_be_to_cpu_32(mask)); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, - rte_be_to_cpu_32(value)); - break; + uint32_t mask, value; + + MLX5_ASSERT(tag_v); + value = tag_v->data; + mask = tag_m ? tag_m->data : UINT32_MAX; + if (tag_v->id == REG_C_0) { + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t msk_c0 = priv->sh->dv_regc0_mask; + uint32_t shl_c0 = rte_bsf32(msk_c0); + + mask &= msk_c0; + mask <<= shl_c0; + value <<= shl_c0; } + flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask); +} + +/** + * Add TAG item to matcher + * + * @param[in] dev + * The devich to configure through. + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + */ +static void +flow_dv_translate_item_tag(struct rte_eth_dev *dev, + void *matcher, void *key, + const struct rte_flow_item *item) +{ + const struct rte_flow_item_tag *tag_v = item->spec; + const struct rte_flow_item_tag *tag_m = item->mask; + enum modify_reg reg; + + MLX5_ASSERT(tag_v); + tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask; + /* Get the metadata register index for the tag. */ + reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL); + MLX5_ASSERT(reg > 0); + flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data); } /** @@ -5003,7 +6570,7 @@ flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, mask = pid_m ? pid_m->id : 0xffff; id = pid_v ? pid_v->id : dev->data->port_id; - priv = mlx5_port_to_eswitch_info(id); + priv = mlx5_port_to_eswitch_info(id, item == NULL); if (!priv) return -rte_errno; /* Translate to vport field or to metadata, depending on mode. */ @@ -5056,6 +6623,12 @@ flow_dv_translate_item_icmp6(void *matcher, void *key, return; if (!icmp6_m) icmp6_m = &rte_flow_item_icmp6_mask; + /* + * Force flow only to match the non-fragmented IPv6 ICMPv6 packets. + * If only the protocol is specified, no need to match the frag. + */ + MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type); MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type, icmp6_v->type & icmp6_m->type); @@ -5103,6 +6676,12 @@ flow_dv_translate_item_icmp(void *matcher, void *key, return; if (!icmp_m) icmp_m = &rte_flow_item_icmp_mask; + /* + * Force flow only to match the non-fragmented IPv4 ICMP packets. + * If only the protocol is specified, no need to match the frag. + */ + MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type, icmp_m->hdr.icmp_type); MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type, @@ -5113,6 +6692,57 @@ flow_dv_translate_item_icmp(void *matcher, void *key, icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code); } +/** + * Add GTP item to matcher and to the value. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[in] inner + * Item is inner pattern. + */ +static void +flow_dv_translate_item_gtp(void *matcher, void *key, + const struct rte_flow_item *item, int inner) +{ + const struct rte_flow_item_gtp *gtp_m = item->mask; + const struct rte_flow_item_gtp *gtp_v = item->spec; + void *headers_m; + void *headers_v; + void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, + misc_parameters_3); + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + uint16_t dport = RTE_GTPU_UDP_PORT; + + if (inner) { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + inner_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); + } else { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + outer_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); + } + if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { + MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); + } + if (!gtp_v) + return; + if (!gtp_m) + gtp_m = &rte_flow_item_gtp_mask; + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type); + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type, + gtp_v->msg_type & gtp_m->msg_type); + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid, + rte_be_to_cpu_32(gtp_m->teid)); + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid, + rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid)); +} + static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; #define HEADER_IS_ZERO(match_criteria, headers) \ @@ -5155,7 +6785,7 @@ flow_dv_matcher_enable(uint32_t *match_criteria) /** * Get a flow table. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in] table_id * Table id to use. @@ -5178,47 +6808,76 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_tbl_resource *tbl; - -#ifdef HAVE_MLX5DV_DR - if (transfer) { - tbl = &sh->fdb_tbl[table_id]; - if (!tbl->obj) - tbl->obj = mlx5_glue->dr_create_flow_tbl - (sh->fdb_domain, table_id); - } else if (egress) { - tbl = &sh->tx_tbl[table_id]; - if (!tbl->obj) - tbl->obj = mlx5_glue->dr_create_flow_tbl - (sh->tx_domain, table_id); - } else { - tbl = &sh->rx_tbl[table_id]; - if (!tbl->obj) - tbl->obj = mlx5_glue->dr_create_flow_tbl - (sh->rx_domain, table_id); + union mlx5_flow_tbl_key table_key = { + { + .table_id = table_id, + .reserved = 0, + .domain = !!transfer, + .direction = !!egress, + } + }; + struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, + table_key.v64); + struct mlx5_flow_tbl_data_entry *tbl_data; + int ret; + void *domain; + + if (pos) { + tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, + entry); + tbl = &tbl_data->tbl; + rte_atomic32_inc(&tbl->refcnt); + return tbl; } - if (!tbl->obj) { + tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0); + if (!tbl_data) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create table"); + NULL, + "cannot allocate flow table data entry"); return NULL; } - rte_atomic32_inc(&tbl->refcnt); - return tbl; -#else - (void)error; - (void)tbl; + tbl = &tbl_data->tbl; + pos = &tbl_data->entry; if (transfer) - return &sh->fdb_tbl[table_id]; + domain = sh->fdb_domain; else if (egress) - return &sh->tx_tbl[table_id]; + domain = sh->tx_domain; else - return &sh->rx_tbl[table_id]; -#endif + domain = sh->rx_domain; + tbl->obj = mlx5_glue->dr_create_flow_tbl(domain, table_id); + if (!tbl->obj) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create flow table object"); + rte_free(tbl_data); + return NULL; + } + /* + * No multi-threads now, but still better to initialize the reference + * count before insert it into the hash list. + */ + rte_atomic32_init(&tbl->refcnt); + /* Jump action reference count is initialized here. */ + rte_atomic32_init(&tbl_data->jump.refcnt); + pos->key = table_key.v64; + ret = mlx5_hlist_insert(sh->flow_tbls, pos); + if (ret < 0) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot insert flow table data entry"); + mlx5_glue->dr_destroy_flow_tbl(tbl->obj); + rte_free(tbl_data); + } + rte_atomic32_inc(&tbl->refcnt); + return tbl; } /** * Release a flow table. * + * @param[in] dev + * Pointer to rte_eth_dev structure. * @param[in] tbl * Table resource to be released. * @@ -5226,13 +6885,24 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, * Returns 0 if table was released, else return 1; */ static int -flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl) +flow_dv_tbl_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow_tbl_resource *tbl) { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!tbl) return 0; if (rte_atomic32_dec_and_test(&tbl->refcnt)) { + struct mlx5_hlist_entry *pos = &tbl_data->entry; + mlx5_glue->dr_destroy_flow_tbl(tbl->obj); tbl->obj = NULL; + /* remove the entry from the hash list and free memory. */ + mlx5_hlist_remove(sh->flow_tbls, pos); + rte_free(tbl_data); return 0; } return 1; @@ -5241,10 +6911,12 @@ flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl) /** * Register the flow matcher. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in, out] matcher * Pointer to flow matcher. + * @param[in, out] key + * Pointer to flow table key. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -5256,6 +6928,7 @@ flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl) static int flow_dv_matcher_register(struct rte_eth_dev *dev, struct mlx5_flow_dv_matcher *matcher, + union mlx5_flow_tbl_key *key, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { @@ -5266,69 +6939,73 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, .type = IBV_FLOW_ATTR_NORMAL, .match_mask = (void *)&matcher->mask, }; - struct mlx5_flow_tbl_resource *tbl = NULL; + struct mlx5_flow_tbl_resource *tbl; + struct mlx5_flow_tbl_data_entry *tbl_data; + tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, + key->domain, error); + if (!tbl) + return -rte_errno; /* No need to refill the error info */ + tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); /* Lookup from cache. */ - LIST_FOREACH(cache_matcher, &sh->matchers, next) { + LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) { if (matcher->crc == cache_matcher->crc && matcher->priority == cache_matcher->priority && - matcher->egress == cache_matcher->egress && - matcher->group == cache_matcher->group && - matcher->transfer == cache_matcher->transfer && !memcmp((const void *)matcher->mask.buf, (const void *)cache_matcher->mask.buf, cache_matcher->mask.size)) { DRV_LOG(DEBUG, - "priority %hd use %s matcher %p: refcnt %d++", + "%s group %u priority %hd use %s " + "matcher %p: refcnt %d++", + key->domain ? "FDB" : "NIC", key->table_id, cache_matcher->priority, - cache_matcher->egress ? "tx" : "rx", + key->direction ? "tx" : "rx", (void *)cache_matcher, rte_atomic32_read(&cache_matcher->refcnt)); rte_atomic32_inc(&cache_matcher->refcnt); - dev_flow->dv.matcher = cache_matcher; + dev_flow->handle->dvh.matcher = cache_matcher; + /* old matcher should not make the table ref++. */ + flow_dv_tbl_resource_release(dev, tbl); return 0; } } /* Register new matcher. */ cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0); - if (!cache_matcher) + if (!cache_matcher) { + flow_dv_tbl_resource_release(dev, tbl); return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate matcher memory"); - tbl = flow_dv_tbl_resource_get(dev, matcher->group, - matcher->egress, matcher->transfer, - error); - if (!tbl) { - rte_free(cache_matcher); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create table"); } *cache_matcher = *matcher; dv_attr.match_criteria_enable = flow_dv_matcher_enable(cache_matcher->mask.buf); dv_attr.priority = matcher->priority; - if (matcher->egress) + if (key->direction) dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; cache_matcher->matcher_object = mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj); if (!cache_matcher->matcher_object) { rte_free(cache_matcher); #ifdef HAVE_MLX5DV_DR - flow_dv_tbl_resource_release(tbl); + flow_dv_tbl_resource_release(dev, tbl); #endif return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create matcher"); } + /* Save the table information */ + cache_matcher->tbl = tbl; + rte_atomic32_init(&cache_matcher->refcnt); + /* only matcher ref++, table ref++ already done above in get API. */ rte_atomic32_inc(&cache_matcher->refcnt); - LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next); - dev_flow->dv.matcher = cache_matcher; - DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d", + LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); + dev_flow->handle->dvh.matcher = cache_matcher; + DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", + key->domain ? "FDB" : "NIC", key->table_id, cache_matcher->priority, - cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher, + key->direction ? "tx" : "rx", (void *)cache_matcher, rte_atomic32_read(&cache_matcher->refcnt)); - rte_atomic32_inc(&tbl->refcnt); return 0; } @@ -5337,8 +7014,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, * * @param dev[in, out] * Pointer to rte_eth_dev structure. - * @param[in, out] resource - * Pointer to tag resource. + * @param[in, out] tag_be24 + * Tag value in big endian then R-shift 8. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -5350,34 +7027,35 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, static int flow_dv_tag_resource_register (struct rte_eth_dev *dev, - struct mlx5_flow_dv_tag_resource *resource, + uint32_t tag_be24, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_tag_resource *cache_resource; + struct mlx5_hlist_entry *entry; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &sh->tags, next) { - if (resource->tag == cache_resource->tag) { - DRV_LOG(DEBUG, "tag resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->flow->tag_resource = cache_resource; - return 0; - } + entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24); + if (entry) { + cache_resource = container_of + (entry, struct mlx5_flow_dv_tag_resource, entry); + rte_atomic32_inc(&cache_resource->refcnt); + dev_flow->handle->dvh.tag_resource = cache_resource; + DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + return 0; } - /* Register new resource. */ + /* Register new resource. */ cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); - *cache_resource = *resource; - cache_resource->action = mlx5_glue->dv_create_flow_action_tag - (resource->tag); + cache_resource->entry.key = (uint64_t)tag_be24; + cache_resource->action = mlx5_glue->dv_create_flow_action_tag(tag_be24); if (!cache_resource->action) { rte_free(cache_resource); return rte_flow_error_set(error, ENOMEM, @@ -5386,9 +7064,15 @@ flow_dv_tag_resource_register } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&sh->tags, cache_resource, next); - dev_flow->flow->tag_resource = cache_resource; - DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++", + if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) { + mlx5_glue->destroy_flow_action(cache_resource->action); + rte_free(cache_resource); + return rte_flow_error_set(error, EEXIST, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot insert tag"); + } + dev_flow->handle->dvh.tag_resource = cache_resource; + DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); return 0; @@ -5409,13 +7093,16 @@ static int flow_dv_tag_release(struct rte_eth_dev *dev, struct mlx5_flow_dv_tag_resource *tag) { - assert(tag); + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + + MLX5_ASSERT(tag); DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", dev->data->port_id, (void *)tag, rte_atomic32_read(&tag->refcnt)); if (rte_atomic32_dec_and_test(&tag->refcnt)) { claim_zero(mlx5_glue->destroy_flow_action(tag->action)); - LIST_REMOVE(tag, next); + mlx5_hlist_remove(sh->tag_table, &tag->entry); DRV_LOG(DEBUG, "port %u tag %p: removed", dev->data->port_id, (void *)tag); rte_free(tag); @@ -5451,16 +7138,26 @@ flow_dv_translate_action_port_id(struct rte_eth_dev *dev, (const struct rte_flow_action_port_id *)action->conf; port = conf->original ? dev->data->port_id : conf->id; - priv = mlx5_port_to_eswitch_info(port); + priv = mlx5_port_to_eswitch_info(port, false); if (!priv) return rte_flow_error_set(error, -rte_errno, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "No eswitch info was found for port"); - if (priv->vport_meta_mask) - *dst_port_id = priv->vport_meta_tag; - else - *dst_port_id = priv->vport_id; +#ifdef HAVE_MLX5DV_DR_DEVX_PORT + /* + * This parameter is transferred to + * mlx5dv_dr_action_create_dest_ib_port(). + */ + *dst_port_id = priv->ibv_port; +#else + /* + * Legacy mode, no LAG configurations is supported. + * This parameter is transferred to + * mlx5dv_dr_action_create_dest_vport(). + */ + *dst_port_id = priv->vport_id; +#endif return 0; } @@ -5510,7 +7207,77 @@ flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev, } /** - * Fill the flow with DV spec. + * Set the hash fields according to the @p flow information. + * + * @param[in] dev_flow + * Pointer to the mlx5_flow. + */ +static void +flow_dv_hashfields_set(struct mlx5_flow *dev_flow) +{ + struct rte_flow *flow = dev_flow->flow; + uint64_t items = dev_flow->handle->layers; + int rss_inner = 0; + uint64_t rss_types = rte_eth_rss_hf_refine(flow->rss.types); + + dev_flow->hash_fields = 0; +#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT + if (flow->rss.level >= 2) { + dev_flow->hash_fields |= IBV_RX_HASH_INNER; + rss_inner = 1; + } +#endif + if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) || + (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) { + if (rss_types & MLX5_IPV4_LAYER_TYPES) { + if (rss_types & ETH_RSS_L3_SRC_ONLY) + dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4; + else if (rss_types & ETH_RSS_L3_DST_ONLY) + dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4; + else + dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH; + } + } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) || + (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) { + if (rss_types & MLX5_IPV6_LAYER_TYPES) { + if (rss_types & ETH_RSS_L3_SRC_ONLY) + dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6; + else if (rss_types & ETH_RSS_L3_DST_ONLY) + dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6; + else + dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH; + } + } + if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) || + (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) { + if (rss_types & ETH_RSS_UDP) { + if (rss_types & ETH_RSS_L4_SRC_ONLY) + dev_flow->hash_fields |= + IBV_RX_HASH_SRC_PORT_UDP; + else if (rss_types & ETH_RSS_L4_DST_ONLY) + dev_flow->hash_fields |= + IBV_RX_HASH_DST_PORT_UDP; + else + dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH; + } + } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) || + (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) { + if (rss_types & ETH_RSS_TCP) { + if (rss_types & ETH_RSS_L4_SRC_ONLY) + dev_flow->hash_fields |= + IBV_RX_HASH_SRC_PORT_TCP; + else if (rss_types & ETH_RSS_L4_DST_ONLY) + dev_flow->hash_fields |= + IBV_RX_HASH_DST_PORT_TCP; + else + dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH; + } + } +} + +/** + * Fill the flow with DV spec, lock free + * (mutex should be acquired by caller). * * @param[in] dev * Pointer to rte_eth_dev structure. @@ -5529,15 +7296,17 @@ flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_translate(struct rte_eth_dev *dev, - struct mlx5_flow *dev_flow, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - struct rte_flow_error *error) +__flow_dv_translate(struct rte_eth_dev *dev, + struct mlx5_flow *dev_flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *dev_conf = &priv->config; struct rte_flow *flow = dev_flow->flow; + struct mlx5_flow_handle *handle = dev_flow->handle; uint64_t item_flags = 0; uint64_t last_item = 0; uint64_t action_flags = 0; @@ -5549,12 +7318,16 @@ flow_dv_translate(struct rte_eth_dev *dev, }; int actions_n = 0; bool actions_end = false; - struct mlx5_flow_dv_modify_hdr_resource res = { - .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : - MLX5DV_FLOW_TABLE_TYPE_NIC_RX - }; + union { + struct mlx5_flow_dv_modify_hdr_resource res; + uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) + + sizeof(struct mlx5_modification_cmd) * + (MLX5_MAX_MODIFY_NUM + 1)]; + } mhdr_dummy; + struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; union flow_dv_attr flow_attr = { .attr = 0 }; - struct mlx5_flow_dv_tag_resource tag_resource; + uint32_t tag_be; + union mlx5_flow_tbl_key tbl_key; uint32_t modify_action_position = UINT32_MAX; void *match_mask = matcher.mask.buf; void *match_value = dev_flow->dv.value.buf; @@ -5563,15 +7336,19 @@ flow_dv_translate(struct rte_eth_dev *dev, uint32_t table; int ret = 0; + mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : + MLX5DV_FLOW_TABLE_TYPE_NIC_RX; ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group, - &table, error); + !!priv->fdb_def_rule, &table, error); if (ret) return ret; - flow->group = table; + dev_flow->dv.group = table; if (attr->transfer) - res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; + mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; if (priority == MLX5_FLOW_PRIO_RSVD) - priority = priv->config.flow_prio - 1; + priority = dev_conf->flow_prio - 1; + /* number of actions must be set to 0 in case of dirty stack. */ + mhdr_res->actions_num = 0; for (; !actions_end ; actions++) { const struct rte_flow_action_queue *queue; const struct rte_flow_action_rss *rss; @@ -5579,7 +7356,7 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_count *count = action->conf; const uint8_t *rss_key; const struct rte_flow_action_jump *jump_data; - struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource; + const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; @@ -5598,68 +7375,119 @@ flow_dv_translate(struct rte_eth_dev *dev, (dev, &port_id_resource, dev_flow, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->dv.port_id_action->action; + handle->dvh.port_id_action->action; action_flags |= MLX5_FLOW_ACTION_PORT_ID; break; case RTE_FLOW_ACTION_TYPE_FLAG: - tag_resource.tag = - mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); - if (!flow->tag_resource) - if (flow_dv_tag_resource_register - (dev, &tag_resource, dev_flow, error)) - return errno; - dev_flow->dv.actions[actions_n++] = - flow->tag_resource->action; action_flags |= MLX5_FLOW_ACTION_FLAG; + if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { + struct rte_flow_action_mark mark = { + .id = MLX5_FLOW_MARK_DEFAULT, + }; + + if (flow_dv_convert_action_mark(dev, &mark, + mhdr_res, + error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_MARK_EXT; + break; + } + tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); + /* + * Only one FLAG or MARK is supported per device flow + * right now. So the pointer to the tag resource must be + * zero before the register process. + */ + MLX5_ASSERT(!handle->dvh.tag_resource); + if (flow_dv_tag_resource_register(dev, tag_be, + dev_flow, error)) + return -rte_errno; + dev_flow->dv.actions[actions_n++] = + handle->dvh.tag_resource->action; break; case RTE_FLOW_ACTION_TYPE_MARK: - tag_resource.tag = mlx5_flow_mark_set + action_flags |= MLX5_FLOW_ACTION_MARK; + if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { + const struct rte_flow_action_mark *mark = + (const struct rte_flow_action_mark *) + actions->conf; + + if (flow_dv_convert_action_mark(dev, mark, + mhdr_res, + error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_MARK_EXT; + break; + } + /* Fall-through */ + case MLX5_RTE_FLOW_ACTION_TYPE_MARK: + /* Legacy (non-extensive) MARK action. */ + tag_be = mlx5_flow_mark_set (((const struct rte_flow_action_mark *) (actions->conf))->id); - if (!flow->tag_resource) - if (flow_dv_tag_resource_register - (dev, &tag_resource, dev_flow, error)) - return errno; + MLX5_ASSERT(!handle->dvh.tag_resource); + if (flow_dv_tag_resource_register(dev, tag_be, + dev_flow, error)) + return -rte_errno; dev_flow->dv.actions[actions_n++] = - flow->tag_resource->action; - action_flags |= MLX5_FLOW_ACTION_MARK; + handle->dvh.tag_resource->action; + break; + case RTE_FLOW_ACTION_TYPE_SET_META: + if (flow_dv_convert_action_set_meta + (dev, mhdr_res, attr, + (const struct rte_flow_action_set_meta *) + actions->conf, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_SET_META; + break; + case RTE_FLOW_ACTION_TYPE_SET_TAG: + if (flow_dv_convert_action_set_tag + (dev, mhdr_res, + (const struct rte_flow_action_set_tag *) + actions->conf, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_SET_TAG; break; case RTE_FLOW_ACTION_TYPE_DROP: action_flags |= MLX5_FLOW_ACTION_DROP; break; case RTE_FLOW_ACTION_TYPE_QUEUE: + MLX5_ASSERT(flow->rss.queue); queue = actions->conf; flow->rss.queue_num = 1; - (*flow->queue)[0] = queue->index; + (*flow->rss.queue)[0] = queue->index; action_flags |= MLX5_FLOW_ACTION_QUEUE; break; case RTE_FLOW_ACTION_TYPE_RSS: + MLX5_ASSERT(flow->rss.queue); rss = actions->conf; - if (flow->queue) - memcpy((*flow->queue), rss->queue, + if (flow->rss.queue) + memcpy((*flow->rss.queue), rss->queue, rss->queue_num * sizeof(uint16_t)); flow->rss.queue_num = rss->queue_num; /* NULL RSS key indicates default RSS key. */ rss_key = !rss->key ? rss_hash_default_key : rss->key; - memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN); - /* RSS type 0 indicates default RSS type ETH_RSS_IP. */ - flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; - flow->rss.level = rss->level; + memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN); + /* + * rss->level and rss.types should be set in advance + * when expanding items for RSS. + */ action_flags |= MLX5_FLOW_ACTION_RSS; break; case RTE_FLOW_ACTION_TYPE_COUNT: - if (!priv->config.devx) { + if (!dev_conf->devx) { rte_errno = ENOTSUP; goto cnt_err; } flow->counter = flow_dv_counter_alloc(dev, - count->shared, - count->id, - flow->group); - if (flow->counter == NULL) + count->shared, + count->id, + dev_flow->dv.group); + if (!flow->counter) goto cnt_err; dev_flow->dv.actions[actions_n++] = - flow->counter->action; + (flow_dv_counter_get_by_idx(dev, + flow->counter, NULL))->action; action_flags |= MLX5_FLOW_ACTION_COUNT; break; cnt_err: @@ -5701,12 +7529,13 @@ cnt_err: (dev, attr, &vlan, dev_flow, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->dv.push_vlan_res->action; + handle->dvh.push_vlan_res->action; action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; break; case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: /* of_vlan_push action handled this action */ - assert(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN); + MLX5_ASSERT(action_flags & + MLX5_FLOW_ACTION_OF_PUSH_VLAN); break; case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) @@ -5715,7 +7544,7 @@ cnt_err: mlx5_update_vlan_vid_pcp(actions, &vlan); /* If no VLAN push - this is a modify header action */ if (flow_dv_convert_action_modify_vlan_vid - (&res, actions, error)) + (mhdr_res, actions, error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; break; @@ -5727,11 +7556,8 @@ cnt_err: error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->dv.encap_decap->verbs_action; - action_flags |= actions->type == - RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ? - MLX5_FLOW_ACTION_VXLAN_ENCAP : - MLX5_FLOW_ACTION_NVGRE_ENCAP; + handle->dvh.encap_decap->verbs_action; + action_flags |= MLX5_FLOW_ACTION_ENCAP; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: @@ -5740,20 +7566,17 @@ cnt_err: error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->dv.encap_decap->verbs_action; - action_flags |= actions->type == - RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ? - MLX5_FLOW_ACTION_VXLAN_DECAP : - MLX5_FLOW_ACTION_NVGRE_DECAP; + handle->dvh.encap_decap->verbs_action; + action_flags |= MLX5_FLOW_ACTION_DECAP; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: /* Handle encap with preceding decap. */ - if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) { + if (action_flags & MLX5_FLOW_ACTION_DECAP) { if (flow_dv_create_action_raw_encap (dev, actions, dev_flow, attr, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->dv.encap_decap->verbs_action; + handle->dvh.encap_decap->verbs_action; } else { /* Handle encap without preceding decap. */ if (flow_dv_create_action_l2_encap @@ -5761,32 +7584,29 @@ cnt_err: error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->dv.encap_decap->verbs_action; + handle->dvh.encap_decap->verbs_action; } - action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP; + action_flags |= MLX5_FLOW_ACTION_ENCAP; break; case RTE_FLOW_ACTION_TYPE_RAW_DECAP: - /* Check if this decap is followed by encap. */ - for (; action->type != RTE_FLOW_ACTION_TYPE_END && - action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP; - action++) { - } - /* Handle decap only if it isn't followed by encap. */ + while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID) + ; if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { if (flow_dv_create_action_l2_decap (dev, dev_flow, attr->transfer, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = - dev_flow->dv.encap_decap->verbs_action; + handle->dvh.encap_decap->verbs_action; } /* If decap is followed by encap, handle it at encap. */ - action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; + action_flags |= MLX5_FLOW_ACTION_DECAP; break; case RTE_FLOW_ACTION_TYPE_JUMP: jump_data = action->conf; ret = mlx5_flow_group_to_table(attr, dev_flow->external, - jump_data->group, &table, - error); + jump_data->group, + !!priv->fdb_def_rule, + &table, error); if (ret) return ret; tbl = flow_dv_tbl_resource_get(dev, table, @@ -5798,10 +7618,9 @@ cnt_err: RTE_FLOW_ERROR_TYPE_ACTION, NULL, "cannot create jump action."); - jump_tbl_resource.tbl = tbl; if (flow_dv_jump_tbl_resource_register - (dev, &jump_tbl_resource, dev_flow, error)) { - flow_dv_tbl_resource_release(tbl); + (dev, tbl, dev_flow, error)) { + flow_dv_tbl_resource_release(dev, tbl); return rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_ACTION, @@ -5809,13 +7628,13 @@ cnt_err: "cannot create jump action."); } dev_flow->dv.actions[actions_n++] = - dev_flow->dv.jump->action; + handle->dvh.jump->action; action_flags |= MLX5_FLOW_ACTION_JUMP; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: - if (flow_dv_convert_action_modify_mac(&res, actions, - error)) + if (flow_dv_convert_action_modify_mac + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? @@ -5824,8 +7643,8 @@ cnt_err: break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: - if (flow_dv_convert_action_modify_ipv4(&res, actions, - error)) + if (flow_dv_convert_action_modify_ipv4 + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? @@ -5834,8 +7653,8 @@ cnt_err: break; case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: - if (flow_dv_convert_action_modify_ipv6(&res, actions, - error)) + if (flow_dv_convert_action_modify_ipv6 + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? @@ -5844,9 +7663,10 @@ cnt_err: break; case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: case RTE_FLOW_ACTION_TYPE_SET_TP_DST: - if (flow_dv_convert_action_modify_tp(&res, actions, - items, &flow_attr, - error)) + if (flow_dv_convert_action_modify_tp + (mhdr_res, actions, items, + &flow_attr, dev_flow, !!(action_flags & + MLX5_FLOW_ACTION_DECAP), error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? @@ -5854,23 +7674,25 @@ cnt_err: MLX5_FLOW_ACTION_SET_TP_DST; break; case RTE_FLOW_ACTION_TYPE_DEC_TTL: - if (flow_dv_convert_action_modify_dec_ttl(&res, items, - &flow_attr, - error)) + if (flow_dv_convert_action_modify_dec_ttl + (mhdr_res, items, &flow_attr, dev_flow, + !!(action_flags & + MLX5_FLOW_ACTION_DECAP), error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_DEC_TTL; break; case RTE_FLOW_ACTION_TYPE_SET_TTL: - if (flow_dv_convert_action_modify_ttl(&res, actions, - items, &flow_attr, - error)) + if (flow_dv_convert_action_modify_ttl + (mhdr_res, actions, items, &flow_attr, + dev_flow, !!(action_flags & + MLX5_FLOW_ACTION_DECAP), error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_SET_TTL; break; case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: - if (flow_dv_convert_action_modify_tcp_seq(&res, actions, - error)) + if (flow_dv_convert_action_modify_tcp_seq + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? @@ -5880,42 +7702,77 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: - if (flow_dv_convert_action_modify_tcp_ack(&res, actions, - error)) + if (flow_dv_convert_action_modify_tcp_ack + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? MLX5_FLOW_ACTION_INC_TCP_ACK : MLX5_FLOW_ACTION_DEC_TCP_ACK; break; - case MLX5_RTE_FLOW_ACTION_TYPE_TAG: - if (flow_dv_convert_action_set_reg(&res, actions, - error)) + case MLX5_RTE_FLOW_ACTION_TYPE_TAG: + if (flow_dv_convert_action_set_reg + (mhdr_res, actions, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_SET_TAG; + break; + case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: + if (flow_dv_convert_action_copy_mreg + (dev, mhdr_res, actions, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_SET_TAG; + break; + case RTE_FLOW_ACTION_TYPE_METER: + mtr = actions->conf; + if (!flow->meter) { + flow->meter = mlx5_flow_meter_attach(priv, + mtr->mtr_id, attr, + error); + if (!flow->meter) + return rte_flow_error_set(error, + rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "meter not found " + "or invalid parameters"); + } + /* Set the meter action. */ + dev_flow->dv.actions[actions_n++] = + flow->meter->mfts->meter_action; + action_flags |= MLX5_FLOW_ACTION_METER; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: + if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res, + actions, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: + if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res, + actions, error)) return -rte_errno; - action_flags |= MLX5_FLOW_ACTION_SET_TAG; + action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; break; case RTE_FLOW_ACTION_TYPE_END: actions_end = true; - if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) { + if (mhdr_res->actions_num) { /* create modify action if needed. */ if (flow_dv_modify_hdr_resource_register - (dev, &res, - dev_flow, - error)) + (dev, mhdr_res, dev_flow, error)) return -rte_errno; dev_flow->dv.actions[modify_action_position] = - dev_flow->dv.modify_hdr->verbs_action; + handle->dvh.modify_hdr->verbs_action; } break; default: break; } - if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) && + if (mhdr_res->actions_num && modify_action_position == UINT32_MAX) modify_action_position = actions_n++; } dev_flow->dv.actions_n = actions_n; - dev_flow->actions = action_flags; + handle->act_flags = action_flags; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); int item_type = items->type; @@ -5947,13 +7804,9 @@ cnt_err: mlx5_flow_tunnel_ip_check(items, next_protocol, &item_flags, &tunnel); flow_dv_translate_item_ipv4(match_mask, match_value, - items, tunnel, flow->group); + items, item_flags, tunnel, + dev_flow->dv.group); matcher.priority = MLX5_PRIORITY_MAP_L3; - dev_flow->dv.hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, - MLX5_IPV4_LAYER_TYPES, - MLX5_IPV4_IBV_RX_HASH); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : MLX5_FLOW_LAYER_OUTER_L3_IPV4; if (items->mask != NULL && @@ -5974,13 +7827,9 @@ cnt_err: mlx5_flow_tunnel_ip_check(items, next_protocol, &item_flags, &tunnel); flow_dv_translate_item_ipv6(match_mask, match_value, - items, tunnel, flow->group); + items, item_flags, tunnel, + dev_flow->dv.group); matcher.priority = MLX5_PRIORITY_MAP_L3; - dev_flow->dv.hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, - MLX5_IPV6_LAYER_TYPES, - MLX5_IPV6_IBV_RX_HASH); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : MLX5_FLOW_LAYER_OUTER_L3_IPV6; if (items->mask != NULL && @@ -6001,11 +7850,6 @@ cnt_err: flow_dv_translate_item_tcp(match_mask, match_value, items, tunnel); matcher.priority = MLX5_PRIORITY_MAP_L4; - dev_flow->dv.hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, ETH_RSS_TCP, - IBV_RX_HASH_SRC_PORT_TCP | - IBV_RX_HASH_DST_PORT_TCP); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : MLX5_FLOW_LAYER_OUTER_L4_TCP; break; @@ -6013,17 +7857,14 @@ cnt_err: flow_dv_translate_item_udp(match_mask, match_value, items, tunnel); matcher.priority = MLX5_PRIORITY_MAP_L4; - dev_flow->dv.hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, ETH_RSS_UDP, - IBV_RX_HASH_SRC_PORT_UDP | - IBV_RX_HASH_DST_PORT_UDP); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : MLX5_FLOW_LAYER_OUTER_L4_UDP; break; case RTE_FLOW_ITEM_TYPE_GRE: flow_dv_translate_item_gre(match_mask, match_value, items, tunnel); + matcher.priority = flow->rss.level >= 2 ? + MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_GRE; break; case RTE_FLOW_ITEM_TYPE_GRE_KEY: @@ -6034,31 +7875,47 @@ cnt_err: case RTE_FLOW_ITEM_TYPE_NVGRE: flow_dv_translate_item_nvgre(match_mask, match_value, items, tunnel); + matcher.priority = flow->rss.level >= 2 ? + MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_GRE; break; case RTE_FLOW_ITEM_TYPE_VXLAN: flow_dv_translate_item_vxlan(match_mask, match_value, items, tunnel); + matcher.priority = flow->rss.level >= 2 ? + MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_VXLAN; break; case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: - flow_dv_translate_item_vxlan(match_mask, match_value, - items, tunnel); + flow_dv_translate_item_vxlan_gpe(match_mask, + match_value, items, + tunnel); + matcher.priority = flow->rss.level >= 2 ? + MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_VXLAN_GPE; break; case RTE_FLOW_ITEM_TYPE_GENEVE: flow_dv_translate_item_geneve(match_mask, match_value, items, tunnel); + matcher.priority = flow->rss.level >= 2 ? + MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_GENEVE; break; case RTE_FLOW_ITEM_TYPE_MPLS: flow_dv_translate_item_mpls(match_mask, match_value, items, last_item, tunnel); + matcher.priority = flow->rss.level >= 2 ? + MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; last_item = MLX5_FLOW_LAYER_MPLS; break; + case RTE_FLOW_ITEM_TYPE_MARK: + flow_dv_translate_item_mark(dev, match_mask, + match_value, items); + last_item = MLX5_FLOW_ITEM_MARK; + break; case RTE_FLOW_ITEM_TYPE_META: - flow_dv_translate_item_meta(match_mask, match_value, - items); + flow_dv_translate_item_meta(dev, match_mask, + match_value, attr, items); last_item = MLX5_FLOW_ITEM_METADATA; break; case RTE_FLOW_ITEM_TYPE_ICMP: @@ -6071,9 +7928,14 @@ cnt_err: items, tunnel); last_item = MLX5_FLOW_LAYER_ICMP6; break; + case RTE_FLOW_ITEM_TYPE_TAG: + flow_dv_translate_item_tag(dev, match_mask, + match_value, items); + last_item = MLX5_FLOW_ITEM_TAG; + break; case MLX5_RTE_FLOW_ITEM_TYPE_TAG: - flow_dv_translate_item_tag(match_mask, match_value, - items); + flow_dv_translate_mlx5_item_tag(dev, match_mask, + match_value, items); last_item = MLX5_FLOW_ITEM_TAG; break; case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: @@ -6082,42 +7944,59 @@ cnt_err: items); last_item = MLX5_FLOW_ITEM_TX_QUEUE; break; + case RTE_FLOW_ITEM_TYPE_GTP: + flow_dv_translate_item_gtp(match_mask, match_value, + items, tunnel); + matcher.priority = flow->rss.level >= 2 ? + MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; + last_item = MLX5_FLOW_LAYER_GTP; + break; default: break; } item_flags |= last_item; } /* - * In case of ingress traffic when E-Switch mode is enabled, - * we have two cases where we need to set the source port manually. + * When E-Switch mode is enabled, we have two cases where we need to + * set the source port manually. * The first one, is in case of Nic steering rule, and the second is * E-Switch rule where no port_id item was found. In both cases * the source port is set according the current port in use. */ - if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) && + if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && (priv->representor || priv->master)) { if (flow_dv_translate_item_port_id(dev, match_mask, match_value, NULL)) return -rte_errno; } - assert(!flow_dv_check_valid_spec(matcher.mask.buf, - dev_flow->dv.value.buf)); - dev_flow->layers = item_flags; +#ifdef RTE_LIBRTE_MLX5_DEBUG + MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf, + dev_flow->dv.value.buf)); +#endif + /* + * Layers may be already initialized from prefix flow if this dev_flow + * is the suffix flow. + */ + handle->layers |= item_flags; + if (action_flags & MLX5_FLOW_ACTION_RSS) + flow_dv_hashfields_set(dev_flow); /* Register matcher. */ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, matcher.mask.size); matcher.priority = mlx5_flow_adjust_priority(dev, priority, matcher.priority); - matcher.egress = attr->egress; - matcher.group = flow->group; - matcher.transfer = attr->transfer; - if (flow_dv_matcher_register(dev, &matcher, dev_flow, error)) + /* reserved field no needs to be set to 0 here. */ + tbl_key.domain = attr->transfer; + tbl_key.direction = attr->egress; + tbl_key.table_id = dev_flow->dv.group; + if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error)) return -rte_errno; return 0; } /** - * Apply the flow to the NIC. + * Apply the flow to the NIC, lock free, + * (mutex should be acquired by caller). * * @param[in] dev * Pointer to the Ethernet device structure. @@ -6130,24 +8009,30 @@ cnt_err: * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, - struct rte_flow_error *error) +__flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, + struct rte_flow_error *error) { - struct mlx5_flow_dv *dv; + struct mlx5_flow_dv_workspace *dv; + struct mlx5_flow_handle *dh; + struct mlx5_flow_handle_dv *dv_h; struct mlx5_flow *dev_flow; struct mlx5_priv *priv = dev->data->dev_private; int n; int err; + int idx; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { + for (idx = priv->flow_idx - 1; idx >= 0; idx--) { + dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; dv = &dev_flow->dv; + dh = dev_flow->handle; + dv_h = &dh->dvh; n = dv->actions_n; - if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) { - if (flow->transfer) { + if (dh->act_flags & MLX5_FLOW_ACTION_DROP) { + if (dv->transfer) { dv->actions[n++] = priv->sh->esw_drop_action; } else { - dv->hrxq = mlx5_hrxq_drop_new(dev); - if (!dv->hrxq) { + dh->hrxq = mlx5_hrxq_drop_new(dev); + if (!dh->hrxq) { rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -6155,23 +8040,26 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, "cannot get drop hash queue"); goto error; } - dv->actions[n++] = dv->hrxq->action; + dv->actions[n++] = dh->hrxq->action; } - } else if (dev_flow->actions & + } else if (dh->act_flags & (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) { struct mlx5_hrxq *hrxq; - hrxq = mlx5_hrxq_get(dev, flow->key, + MLX5_ASSERT(flow->rss.queue); + hrxq = mlx5_hrxq_get(dev, flow->rss.key, MLX5_RSS_HASH_KEY_LEN, - dv->hash_fields, - (*flow->queue), + dev_flow->hash_fields, + (*flow->rss.queue), flow->rss.queue_num); if (!hrxq) { hrxq = mlx5_hrxq_new - (dev, flow->key, MLX5_RSS_HASH_KEY_LEN, - dv->hash_fields, (*flow->queue), + (dev, flow->rss.key, + MLX5_RSS_HASH_KEY_LEN, + dev_flow->hash_fields, + (*flow->rss.queue), flow->rss.queue_num, - !!(dev_flow->layers & + !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL)); } if (!hrxq) { @@ -6181,14 +8069,14 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, "cannot get hash queue"); goto error; } - dv->hrxq = hrxq; - dv->actions[n++] = dv->hrxq->action; + dh->hrxq = hrxq; + dv->actions[n++] = dh->hrxq->action; } - dv->flow = - mlx5_glue->dv_create_flow(dv->matcher->matcher_object, + dh->ib_flow = + mlx5_glue->dv_create_flow(dv_h->matcher->matcher_object, (void *)&dv->value, n, dv->actions); - if (!dv->flow) { + if (!dh->ib_flow) { rte_flow_error_set(error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -6196,32 +8084,29 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, goto error; } if (priv->vmwa_context && - dev_flow->dv.vf_vlan.tag && - !dev_flow->dv.vf_vlan.created) { + dh->vf_vlan.tag && !dh->vf_vlan.created) { /* * The rule contains the VLAN pattern. * For VF we are going to create VLAN * interface to make hypervisor set correct * e-Switch vport context. */ - mlx5_vlan_vmwa_acquire(dev, &dev_flow->dv.vf_vlan); + mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan); } } return 0; error: err = rte_errno; /* Save rte_errno before cleanup. */ - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { - struct mlx5_flow_dv *dv = &dev_flow->dv; - if (dv->hrxq) { - if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) + LIST_FOREACH(dh, &flow->dev_handles, next) { + if (dh->hrxq) { + if (dh->act_flags & MLX5_FLOW_ACTION_DROP) mlx5_hrxq_drop_release(dev); else - mlx5_hrxq_release(dev, dv->hrxq); - dv->hrxq = NULL; + mlx5_hrxq_release(dev, dh->hrxq); + dh->hrxq = NULL; } - if (dev_flow->dv.vf_vlan.tag && - dev_flow->dv.vf_vlan.created) - mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan); + if (dh->vf_vlan.tag && dh->vf_vlan.created) + mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); } rte_errno = err; /* Restore rte_errno. */ return -rte_errno; @@ -6232,22 +8117,19 @@ error: * * @param dev * Pointer to Ethernet device. - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int flow_dv_matcher_release(struct rte_eth_dev *dev, - struct mlx5_flow *flow) + struct mlx5_flow_handle *handle) { - struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher; - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_ibv_shared *sh = priv->sh; - struct mlx5_flow_tbl_resource *tbl; + struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher; - assert(matcher->matcher_object); + MLX5_ASSERT(matcher->matcher_object); DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", dev->data->port_id, (void *)matcher, rte_atomic32_read(&matcher->refcnt)); @@ -6255,11 +8137,8 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, claim_zero(mlx5_glue->dv_destroy_flow_matcher (matcher->matcher_object)); LIST_REMOVE(matcher, next); - if (matcher->egress) - tbl = &sh->tx_tbl[matcher->group]; - else - tbl = &sh->rx_tbl[matcher->group]; - flow_dv_tbl_resource_release(tbl); + /* table ref-- in release interface. */ + flow_dv_tbl_resource_release(dev, matcher->tbl); rte_free(matcher); DRV_LOG(DEBUG, "port %u matcher %p: removed", dev->data->port_id, (void *)matcher); @@ -6271,19 +8150,19 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, /** * Release an encap/decap resource. * - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) +flow_dv_encap_decap_resource_release(struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_encap_decap_resource *cache_resource = - flow->dv.encap_decap; + handle->dvh.encap_decap; - assert(cache_resource->verbs_action); + MLX5_ASSERT(cache_resource->verbs_action); DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -6302,28 +8181,33 @@ flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) /** * Release an jump to table action resource. * - * @param flow - * Pointer to mlx5_flow. + * @param dev + * Pointer to Ethernet device. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow) +flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_jump_tbl_resource *cache_resource = - flow->dv.jump; + handle->dvh.jump; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(cache_resource, + struct mlx5_flow_tbl_data_entry, jump); - assert(cache_resource->action); + MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { claim_zero(mlx5_glue->destroy_flow_action (cache_resource->action)); - LIST_REMOVE(cache_resource, next); - flow_dv_tbl_resource_release(cache_resource->tbl); - rte_free(cache_resource); + /* jump action memory free is inside the table release. */ + flow_dv_tbl_resource_release(dev, &tbl_data->tbl); DRV_LOG(DEBUG, "jump table resource %p: removed", (void *)cache_resource); return 0; @@ -6334,19 +8218,19 @@ flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow) /** * Release a modify-header resource. * - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow) +flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_modify_hdr_resource *cache_resource = - flow->dv.modify_hdr; + handle->dvh.modify_hdr; - assert(cache_resource->verbs_action); + MLX5_ASSERT(cache_resource->verbs_action); DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -6365,19 +8249,19 @@ flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow) /** * Release port ID action resource. * - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_port_id_action_resource_release(struct mlx5_flow *flow) +flow_dv_port_id_action_resource_release(struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_port_id_action_resource *cache_resource = - flow->dv.port_id_action; + handle->dvh.port_id_action; - assert(cache_resource->action); + MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -6396,19 +8280,19 @@ flow_dv_port_id_action_resource_release(struct mlx5_flow *flow) /** * Release push vlan action resource. * - * @param flow - * Pointer to mlx5_flow. + * @param handle + * Pointer to mlx5_flow_handle. * * @return * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow) +flow_dv_push_vlan_action_resource_release(struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_push_vlan_action_resource *cache_resource = - flow->dv.push_vlan_res; + handle->dvh.push_vlan_res; - assert(cache_resource->action); + MLX5_ASSERT(cache_resource->action); DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -6426,6 +8310,7 @@ flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow) /** * Remove the flow from the NIC but keeps it in memory. + * Lock free, (mutex should be acquired by caller). * * @param[in] dev * Pointer to Ethernet device. @@ -6433,34 +8318,32 @@ flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow) * Pointer to flow structure. */ static void -flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) +__flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow_dv *dv; - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dh; if (!flow) return; - LIST_FOREACH(dev_flow, &flow->dev_flows, next) { - dv = &dev_flow->dv; - if (dv->flow) { - claim_zero(mlx5_glue->dv_destroy_flow(dv->flow)); - dv->flow = NULL; + LIST_FOREACH(dh, &flow->dev_handles, next) { + if (dh->ib_flow) { + claim_zero(mlx5_glue->dv_destroy_flow(dh->ib_flow)); + dh->ib_flow = NULL; } - if (dv->hrxq) { - if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) + if (dh->hrxq) { + if (dh->act_flags & MLX5_FLOW_ACTION_DROP) mlx5_hrxq_drop_release(dev); else - mlx5_hrxq_release(dev, dv->hrxq); - dv->hrxq = NULL; + mlx5_hrxq_release(dev, dh->hrxq); + dh->hrxq = NULL; } - if (dev_flow->dv.vf_vlan.tag && - dev_flow->dv.vf_vlan.created) - mlx5_vlan_vmwa_release(dev, &dev_flow->dv.vf_vlan); + if (dh->vf_vlan.tag && dh->vf_vlan.created) + mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); } } /** * Remove the flow from the NIC and the memory. + * Lock free, (mutex should be acquired by caller). * * @param[in] dev * Pointer to the Ethernet device structure. @@ -6468,37 +8351,40 @@ flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) * Pointer to flow structure. */ static void -flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) +__flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct mlx5_flow *dev_flow; + struct mlx5_flow_handle *dev_handle; if (!flow) return; - flow_dv_remove(dev, flow); + __flow_dv_remove(dev, flow); if (flow->counter) { flow_dv_counter_release(dev, flow->counter); - flow->counter = NULL; + flow->counter = 0; } - if (flow->tag_resource) { - flow_dv_tag_release(dev, flow->tag_resource); - flow->tag_resource = NULL; + if (flow->meter) { + mlx5_flow_meter_detach(flow->meter); + flow->meter = NULL; } - while (!LIST_EMPTY(&flow->dev_flows)) { - dev_flow = LIST_FIRST(&flow->dev_flows); - LIST_REMOVE(dev_flow, next); - if (dev_flow->dv.matcher) - flow_dv_matcher_release(dev, dev_flow); - if (dev_flow->dv.encap_decap) - flow_dv_encap_decap_resource_release(dev_flow); - if (dev_flow->dv.modify_hdr) - flow_dv_modify_hdr_resource_release(dev_flow); - if (dev_flow->dv.jump) - flow_dv_jump_tbl_resource_release(dev_flow); - if (dev_flow->dv.port_id_action) - flow_dv_port_id_action_resource_release(dev_flow); - if (dev_flow->dv.push_vlan_res) - flow_dv_push_vlan_action_resource_release(dev_flow); - rte_free(dev_flow); + while (!LIST_EMPTY(&flow->dev_handles)) { + dev_handle = LIST_FIRST(&flow->dev_handles); + LIST_REMOVE(dev_handle, next); + if (dev_handle->dvh.matcher) + flow_dv_matcher_release(dev, dev_handle); + if (dev_handle->dvh.encap_decap) + flow_dv_encap_decap_resource_release(dev_handle); + if (dev_handle->dvh.modify_hdr) + flow_dv_modify_hdr_resource_release(dev_handle); + if (dev_handle->dvh.jump) + flow_dv_jump_tbl_resource_release(dev, dev_handle); + if (dev_handle->dvh.port_id_action) + flow_dv_port_id_action_resource_release(dev_handle); + if (dev_handle->dvh.push_vlan_res) + flow_dv_push_vlan_action_resource_release(dev_handle); + if (dev_handle->dvh.tag_resource) + flow_dv_tag_release(dev, + dev_handle->dvh.tag_resource); + rte_free(dev_handle); } } @@ -6531,6 +8417,10 @@ flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, "counters are not supported"); if (flow->counter) { uint64_t pkts, bytes; + struct mlx5_flow_counter *cnt; + + cnt = flow_dv_counter_get_by_idx(dev, flow->counter, + NULL); int err = _flow_dv_query_count(dev, flow->counter, &pkts, &bytes); @@ -6540,11 +8430,11 @@ flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, NULL, "cannot read counters"); qc->hits_set = 1; qc->bytes_set = 1; - qc->hits = pkts - flow->counter->hits; - qc->bytes = bytes - flow->counter->bytes; + qc->hits = pkts - cnt->hits; + qc->bytes = bytes - cnt->bytes; if (qc->reset) { - flow->counter->hits = pkts; - flow->counter->bytes = bytes; + cnt->hits = pkts; + cnt->bytes = bytes; } return 0; } @@ -6586,71 +8476,571 @@ flow_dv_query(struct rte_eth_dev *dev, return ret; } +/** + * Destroy the meter table set. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] tbl + * Pointer to the meter table set. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, + struct mlx5_meter_domains_infos *tbl) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_meter_domains_infos *mtd = + (struct mlx5_meter_domains_infos *)tbl; + + if (!mtd || !priv->config.dv_flow_en) + return 0; + if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) + claim_zero(mlx5_glue->dv_destroy_flow + (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); + if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) + claim_zero(mlx5_glue->dv_destroy_flow + (mtd->egress.policer_rules[RTE_MTR_DROPPED])); + if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) + claim_zero(mlx5_glue->dv_destroy_flow + (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); + if (mtd->egress.color_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->egress.color_matcher)); + if (mtd->egress.any_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->egress.any_matcher)); + if (mtd->egress.tbl) + claim_zero(flow_dv_tbl_resource_release(dev, + mtd->egress.tbl)); + if (mtd->ingress.color_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->ingress.color_matcher)); + if (mtd->ingress.any_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->ingress.any_matcher)); + if (mtd->ingress.tbl) + claim_zero(flow_dv_tbl_resource_release(dev, + mtd->ingress.tbl)); + if (mtd->transfer.color_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->transfer.color_matcher)); + if (mtd->transfer.any_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->transfer.any_matcher)); + if (mtd->transfer.tbl) + claim_zero(flow_dv_tbl_resource_release(dev, + mtd->transfer.tbl)); + if (mtd->drop_actn) + claim_zero(mlx5_glue->destroy_flow_action(mtd->drop_actn)); + rte_free(mtd); + return 0; +} + +/* Number of meter flow actions, count and jump or count and drop. */ +#define METER_ACTIONS 2 + +/** + * Create specify domain meter table and suffix table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] mtb + * Pointer to DV meter table set. + * @param[in] egress + * Table attribute. + * @param[in] transfer + * Table attribute. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * + * @return + * 0 on success, -1 otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, + struct mlx5_meter_domains_infos *mtb, + uint8_t egress, uint8_t transfer, + uint32_t color_reg_c_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_dv_match_params mask = { + .size = sizeof(mask.buf), + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf), + }; + struct mlx5dv_flow_matcher_attr dv_attr = { + .type = IBV_FLOW_ATTR_NORMAL, + .priority = 0, + .match_criteria_enable = 0, + .match_mask = (void *)&mask, + }; + void *actions[METER_ACTIONS]; + struct mlx5_flow_tbl_resource **sfx_tbl; + struct mlx5_meter_domain_info *dtb; + struct rte_flow_error error; + int i = 0; + + if (transfer) { + sfx_tbl = &sh->fdb_mtr_sfx_tbl; + dtb = &mtb->transfer; + } else if (egress) { + sfx_tbl = &sh->tx_mtr_sfx_tbl; + dtb = &mtb->egress; + } else { + sfx_tbl = &sh->rx_mtr_sfx_tbl; + dtb = &mtb->ingress; + } + /* If the suffix table in missing, create it. */ + if (!(*sfx_tbl)) { + *sfx_tbl = flow_dv_tbl_resource_get(dev, + MLX5_FLOW_TABLE_LEVEL_SUFFIX, + egress, transfer, &error); + if (!(*sfx_tbl)) { + DRV_LOG(ERR, "Failed to create meter suffix table."); + return -1; + } + } + /* Create the meter table with METER level. */ + dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, + egress, transfer, &error); + if (!dtb->tbl) { + DRV_LOG(ERR, "Failed to create meter policer table."); + return -1; + } + /* Create matchers, Any and Color. */ + dv_attr.priority = 3; + dv_attr.match_criteria_enable = 0; + dtb->any_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx, + &dv_attr, + dtb->tbl->obj); + if (!dtb->any_matcher) { + DRV_LOG(ERR, "Failed to create meter" + " policer default matcher."); + goto error_exit; + } + dv_attr.priority = 0; + dv_attr.match_criteria_enable = + 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; + flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, + rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); + dtb->color_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx, + &dv_attr, + dtb->tbl->obj); + if (!dtb->color_matcher) { + DRV_LOG(ERR, "Failed to create meter policer color matcher."); + goto error_exit; + } + if (mtb->count_actns[RTE_MTR_DROPPED]) + actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; + actions[i++] = mtb->drop_actn; + /* Default rule: lowest priority, match any, actions: drop. */ + dtb->policer_rules[RTE_MTR_DROPPED] = + mlx5_glue->dv_create_flow(dtb->any_matcher, + (void *)&value, i, actions); + if (!dtb->policer_rules[RTE_MTR_DROPPED]) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error_exit; + } + return 0; +error_exit: + return -1; +} + +/** + * Create the needed meter and suffix tables. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] fm + * Pointer to the flow meter. + * + * @return + * Pointer to table set on success, NULL otherwise and rte_errno is set. + */ +static struct mlx5_meter_domains_infos * +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, + const struct mlx5_flow_meter *fm) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_meter_domains_infos *mtb; + int ret; + int i; + + if (!priv->mtr_en) { + rte_errno = ENOTSUP; + return NULL; + } + mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0); + if (!mtb) { + DRV_LOG(ERR, "Failed to allocate memory for meter."); + return NULL; + } + /* Create meter count actions */ + for (i = 0; i <= RTE_MTR_DROPPED; i++) { + struct mlx5_flow_counter *cnt; + if (!fm->policer_stats.cnt[i]) + continue; + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[i], NULL); + mtb->count_actns[i] = cnt->action; + } + /* Create drop action. */ + mtb->drop_actn = mlx5_glue->dr_create_flow_action_drop(); + if (!mtb->drop_actn) { + DRV_LOG(ERR, "Failed to create drop action."); + goto error_exit; + } + /* Egress meter table. */ + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to prepare egress meter table."); + goto error_exit; + } + /* Ingress meter table. */ + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to prepare ingress meter table."); + goto error_exit; + } + /* FDB meter table. */ + if (priv->config.dv_esw_en) { + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, + priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to prepare fdb meter table."); + goto error_exit; + } + } + return mtb; +error_exit: + flow_dv_destroy_mtr_tbl(dev, mtb); + return NULL; +} + +/** + * Destroy domain policer rule. + * + * @param[in] dt + * Pointer to domain table. + */ +static void +flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +{ + int i; + + for (i = 0; i < RTE_MTR_DROPPED; i++) { + if (dt->policer_rules[i]) { + claim_zero(mlx5_glue->dv_destroy_flow + (dt->policer_rules[i])); + dt->policer_rules[i] = NULL; + } + } + if (dt->jump_actn) { + claim_zero(mlx5_glue->destroy_flow_action(dt->jump_actn)); + dt->jump_actn = NULL; + } +} + +/** + * Destroy policer rules. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. + * @param[in] attr + * Pointer to flow attributes. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, + const struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) +{ + struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; + + if (!mtb) + return 0; + if (attr->egress) + flow_dv_destroy_domain_policer_rule(&mtb->egress); + if (attr->ingress) + flow_dv_destroy_domain_policer_rule(&mtb->ingress); + if (attr->transfer) + flow_dv_destroy_domain_policer_rule(&mtb->transfer); + return 0; +} + +/** + * Create specify domain meter policer rule. + * + * @param[in] fm + * Pointer to flow meter structure. + * @param[in] mtb + * Pointer to DV meter table set. + * @param[in] sfx_tb + * Pointer to suffix table. + * @param[in] mtr_reg_c + * Color match REG_C. + * + * @return + * 0 on success, -1 otherwise. + */ +static int +flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, + struct mlx5_meter_domain_info *dtb, + struct mlx5_flow_tbl_resource *sfx_tb, + uint8_t mtr_reg_c) +{ + struct mlx5_flow_dv_match_params matcher = { + .size = sizeof(matcher.buf), + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf), + }; + struct mlx5_meter_domains_infos *mtb = fm->mfts; + void *actions[METER_ACTIONS]; + int i; + + /* Create jump action. */ + if (!sfx_tb) + return -1; + if (!dtb->jump_actn) + dtb->jump_actn = + mlx5_glue->dr_create_flow_action_dest_flow_tbl + (sfx_tb->obj); + if (!dtb->jump_actn) { + DRV_LOG(ERR, "Failed to create policer jump action."); + goto error; + } + for (i = 0; i < RTE_MTR_DROPPED; i++) { + int j = 0; + + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, + rte_col_2_mlx5_col(i), UINT8_MAX); + if (mtb->count_actns[i]) + actions[j++] = mtb->count_actns[i]; + if (fm->params.action[i] == MTR_POLICER_ACTION_DROP) + actions[j++] = mtb->drop_actn; + else + actions[j++] = dtb->jump_actn; + dtb->policer_rules[i] = + mlx5_glue->dv_create_flow(dtb->color_matcher, + (void *)&value, + j, actions); + if (!dtb->policer_rules[i]) { + DRV_LOG(ERR, "Failed to create policer rule."); + goto error; + } + } + return 0; +error: + rte_errno = errno; + return -1; +} + +/** + * Create policer rules. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. + * @param[in] attr + * Pointer to flow attributes. + * + * @return + * 0 on success, -1 otherwise. + */ +static int +flow_dv_create_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_meter_domains_infos *mtb = fm->mfts; + int ret; + + if (attr->egress) { + ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, + priv->sh->tx_mtr_sfx_tbl, + priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to create egress policer."); + goto error; + } + } + if (attr->ingress) { + ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, + priv->sh->rx_mtr_sfx_tbl, + priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to create ingress policer."); + goto error; + } + } + if (attr->transfer) { + ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, + priv->sh->fdb_mtr_sfx_tbl, + priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to create transfer policer."); + goto error; + } + } + return 0; +error: + flow_dv_destroy_policer_rules(dev, fm, attr); + return -1; +} + +/** + * Query a devx counter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] cnt + * Index to the flow counter. + * @param[in] clear + * Set to clear the counter statistics. + * @param[out] pkts + * The statistics value of packets. + * @param[out] bytes + * The statistics value of bytes. + * + * @return + * 0 on success, otherwise return -1. + */ +static int +flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear, + uint64_t *pkts, uint64_t *bytes) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_counter *cnt; + uint64_t inn_pkts, inn_bytes; + int ret; + + if (!priv->config.devx) + return -1; + + ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes); + if (ret) + return -1; + cnt = flow_dv_counter_get_by_idx(dev, counter, NULL); + *pkts = inn_pkts - cnt->hits; + *bytes = inn_bytes - cnt->bytes; + if (clear) { + cnt->hits = inn_pkts; + cnt->bytes = inn_bytes; + } + return 0; +} + /* - * Mutex-protected thunk to flow_dv_translate(). + * Mutex-protected thunk to lock-free __flow_dv_translate(). */ static int -flow_d_translate(struct rte_eth_dev *dev, - struct mlx5_flow *dev_flow, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - struct rte_flow_error *error) +flow_dv_translate(struct rte_eth_dev *dev, + struct mlx5_flow *dev_flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) { int ret; - flow_d_shared_lock(dev); - ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error); - flow_d_shared_unlock(dev); + flow_dv_shared_lock(dev); + ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error); + flow_dv_shared_unlock(dev); return ret; } /* - * Mutex-protected thunk to flow_dv_apply(). + * Mutex-protected thunk to lock-free __flow_dv_apply(). */ static int -flow_d_apply(struct rte_eth_dev *dev, - struct rte_flow *flow, - struct rte_flow_error *error) +flow_dv_apply(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) { int ret; - flow_d_shared_lock(dev); - ret = flow_dv_apply(dev, flow, error); - flow_d_shared_unlock(dev); + flow_dv_shared_lock(dev); + ret = __flow_dv_apply(dev, flow, error); + flow_dv_shared_unlock(dev); return ret; } /* - * Mutex-protected thunk to flow_dv_remove(). + * Mutex-protected thunk to lock-free __flow_dv_remove(). + */ +static void +flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) +{ + flow_dv_shared_lock(dev); + __flow_dv_remove(dev, flow); + flow_dv_shared_unlock(dev); +} + +/* + * Mutex-protected thunk to lock-free __flow_dv_destroy(). */ static void -flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow) +flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) +{ + flow_dv_shared_lock(dev); + __flow_dv_destroy(dev, flow); + flow_dv_shared_unlock(dev); +} + +/* + * Mutex-protected thunk to lock-free flow_dv_counter_alloc(). + */ +static uint32_t +flow_dv_counter_allocate(struct rte_eth_dev *dev) { - flow_d_shared_lock(dev); - flow_dv_remove(dev, flow); - flow_d_shared_unlock(dev); + uint32_t cnt; + + flow_dv_shared_lock(dev); + cnt = flow_dv_counter_alloc(dev, 0, 0, 1); + flow_dv_shared_unlock(dev); + return cnt; } /* - * Mutex-protected thunk to flow_dv_destroy(). + * Mutex-protected thunk to lock-free flow_dv_counter_release(). */ static void -flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) +flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt) { - flow_d_shared_lock(dev); - flow_dv_destroy(dev, flow); - flow_d_shared_unlock(dev); + flow_dv_shared_lock(dev); + flow_dv_counter_release(dev, cnt); + flow_dv_shared_unlock(dev); } const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .validate = flow_dv_validate, .prepare = flow_dv_prepare, - .translate = flow_d_translate, - .apply = flow_d_apply, - .remove = flow_d_remove, - .destroy = flow_d_destroy, + .translate = flow_dv_translate, + .apply = flow_dv_apply, + .remove = flow_dv_remove, + .destroy = flow_dv_destroy, .query = flow_dv_query, + .create_mtr_tbls = flow_dv_create_mtr_tbl, + .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, + .create_policer_rules = flow_dv_create_policer_rules, + .destroy_policer_rules = flow_dv_destroy_policer_rules, + .counter_alloc = flow_dv_counter_allocate, + .counter_free = flow_dv_counter_free, + .counter_query = flow_dv_counter_query, }; #endif /* HAVE_IBV_FLOW_DV_SUPPORT */