X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=d4d88aa7365a67e73e66a3696298da3019830e5f;hb=83306d6c465ad2320ecc5dfa8275b1d21a6cec6e;hp=1a74d5ac2b5c61e1e27fd0db7fafa82c0a0b3854;hpb=aa1c17f1d8202a1a6e64e813103cdbe6b357b0ba;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 1a74d5ac2b..d4d88aa736 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -32,7 +32,8 @@ #include "mlx5_common_os.h" #include "mlx5_flow.h" #include "mlx5_flow_os.h" -#include "mlx5_rxtx.h" +#include "mlx5_rx.h" +#include "mlx5_tx.h" #include "rte_pmd_mlx5.h" #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) @@ -1325,16 +1326,77 @@ flow_dv_convert_action_modify_ipv6_dscp MLX5_MODIFICATION_TYPE_SET, error); } +static int +mlx5_flow_item_field_width(enum rte_flow_field_id field) +{ + switch (field) { + case RTE_FLOW_FIELD_START: + return 32; + case RTE_FLOW_FIELD_MAC_DST: + case RTE_FLOW_FIELD_MAC_SRC: + return 48; + case RTE_FLOW_FIELD_VLAN_TYPE: + return 16; + case RTE_FLOW_FIELD_VLAN_ID: + return 12; + case RTE_FLOW_FIELD_MAC_TYPE: + return 16; + case RTE_FLOW_FIELD_IPV4_DSCP: + return 6; + case RTE_FLOW_FIELD_IPV4_TTL: + return 8; + case RTE_FLOW_FIELD_IPV4_SRC: + case RTE_FLOW_FIELD_IPV4_DST: + return 32; + case RTE_FLOW_FIELD_IPV6_DSCP: + return 6; + case RTE_FLOW_FIELD_IPV6_HOPLIMIT: + return 8; + case RTE_FLOW_FIELD_IPV6_SRC: + case RTE_FLOW_FIELD_IPV6_DST: + return 128; + case RTE_FLOW_FIELD_TCP_PORT_SRC: + case RTE_FLOW_FIELD_TCP_PORT_DST: + return 16; + case RTE_FLOW_FIELD_TCP_SEQ_NUM: + case RTE_FLOW_FIELD_TCP_ACK_NUM: + return 32; + case RTE_FLOW_FIELD_TCP_FLAGS: + return 6; + case RTE_FLOW_FIELD_UDP_PORT_SRC: + case RTE_FLOW_FIELD_UDP_PORT_DST: + return 16; + case RTE_FLOW_FIELD_VXLAN_VNI: + case RTE_FLOW_FIELD_GENEVE_VNI: + return 24; + case RTE_FLOW_FIELD_GTP_TEID: + case RTE_FLOW_FIELD_TAG: + return 32; + case RTE_FLOW_FIELD_MARK: + return 24; + case RTE_FLOW_FIELD_META: + return 32; + case RTE_FLOW_FIELD_POINTER: + case RTE_FLOW_FIELD_VALUE: + return 64; + default: + MLX5_ASSERT(false); + } + return 0; +} + static void mlx5_flow_field_id_to_modify_info (const struct rte_flow_action_modify_data *data, struct field_modify_info *info, - uint32_t *mask, uint32_t *value, uint32_t width, + uint32_t *mask, uint32_t *value, + uint32_t width, uint32_t dst_width, struct rte_eth_dev *dev, const struct rte_flow_attr *attr, struct rte_flow_error *error) { uint32_t idx = 0; + uint64_t val = 0; switch (data->field) { case RTE_FLOW_FIELD_START: /* not supported yet */ @@ -1345,11 +1407,13 @@ mlx5_flow_field_id_to_modify_info if (data->offset < 32) { info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_DMAC_47_16}; - mask[idx] = 0xffffffff; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1358,10 +1422,7 @@ mlx5_flow_field_id_to_modify_info } info[idx] = (struct field_modify_info){2, 4 * idx, MLX5_MODI_OUT_DMAC_15_0}; - mask[idx] = (width) ? 0x0000ffff : 0x0; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); } else { if (data->offset < 32) info[idx++] = (struct field_modify_info){4, 0, @@ -1375,11 +1436,13 @@ mlx5_flow_field_id_to_modify_info if (data->offset < 32) { info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_SMAC_47_16}; - mask[idx] = 0xffffffff; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1388,10 +1451,7 @@ mlx5_flow_field_id_to_modify_info } info[idx] = (struct field_modify_info){2, 4 * idx, MLX5_MODI_OUT_SMAC_15_0}; - mask[idx] = (width) ? 0x0000ffff : 0x0; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); } else { if (data->offset < 32) info[idx++] = (struct field_modify_info){4, 0, @@ -1406,91 +1466,66 @@ mlx5_flow_field_id_to_modify_info case RTE_FLOW_FIELD_VLAN_ID: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_FIRST_VID}; - if (mask) { - mask[idx] = 0x00000fff; - if (width < 12) - mask[idx] = (mask[idx] << (12 - width)) & - 0x00000fff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width)); break; case RTE_FLOW_FIELD_MAC_TYPE: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_ETHERTYPE}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_IPV4_DSCP: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_IP_DSCP}; - if (mask) { - mask[idx] = 0x0000003f; - if (width < 6) - mask[idx] = (mask[idx] << (6 - width)) & - 0x0000003f; - } + if (mask) + mask[idx] = 0x3f >> (6 - width); break; case RTE_FLOW_FIELD_IPV4_TTL: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_IPV4_TTL}; - if (mask) { - mask[idx] = 0x000000ff; - if (width < 8) - mask[idx] = (mask[idx] << (8 - width)) & - 0x000000ff; - } + if (mask) + mask[idx] = 0xff >> (8 - width); break; case RTE_FLOW_FIELD_IPV4_SRC: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_SIPV4}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_IPV4_DST: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_DIPV4}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_IPV6_DSCP: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_IP_DSCP}; - if (mask) { - mask[idx] = 0x0000003f; - if (width < 6) - mask[idx] = (mask[idx] << (6 - width)) & - 0x0000003f; - } + if (mask) + mask[idx] = 0x3f >> (6 - width); break; case RTE_FLOW_FIELD_IPV6_HOPLIMIT: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_IPV6_HOPLIMIT}; - if (mask) { - mask[idx] = 0x000000ff; - if (width < 8) - mask[idx] = (mask[idx] << (8 - width)) & - 0x000000ff; - } + if (mask) + mask[idx] = 0xff >> (8 - width); break; case RTE_FLOW_FIELD_IPV6_SRC: if (mask) { if (data->offset < 32) { - info[idx] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_127_96}; - mask[idx] = 0xffffffff; + info[idx] = (struct field_modify_info){4, + 4 * idx, + MLX5_MODI_OUT_SIPV6_31_0}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1500,12 +1535,14 @@ mlx5_flow_field_id_to_modify_info if (data->offset < 64) { info[idx] = (struct field_modify_info){4, 4 * idx, - MLX5_MODI_OUT_SIPV6_95_64}; - mask[idx] = 0xffffffff; + MLX5_MODI_OUT_SIPV6_63_32}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1514,49 +1551,53 @@ mlx5_flow_field_id_to_modify_info } if (data->offset < 96) { info[idx] = (struct field_modify_info){4, - 8 * idx, - MLX5_MODI_OUT_SIPV6_63_32}; - mask[idx] = 0xffffffff; + 4 * idx, + MLX5_MODI_OUT_SIPV6_95_64}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) break; ++idx; } - info[idx] = (struct field_modify_info){4, 12 * idx, - MLX5_MODI_OUT_SIPV6_31_0}; - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); + info[idx] = (struct field_modify_info){4, 4 * idx, + MLX5_MODI_OUT_SIPV6_127_96}; + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); } else { if (data->offset < 32) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_127_96}; + MLX5_MODI_OUT_SIPV6_31_0}; if (data->offset < 64) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_95_64}; + MLX5_MODI_OUT_SIPV6_63_32}; if (data->offset < 96) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_63_32}; + MLX5_MODI_OUT_SIPV6_95_64}; if (data->offset < 128) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_31_0}; + MLX5_MODI_OUT_SIPV6_127_96}; } break; case RTE_FLOW_FIELD_IPV6_DST: if (mask) { if (data->offset < 32) { - info[idx] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_127_96}; - mask[idx] = 0xffffffff; + info[idx] = (struct field_modify_info){4, + 4 * idx, + MLX5_MODI_OUT_DIPV6_31_0}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1566,12 +1607,14 @@ mlx5_flow_field_id_to_modify_info if (data->offset < 64) { info[idx] = (struct field_modify_info){4, 4 * idx, - MLX5_MODI_OUT_DIPV6_95_64}; - mask[idx] = 0xffffffff; + MLX5_MODI_OUT_DIPV6_63_32}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1580,106 +1623,83 @@ mlx5_flow_field_id_to_modify_info } if (data->offset < 96) { info[idx] = (struct field_modify_info){4, - 8 * idx, - MLX5_MODI_OUT_DIPV6_63_32}; - mask[idx] = 0xffffffff; + 4 * idx, + MLX5_MODI_OUT_DIPV6_95_64}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) break; ++idx; } - info[idx] = (struct field_modify_info){4, 12 * idx, - MLX5_MODI_OUT_DIPV6_31_0}; - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); + info[idx] = (struct field_modify_info){4, 4 * idx, + MLX5_MODI_OUT_DIPV6_127_96}; + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); } else { if (data->offset < 32) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_127_96}; + MLX5_MODI_OUT_DIPV6_31_0}; if (data->offset < 64) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_95_64}; + MLX5_MODI_OUT_DIPV6_63_32}; if (data->offset < 96) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_63_32}; + MLX5_MODI_OUT_DIPV6_95_64}; if (data->offset < 128) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_31_0}; + MLX5_MODI_OUT_DIPV6_127_96}; } break; case RTE_FLOW_FIELD_TCP_PORT_SRC: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_TCP_SPORT}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_TCP_PORT_DST: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_TCP_DPORT}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_TCP_SEQ_NUM: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_TCP_SEQ_NUM}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = (mask[idx] << (32 - width)); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_TCP_ACK_NUM: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_TCP_ACK_NUM}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = (mask[idx] << (32 - width)); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_TCP_FLAGS: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_TCP_FLAGS}; - if (mask) { - mask[idx] = 0x0000003f; - if (width < 6) - mask[idx] = (mask[idx] << (6 - width)) & - 0x0000003f; - } + if (mask) + mask[idx] = 0x3f >> (6 - width); break; case RTE_FLOW_FIELD_UDP_PORT_SRC: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_UDP_SPORT}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_UDP_PORT_DST: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_UDP_DPORT}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_VXLAN_VNI: /* not supported yet */ @@ -1690,11 +1710,9 @@ mlx5_flow_field_id_to_modify_info case RTE_FLOW_FIELD_GTP_TEID: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_GTP_TEID}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_TAG: { @@ -1706,11 +1724,10 @@ mlx5_flow_field_id_to_modify_info MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); info[idx] = (struct field_modify_info){4, 0, reg_to_field[reg]}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); } break; case RTE_FLOW_FIELD_MARK: @@ -1723,11 +1740,10 @@ mlx5_flow_field_id_to_modify_info MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); info[idx] = (struct field_modify_info){4, 0, reg_to_field[reg]}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); } break; case RTE_FLOW_FIELD_META: @@ -1739,28 +1755,33 @@ mlx5_flow_field_id_to_modify_info MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); info[idx] = (struct field_modify_info){4, 0, reg_to_field[reg]}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); } break; case RTE_FLOW_FIELD_POINTER: - for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) { - if (mask[idx]) { - memcpy(&value[idx], - (void *)(uintptr_t)data->value, 32); - value[idx] = RTE_BE32(value[idx]); - break; - } - } - break; case RTE_FLOW_FIELD_VALUE: + if (data->field == RTE_FLOW_FIELD_POINTER) + memcpy(&val, (void *)(uintptr_t)data->value, + sizeof(uint64_t)); + else + val = data->value; for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) { if (mask[idx]) { - value[idx] = RTE_BE32((uint32_t)data->value); - break; + if (dst_width > 16) { + value[idx] = rte_cpu_to_be_32(val); + val >>= 32; + } else if (dst_width > 8) { + value[idx] = rte_cpu_to_be_16(val); + val >>= 16; + } else { + value[idx] = (uint8_t)val; + val >>= 8; + } + if (!val) + break; } } break; @@ -1805,25 +1826,26 @@ flow_dv_convert_action_modify_field uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; uint32_t type; + uint32_t dst_width = mlx5_flow_item_field_width(conf->dst.field); if (conf->src.field == RTE_FLOW_FIELD_POINTER || conf->src.field == RTE_FLOW_FIELD_VALUE) { type = MLX5_MODIFICATION_TYPE_SET; /** For SET fill the destination field (field) first. */ mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask, - value, conf->width, dev, attr, error); + value, conf->width, dst_width, dev, attr, error); /** Then copy immediate value from source as per mask. */ mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask, - value, conf->width, dev, attr, error); + value, conf->width, dst_width, dev, attr, error); item.spec = &value; } else { type = MLX5_MODIFICATION_TYPE_COPY; /** For COPY fill the destination field (dcopy) without mask. */ mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL, - value, conf->width, dev, attr, error); + value, conf->width, dst_width, dev, attr, error); /** Then construct the source field (field) with mask. */ mlx5_flow_field_id_to_modify_info(&conf->src, field, mask, - value, conf->width, dev, attr, error); + value, conf->width, dst_width, dev, attr, error); } item.mask = &mask; return flow_dv_convert_modify_action(&item, @@ -3612,6 +3634,7 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list, "cannot create action"); return NULL; } + cache->idx = idx; return &cache->entry; } @@ -3703,6 +3726,7 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list, "cannot create push vlan action"); return NULL; } + cache->idx = idx; return &cache->entry; } @@ -4515,73 +4539,16 @@ flow_dv_validate_action_modify_ttl(const uint64_t action_flags, return ret; } -static int -mlx5_flow_item_field_width(enum rte_flow_field_id field) -{ - switch (field) { - case RTE_FLOW_FIELD_START: - return 32; - case RTE_FLOW_FIELD_MAC_DST: - case RTE_FLOW_FIELD_MAC_SRC: - return 48; - case RTE_FLOW_FIELD_VLAN_TYPE: - return 16; - case RTE_FLOW_FIELD_VLAN_ID: - return 12; - case RTE_FLOW_FIELD_MAC_TYPE: - return 16; - case RTE_FLOW_FIELD_IPV4_DSCP: - return 6; - case RTE_FLOW_FIELD_IPV4_TTL: - return 8; - case RTE_FLOW_FIELD_IPV4_SRC: - case RTE_FLOW_FIELD_IPV4_DST: - return 32; - case RTE_FLOW_FIELD_IPV6_DSCP: - return 6; - case RTE_FLOW_FIELD_IPV6_HOPLIMIT: - return 8; - case RTE_FLOW_FIELD_IPV6_SRC: - case RTE_FLOW_FIELD_IPV6_DST: - return 128; - case RTE_FLOW_FIELD_TCP_PORT_SRC: - case RTE_FLOW_FIELD_TCP_PORT_DST: - return 16; - case RTE_FLOW_FIELD_TCP_SEQ_NUM: - case RTE_FLOW_FIELD_TCP_ACK_NUM: - return 32; - case RTE_FLOW_FIELD_TCP_FLAGS: - return 6; - case RTE_FLOW_FIELD_UDP_PORT_SRC: - case RTE_FLOW_FIELD_UDP_PORT_DST: - return 16; - case RTE_FLOW_FIELD_VXLAN_VNI: - case RTE_FLOW_FIELD_GENEVE_VNI: - return 24; - case RTE_FLOW_FIELD_GTP_TEID: - case RTE_FLOW_FIELD_TAG: - return 32; - case RTE_FLOW_FIELD_MARK: - return 24; - case RTE_FLOW_FIELD_META: - case RTE_FLOW_FIELD_POINTER: - case RTE_FLOW_FIELD_VALUE: - return 32; - default: - MLX5_ASSERT(false); - } - return 0; -} - /** * Validate the generic modify field actions. - * + * @param[in] dev + * Pointer to the rte_eth_dev structure. * @param[in] action_flags * Holds the actions detected until now. * @param[in] action * Pointer to the modify action. - * @param[in] item_flags - * Holds the items detected. + * @param[in] attr + * Pointer to the flow attributes. * @param[out] error * Pointer to error structure. * @@ -4590,11 +4557,15 @@ mlx5_flow_item_field_width(enum rte_flow_field_id field) * a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_modify_field(const uint64_t action_flags, +flow_dv_validate_action_modify_field(struct rte_eth_dev *dev, + const uint64_t action_flags, const struct rte_flow_action *action, + const struct rte_flow_attr *attr, struct rte_flow_error *error) { int ret = 0; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; const struct rte_flow_action_modify_field *action_modify_field = action->conf; uint32_t dst_width = @@ -4606,68 +4577,103 @@ flow_dv_validate_action_modify_field(const uint64_t action_flags, if (ret) return ret; + if (action_modify_field->width == 0) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "no bits are requested to be modified"); + else if (action_modify_field->width > dst_width || + action_modify_field->width > src_width) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "cannot modify more bits than" + " the width of a field"); if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE && action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) { - if (action_modify_field->dst.offset >= dst_width || + if ((action_modify_field->dst.offset + + action_modify_field->width > dst_width) || (action_modify_field->dst.offset % 32)) return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "destination offset is too big" - " or not aligned to 4 bytes"); + RTE_FLOW_ERROR_TYPE_ACTION, action, + "destination offset is too big" + " or not aligned to 4 bytes"); if (action_modify_field->dst.level && action_modify_field->dst.field != RTE_FLOW_FIELD_TAG) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot modify inner headers"); + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "inner header fields modification" + " is not supported"); } if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE && action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) { - if (action_modify_field->src.offset >= src_width || + if (!attr->transfer && !attr->group) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "modify field action is not" + " supported for group 0"); + if ((action_modify_field->src.offset + + action_modify_field->width > src_width) || (action_modify_field->src.offset % 32)) return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "source offset is too big" - " or not aligned to 4 bytes"); + RTE_FLOW_ERROR_TYPE_ACTION, action, + "source offset is too big" + " or not aligned to 4 bytes"); if (action_modify_field->src.level && action_modify_field->src.field != RTE_FLOW_FIELD_TAG) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot copy from inner headers"); + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "inner header fields modification" + " is not supported"); } - if (action_modify_field->width == 0) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "width is required for modify action"); if (action_modify_field->dst.field == action_modify_field->src.field) return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "source and destination fields" - " cannot be the same"); + RTE_FLOW_ERROR_TYPE_ACTION, action, + "source and destination fields" + " cannot be the same"); if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE || action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER) return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "immediate value or a pointer to it" - " cannot be used as a destination"); + RTE_FLOW_ERROR_TYPE_ACTION, action, + "immediate value or a pointer to it" + " cannot be used as a destination"); if (action_modify_field->dst.field == RTE_FLOW_FIELD_START || action_modify_field->src.field == RTE_FLOW_FIELD_START) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, "modifications of an arbitrary" " place in a packet is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE || + action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "modifications of the 802.1Q Tag" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI || + action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "modifications of the VXLAN Network" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI || + action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "modifications of the GENEVE Network" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK || + action_modify_field->src.field == RTE_FLOW_FIELD_MARK || + action_modify_field->dst.field == RTE_FLOW_FIELD_META || + action_modify_field->src.field == RTE_FLOW_FIELD_META) { + if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || + !mlx5_flow_ext_mreg_supported(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "cannot modify mark or metadata without" + " extended metadata register support"); + } if (action_modify_field->operation != RTE_FLOW_MODIFY_SET) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, "add and sub operations" " are not supported"); return (action_modify_field->width / 32) + @@ -5242,6 +5248,17 @@ flow_dv_validate_action_sample(uint64_t *action_flags, return ret; ++actions_n; break; + case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: + ret = flow_dv_validate_action_l2_encap(dev, + sub_action_flags, + act, attr, + error); + if (ret < 0) + return ret; + sub_action_flags |= MLX5_FLOW_ACTION_ENCAP; + ++actions_n; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -6580,6 +6597,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, item_flags, attr, error)) return -rte_errno; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; ++actions_n; break; @@ -6591,6 +6610,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; ++actions_n; break; @@ -6599,6 +6620,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, (action_flags, actions, error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* Count PCP with push_vlan command. */ action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP; break; @@ -6608,6 +6631,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, actions, error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* Count VID with push_vlan command. */ action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; rw_act_num += MLX5_ACT_NUM_MDF_VID; @@ -6630,6 +6655,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, attr, error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_DECAP; ++actions_n; break; @@ -6657,6 +6684,9 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, actions, item_flags, error); if (ret < 0) return ret; + if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && + (action_flags & MLX5_FLOW_ACTION_DECAP)) + modify_after_mirror = 1; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: @@ -6933,16 +6963,15 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET; break; case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: - if (!attr->transfer && !attr->group) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, "modify field action " - "is not supported for group 0"); - ret = flow_dv_validate_action_modify_field(action_flags, - actions, - error); + ret = flow_dv_validate_action_modify_field(dev, + action_flags, + actions, + attr, + error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_ACTION_MODIFY_FIELD)) ++actions_n; @@ -10142,24 +10171,8 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused, return &cache_resource->entry; error: for (idx = 0; idx < resource->num_of_dest; idx++) { - struct mlx5_flow_sub_actions_idx *act_res = - &cache_resource->sample_idx[idx]; - if (act_res->rix_hrxq && - !mlx5_hrxq_release(dev, - act_res->rix_hrxq)) - act_res->rix_hrxq = 0; - if (act_res->rix_encap_decap && - !flow_dv_encap_decap_resource_release(dev, - act_res->rix_encap_decap)) - act_res->rix_encap_decap = 0; - if (act_res->rix_port_id_action && - !flow_dv_port_id_action_resource_release(dev, - act_res->rix_port_id_action)) - act_res->rix_port_id_action = 0; - if (act_res->rix_jump && - !flow_dv_jump_tbl_resource_release(dev, - act_res->rix_jump)) - act_res->rix_jump = 0; + flow_dv_sample_sub_actions_release(dev, + &cache_resource->sample_idx[idx]); if (dest_attr[idx]) mlx5_free(dest_attr[idx]); } @@ -10407,6 +10420,8 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, action_flags |= MLX5_FLOW_ACTION_PORT_ID; break; } + case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: /* Save the encap resource before sample */ pre_rix = dev_flow->handle->dvh.rix_encap_decap; @@ -10527,6 +10542,7 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, dev_flow->handle->dvh.rix_encap_decap; sample_act->dr_encap_action = dev_flow->dv.encap_decap->action; + dev_flow->handle->dvh.rix_encap_decap = 0; } if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) { normal_idx++; @@ -10534,6 +10550,7 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, dev_flow->handle->rix_port_id_action; sample_act->dr_port_id_action = dev_flow->dv.port_id_action->action; + dev_flow->handle->rix_port_id_action = 0; } if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) { normal_idx++; @@ -10980,14 +10997,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11412,33 +11427,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -11882,28 +11877,51 @@ flow_dv_translate(struct rte_eth_dev *dev, static int __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, const uint64_t hash_fields, - const int tunnel, uint32_t hrxq_idx) { - uint32_t *hrxqs = tunnel ? action->hrxq : action->hrxq_tunnel; + uint32_t *hrxqs = action->hrxq; switch (hash_fields & ~IBV_RX_HASH_INNER) { case MLX5_RSS_HASH_IPV4: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_SRC_ONLY: hrxqs[0] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV4_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY: hrxqs[1] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV4_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY: hrxqs[2] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV6: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_SRC_ONLY: hrxqs[3] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV6_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY: hrxqs[4] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV6_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY: hrxqs[5] = hrxq_idx; return 0; case MLX5_RSS_HASH_NONE: @@ -11932,33 +11950,56 @@ __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, */ static uint32_t __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx, - const uint64_t hash_fields, - const int tunnel) + const uint64_t hash_fields) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_shared_action_rss *shared_rss = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); - const uint32_t *hrxqs = tunnel ? shared_rss->hrxq : - shared_rss->hrxq_tunnel; + const uint32_t *hrxqs = shared_rss->hrxq; switch (hash_fields & ~IBV_RX_HASH_INNER) { case MLX5_RSS_HASH_IPV4: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_SRC_ONLY: return hrxqs[0]; case MLX5_RSS_HASH_IPV4_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY: return hrxqs[1]; case MLX5_RSS_HASH_IPV4_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY: return hrxqs[2]; case MLX5_RSS_HASH_IPV6: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_SRC_ONLY: return hrxqs[3]; case MLX5_RSS_HASH_IPV6_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY: return hrxqs[4]; case MLX5_RSS_HASH_IPV6_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY: return hrxqs[5]; case MLX5_RSS_HASH_NONE: return hrxqs[6]; default: return 0; } + } /** @@ -12000,11 +12041,19 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, n = dv->actions_n; if (dh->fate_action == MLX5_FLOW_FATE_DROP) { if (dv->transfer) { - dv->actions[n++] = priv->sh->esw_drop_action; + MLX5_ASSERT(priv->sh->dr_drop_action); + dv->actions[n++] = priv->sh->dr_drop_action; } else { +#ifdef HAVE_MLX5DV_DR + /* DR supports drop action placeholder. */ + MLX5_ASSERT(priv->sh->dr_drop_action); + dv->actions[n++] = priv->sh->dr_drop_action; +#else + /* For DV we use the explicit drop queue. */ MLX5_ASSERT(priv->drop_queue.hrxq); dv->actions[n++] = priv->drop_queue.hrxq->action; +#endif } } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE && !dv_h->rix_sample && !dv_h->rix_dest_array)) { @@ -12028,9 +12077,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev, rss_desc->shared_rss, - dev_flow->hash_fields, - !!(dh->layers & - MLX5_FLOW_LAYER_TUNNEL)); + dev_flow->hash_fields); if (hrxq_idx) hrxq = mlx5_ipool_get (priv->sh->ipool[MLX5_IPOOL_HRXQ], @@ -12348,7 +12395,8 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev, return; switch (handle->fate_action) { case MLX5_FLOW_FATE_QUEUE: - mlx5_hrxq_release(dev, handle->rix_hrxq); + if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array) + mlx5_hrxq_release(dev, handle->rix_hrxq); break; case MLX5_FLOW_FATE_JUMP: flow_dv_jump_tbl_resource_release(dev, handle->rix_jump); @@ -12534,6 +12582,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12544,8 +12593,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12587,6 +12634,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -12641,8 +12692,85 @@ static int __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, struct mlx5_shared_action_rss *shared_rss) { - return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq) + - __flow_dv_hrxqs_release(dev, &shared_rss->hrxq_tunnel); + return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq); +} + +/** + * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to + * user input. + * + * Only one hash value is available for one L3+L4 combination: + * for example: + * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and + * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share + * same slot in mlx5_rss_hash_fields. + * + * @param[in] rss + * Pointer to the shared action RSS conf. + * @param[in, out] hash_field + * hash_field variable needed to be adjusted. + * + * @return + * void + */ +static void +__flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss, + uint64_t *hash_field) +{ + uint64_t rss_types = rss->origin.types; + + switch (*hash_field & ~IBV_RX_HASH_INNER) { + case MLX5_RSS_HASH_IPV4: + if (rss_types & MLX5_IPV4_LAYER_TYPES) { + *hash_field &= ~MLX5_RSS_HASH_IPV4; + if (rss_types & ETH_RSS_L3_DST_ONLY) + *hash_field |= IBV_RX_HASH_DST_IPV4; + else if (rss_types & ETH_RSS_L3_SRC_ONLY) + *hash_field |= IBV_RX_HASH_SRC_IPV4; + else + *hash_field |= MLX5_RSS_HASH_IPV4; + } + return; + case MLX5_RSS_HASH_IPV6: + if (rss_types & MLX5_IPV6_LAYER_TYPES) { + *hash_field &= ~MLX5_RSS_HASH_IPV6; + if (rss_types & ETH_RSS_L3_DST_ONLY) + *hash_field |= IBV_RX_HASH_DST_IPV6; + else if (rss_types & ETH_RSS_L3_SRC_ONLY) + *hash_field |= IBV_RX_HASH_SRC_IPV6; + else + *hash_field |= MLX5_RSS_HASH_IPV6; + } + return; + case MLX5_RSS_HASH_IPV4_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP: + if (rss_types & ETH_RSS_UDP) { + *hash_field &= ~MLX5_UDP_IBV_RX_HASH; + if (rss_types & ETH_RSS_L4_DST_ONLY) + *hash_field |= IBV_RX_HASH_DST_PORT_UDP; + else if (rss_types & ETH_RSS_L4_SRC_ONLY) + *hash_field |= IBV_RX_HASH_SRC_PORT_UDP; + else + *hash_field |= MLX5_UDP_IBV_RX_HASH; + } + return; + case MLX5_RSS_HASH_IPV4_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP: + if (rss_types & ETH_RSS_TCP) { + *hash_field &= ~MLX5_TCP_IBV_RX_HASH; + if (rss_types & ETH_RSS_L4_DST_ONLY) + *hash_field |= IBV_RX_HASH_DST_PORT_TCP; + else if (rss_types & ETH_RSS_L4_SRC_ONLY) + *hash_field |= IBV_RX_HASH_SRC_PORT_TCP; + else + *hash_field |= MLX5_TCP_IBV_RX_HASH; + } + return; + default: + return; + } } /** @@ -12688,23 +12816,26 @@ __flow_dv_action_rss_setup(struct rte_eth_dev *dev, for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) { uint32_t hrxq_idx; uint64_t hash_fields = mlx5_rss_hash_fields[i]; - int tunnel; + int tunnel = 0; - for (tunnel = 0; tunnel < 2; tunnel++) { - rss_desc.tunnel = tunnel; - rss_desc.hash_fields = hash_fields; - hrxq_idx = mlx5_hrxq_get(dev, &rss_desc); - if (!hrxq_idx) { - rte_flow_error_set - (error, rte_errno, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot get hash queue"); - goto error_hrxq_new; - } - err = __flow_dv_action_rss_hrxq_set - (shared_rss, hash_fields, tunnel, hrxq_idx); - MLX5_ASSERT(!err); + __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields); + if (shared_rss->origin.level > 1) { + hash_fields |= IBV_RX_HASH_INNER; + tunnel = 1; } + rss_desc.tunnel = tunnel; + rss_desc.hash_fields = hash_fields; + hrxq_idx = mlx5_hrxq_get(dev, &rss_desc); + if (!hrxq_idx) { + rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot get hash queue"); + goto error_hrxq_new; + } + err = __flow_dv_action_rss_hrxq_set + (shared_rss, hash_fields, hrxq_idx); + MLX5_ASSERT(!err); } return 0; error_hrxq_new: @@ -12735,7 +12866,7 @@ error_hrxq_new: */ static uint32_t __flow_dv_action_rss_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action_rss *rss, struct rte_flow_error *error) { @@ -12758,7 +12889,7 @@ __flow_dv_action_rss_create(struct rte_eth_dev *dev, "cannot allocate resource memory"); goto error_rss_init; } - if (idx > (1u << MLX5_SHARED_ACTION_TYPE_OFFSET)) { + if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) { rte_flow_error_set(error, E2BIG, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "rss action number out of range"); @@ -12871,7 +13002,7 @@ __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, } /** - * Create shared action, lock free, + * Create indirect action, lock free, * (mutex should be acquired by caller). * Dispatcher for action type specific call. * @@ -12880,7 +13011,7 @@ __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, * @param[in] conf * Shared action configuration. * @param[in] action - * Action specification used to create shared action. + * Action specification used to create indirect action. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -12889,9 +13020,9 @@ __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, * A valid shared action handle in case of success, NULL otherwise and * rte_errno is set. */ -static struct rte_flow_shared_action * +static struct rte_flow_action_handle * flow_dv_action_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action *action, struct rte_flow_error *err) { @@ -12901,13 +13032,13 @@ flow_dv_action_create(struct rte_eth_dev *dev, switch (action->type) { case RTE_FLOW_ACTION_TYPE_RSS: ret = __flow_dv_action_rss_create(dev, conf, action->conf, err); - idx = (MLX5_SHARED_ACTION_TYPE_RSS << - MLX5_SHARED_ACTION_TYPE_OFFSET) | ret; + idx = (MLX5_INDIRECT_ACTION_TYPE_RSS << + MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret; break; case RTE_FLOW_ACTION_TYPE_AGE: ret = flow_dv_translate_create_aso_age(dev, action->conf, err); - idx = (MLX5_SHARED_ACTION_TYPE_AGE << - MLX5_SHARED_ACTION_TYPE_OFFSET) | ret; + idx = (MLX5_INDIRECT_ACTION_TYPE_AGE << + MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret; if (ret) { struct mlx5_aso_age_action *aso_age = flow_aso_age_get_by_idx(dev, ret); @@ -12922,19 +13053,19 @@ flow_dv_action_create(struct rte_eth_dev *dev, NULL, "action type not supported"); break; } - return ret ? (struct rte_flow_shared_action *)(uintptr_t)idx : NULL; + return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL; } /** - * Destroy the shared action. + * Destroy the indirect action. * Release action related resources on the NIC and the memory. * Lock free, (mutex should be acquired by caller). * Dispatcher for action type specific call. * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] action - * The shared action object to be removed. + * @param[in] handle + * The indirect action object handle to be removed. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -12944,25 +13075,25 @@ flow_dv_action_create(struct rte_eth_dev *dev, */ static int flow_dv_action_destroy(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, + struct rte_flow_action_handle *handle, struct rte_flow_error *error) { - uint32_t act_idx = (uint32_t)(uintptr_t)action; - uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; - uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + uint32_t act_idx = (uint32_t)(uintptr_t)handle; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); int ret; switch (type) { - case MLX5_SHARED_ACTION_TYPE_RSS: + case MLX5_INDIRECT_ACTION_TYPE_RSS: return __flow_dv_action_rss_release(dev, idx, error); - case MLX5_SHARED_ACTION_TYPE_AGE: + case MLX5_INDIRECT_ACTION_TYPE_AGE: ret = flow_dv_aso_age_release(dev, idx); if (ret) /* * In this case, the last flow has a reference will * actually release the age action. */ - DRV_LOG(DEBUG, "Shared age action %" PRIu32 " was" + DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was" " released with references %d.", idx, ret); return 0; default: @@ -13045,12 +13176,13 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx, * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] action - * The shared action object to be updated. - * @param[in] action_conf - * Action specification used to modify *action*. - * *action_conf* should be of type correlating with type of the *action*, - * otherwise considered as invalid. + * @param[in] handle + * The indirect action object handle to be updated. + * @param[in] update + * Action specification used to modify the action pointed by *handle*. + * *update* could be of same type with the action pointed by the *handle* + * handle argument, or some other structures like a wrapper, depending on + * the indirect action type. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -13060,16 +13192,18 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx, */ static int flow_dv_action_update(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - const void *action_conf, + struct rte_flow_action_handle *handle, + const void *update, struct rte_flow_error *err) { - uint32_t act_idx = (uint32_t)(uintptr_t)action; - uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; - uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + uint32_t act_idx = (uint32_t)(uintptr_t)handle; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); + const void *action_conf; switch (type) { - case MLX5_SHARED_ACTION_TYPE_RSS: + case MLX5_INDIRECT_ACTION_TYPE_RSS: + action_conf = ((const struct rte_flow_action *)update)->conf; return __flow_dv_action_rss_update(dev, idx, action_conf, err); default: return rte_flow_error_set(err, ENOTSUP, @@ -13081,17 +13215,17 @@ flow_dv_action_update(struct rte_eth_dev *dev, static int flow_dv_action_query(struct rte_eth_dev *dev, - const struct rte_flow_shared_action *action, void *data, + const struct rte_flow_action_handle *handle, void *data, struct rte_flow_error *error) { struct mlx5_age_param *age_param; struct rte_flow_query_age *resp; - uint32_t act_idx = (uint32_t)(uintptr_t)action; - uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; - uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + uint32_t act_idx = (uint32_t)(uintptr_t)handle; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); switch (type) { - case MLX5_SHARED_ACTION_TYPE_AGE: + case MLX5_INDIRECT_ACTION_TYPE_AGE: age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params; resp = data; resp->aged = __atomic_load_n(&age_param->state, @@ -13272,49 +13406,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13333,37 +13438,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13388,41 +13473,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13431,20 +13482,16 @@ error_exit: * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13455,37 +13502,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13497,24 +13528,152 @@ error_exit: return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | color_mask)); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, color_mask); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; + } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13535,7 +13694,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13544,39 +13703,56 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = + ((UINT32_C(1) << priv->max_mtr_bits) - 1) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13589,25 +13765,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->dr_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13615,7 +13818,8 @@ error: } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13628,41 +13832,107 @@ error: * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initialized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -13712,7 +13982,8 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) &actions[0]); if (ret) goto err; - actions[1] = priv->drop_queue.hrxq->action; + actions[1] = sh->dr_drop_action ? sh->dr_drop_action : + priv->drop_queue.hrxq->action; dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf); ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj, &matcher); @@ -13866,7 +14137,7 @@ flow_dv_counter_allocate(struct rte_eth_dev *dev) } /** - * Validate shared action. + * Validate indirect action. * Dispatcher for action type specific validation. * * @param[in] dev @@ -13874,7 +14145,7 @@ flow_dv_counter_allocate(struct rte_eth_dev *dev) * @param[in] conf * Shared action configuration. * @param[in] action - * The shared action object to validate. + * The indirect action object to validate. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -13884,7 +14155,7 @@ flow_dv_counter_allocate(struct rte_eth_dev *dev) */ static int flow_dv_action_validate(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action *action, struct rte_flow_error *err) { @@ -13958,7 +14229,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free,