X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=2de6b4f9e2f8374fbc9237a20b5c291269a68968;hb=f17e4b4ffef9ecf2547ad8ee628e94db8f561b2f;hp=e0874e3f5f429cabc17edbbe5134379a5fd138ee;hpb=c83456cdd7953e0659d13dcf0affc39bf76ef460;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index e0874e3f5f..2de6b4f9e2 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -21,6 +21,9 @@ #include #include #include +#include +#include +#include #include #include @@ -32,7 +35,8 @@ #include "mlx5_common_os.h" #include "mlx5_flow.h" #include "mlx5_flow_os.h" -#include "mlx5_rxtx.h" +#include "mlx5_rx.h" +#include "mlx5_tx.h" #include "rte_pmd_mlx5.h" #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) @@ -184,7 +188,7 @@ flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr, attr->valid = 1; } -/** +/* * Convert rte_mtr_color to mlx5 color. * * @param[in] rcol @@ -193,7 +197,7 @@ flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr, * @return * mlx5 color. */ -static int +static inline int rte_col_2_mlx5_col(enum rte_color rcol) { switch (rcol) { @@ -265,6 +269,31 @@ struct field_modify_info modify_tcp[] = { {0, 0, 0}, }; +static const struct rte_flow_item * +mlx5_flow_find_tunnel_item(const struct rte_flow_item *item) +{ + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + switch (item->type) { + default: + break; + case RTE_FLOW_ITEM_TYPE_VXLAN: + case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: + case RTE_FLOW_ITEM_TYPE_GRE: + case RTE_FLOW_ITEM_TYPE_MPLS: + case RTE_FLOW_ITEM_TYPE_NVGRE: + case RTE_FLOW_ITEM_TYPE_GENEVE: + return item; + case RTE_FLOW_ITEM_TYPE_IPV4: + case RTE_FLOW_ITEM_TYPE_IPV6: + if (item[1].type == RTE_FLOW_ITEM_TYPE_IPV4 || + item[1].type == RTE_FLOW_ITEM_TYPE_IPV6) + return item; + break; + } + } + return NULL; +} + static void mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, uint8_t next_protocol, uint64_t *item_flags, @@ -282,6 +311,41 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, } } +static inline struct mlx5_hlist * +flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl, + const char *name, uint32_t size, bool direct_key, + bool lcores_share, void *ctx, + mlx5_list_create_cb cb_create, + mlx5_list_match_cb cb_match, + mlx5_list_remove_cb cb_remove, + mlx5_list_clone_cb cb_clone, + mlx5_list_clone_free_cb cb_clone_free) +{ + struct mlx5_hlist *hl; + struct mlx5_hlist *expected = NULL; + char s[MLX5_NAME_SIZE]; + + hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST); + if (likely(hl)) + return hl; + snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name); + hl = mlx5_hlist_create(s, size, direct_key, lcores_share, + ctx, cb_create, cb_match, cb_remove, cb_clone, + cb_clone_free); + if (!hl) { + DRV_LOG(ERR, "%s hash creation failed", name); + rte_errno = ENOMEM; + return NULL; + } + if (!__atomic_compare_exchange_n(phl, &expected, hl, false, + __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST)) { + mlx5_hlist_destroy(hl); + hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST); + } + return hl; +} + /* Update VLAN's VID/PCP based on input rte_flow_action. * * @param[in] action @@ -385,6 +449,7 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, { uint32_t i = resource->actions_num; struct mlx5_modification_cmd *actions = resource->actions; + uint32_t carry_b = 0; /* * The item and mask are provided in big-endian format. @@ -394,10 +459,12 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, MLX5_ASSERT(item->mask); MLX5_ASSERT(field->size); do { - unsigned int size_b; - unsigned int off_b; + uint32_t size_b; + uint32_t off_b; uint32_t mask; uint32_t data; + bool next_field = true; + bool next_dcopy = true; if (i >= MLX5_MAX_MODIFY_NUM) return rte_flow_error_set(error, EINVAL, @@ -411,19 +478,17 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, continue; } /* Deduce actual data width in bits from mask value. */ - off_b = rte_bsf32(mask); + off_b = rte_bsf32(mask) + carry_b; size_b = sizeof(uint32_t) * CHAR_BIT - off_b - __builtin_clz(mask); MLX5_ASSERT(size_b); - size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b; actions[i] = (struct mlx5_modification_cmd) { .action_type = type, .field = field->id, .offset = off_b, - .length = size_b, + .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ? + 0 : size_b, }; - /* Convert entire record to expected big-endian format. */ - actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); if (type == MLX5_MODIFICATION_TYPE_COPY) { MLX5_ASSERT(dcopy); actions[i].dst_field = dcopy->id; @@ -431,7 +496,31 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, (int)dcopy->offset < 0 ? off_b : dcopy->offset; /* Convert entire record to big-endian format. */ actions[i].data1 = rte_cpu_to_be_32(actions[i].data1); - ++dcopy; + /* + * Destination field overflow. Copy leftovers of + * a source field to the next destination field. + */ + carry_b = 0; + if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) && + dcopy->size != 0) { + actions[i].length = + dcopy->size * CHAR_BIT - dcopy->offset; + carry_b = actions[i].length; + next_field = false; + } + /* + * Not enough bits in a source filed to fill a + * destination field. Switch to the next source. + */ + if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) && + (size_b == field->size * CHAR_BIT - off_b)) { + actions[i].length = + field->size * CHAR_BIT - off_b; + dcopy->offset += actions[i].length; + next_dcopy = false; + } + if (next_dcopy) + ++dcopy; } else { MLX5_ASSERT(item->spec); data = flow_dv_fetch_field((const uint8_t *)item->spec + @@ -440,8 +529,11 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, data = (data & mask) >> off_b; actions[i].data1 = rte_cpu_to_be_32(data); } + /* Convert entire record to expected big-endian format. */ + actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); + if (next_field) + ++field; ++i; - ++field; } while (field->size); if (resource->actions_num == i) return rte_flow_error_set(error, EINVAL, @@ -964,6 +1056,8 @@ flow_dv_convert_action_set_reg actions[i] = (struct mlx5_modification_cmd) { .action_type = MLX5_MODIFICATION_TYPE_SET, .field = reg_to_field[conf->id], + .offset = conf->offset, + .length = conf->length, }; actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); actions[i].data1 = rte_cpu_to_be_32(conf->data); @@ -1063,29 +1157,10 @@ flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev, if (conf->dst == REG_C_0) { /* Copy to reg_c[0], within mask only. */ reg_dst.offset = rte_bsf32(reg_c0); - /* - * Mask is ignoring the enianness, because - * there is no conversion in datapath. - */ -#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN - /* Copy from destination lower bits to reg_c[0]. */ - mask = reg_c0 >> reg_dst.offset; -#else - /* Copy from destination upper bits to reg_c[0]. */ - mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT - - rte_fls_u32(reg_c0)); -#endif + mask = rte_cpu_to_be_32(reg_c0 >> reg_dst.offset); } else { - mask = rte_cpu_to_be_32(reg_c0); -#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN - /* Copy from reg_c[0] to destination lower bits. */ reg_dst.offset = 0; -#else - /* Copy from reg_c[0] to destination upper bits. */ - reg_dst.offset = sizeof(reg_c0) * CHAR_BIT - - (rte_fls_u32(reg_c0) - - rte_bsf32(reg_c0)); -#endif + mask = rte_cpu_to_be_32(reg_c0); } } return flow_dv_convert_modify_action(&item, @@ -1209,8 +1284,8 @@ flow_dv_convert_action_set_meta const struct rte_flow_action_set_meta *conf, struct rte_flow_error *error) { - uint32_t data = conf->data; - uint32_t mask = conf->mask; + uint32_t mask = rte_cpu_to_be_32(conf->mask); + uint32_t data = rte_cpu_to_be_32(conf->data) & mask; struct rte_flow_item item = { .spec = &data, .mask = &mask, @@ -1223,25 +1298,14 @@ flow_dv_convert_action_set_meta if (reg < 0) return reg; MLX5_ASSERT(reg != REG_NON); - /* - * In datapath code there is no endianness - * coversions for perfromance reasons, all - * pattern conversions are done in rte_flow. - */ if (reg == REG_C_0) { struct mlx5_priv *priv = dev->data->dev_private; uint32_t msk_c0 = priv->sh->dv_regc0_mask; - uint32_t shl_c0; + uint32_t shl_c0 = rte_bsf32(msk_c0); - MLX5_ASSERT(msk_c0); -#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN - shl_c0 = rte_bsf32(msk_c0); -#else - shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0); -#endif - mask <<= shl_c0; - data <<= shl_c0; - MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask))); + data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0); + mask = rte_cpu_to_be_32(mask) & msk_c0; + mask = rte_cpu_to_be_32(mask << shl_c0); } reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; /* The routine expects parameters in memory as big-endian ones. */ @@ -1325,79 +1389,143 @@ flow_dv_convert_action_modify_ipv6_dscp MLX5_MODIFICATION_TYPE_SET, error); } +static int +mlx5_flow_item_field_width(struct mlx5_priv *priv, + enum rte_flow_field_id field) +{ + switch (field) { + case RTE_FLOW_FIELD_START: + return 32; + case RTE_FLOW_FIELD_MAC_DST: + case RTE_FLOW_FIELD_MAC_SRC: + return 48; + case RTE_FLOW_FIELD_VLAN_TYPE: + return 16; + case RTE_FLOW_FIELD_VLAN_ID: + return 12; + case RTE_FLOW_FIELD_MAC_TYPE: + return 16; + case RTE_FLOW_FIELD_IPV4_DSCP: + return 6; + case RTE_FLOW_FIELD_IPV4_TTL: + return 8; + case RTE_FLOW_FIELD_IPV4_SRC: + case RTE_FLOW_FIELD_IPV4_DST: + return 32; + case RTE_FLOW_FIELD_IPV6_DSCP: + return 6; + case RTE_FLOW_FIELD_IPV6_HOPLIMIT: + return 8; + case RTE_FLOW_FIELD_IPV6_SRC: + case RTE_FLOW_FIELD_IPV6_DST: + return 128; + case RTE_FLOW_FIELD_TCP_PORT_SRC: + case RTE_FLOW_FIELD_TCP_PORT_DST: + return 16; + case RTE_FLOW_FIELD_TCP_SEQ_NUM: + case RTE_FLOW_FIELD_TCP_ACK_NUM: + return 32; + case RTE_FLOW_FIELD_TCP_FLAGS: + return 9; + case RTE_FLOW_FIELD_UDP_PORT_SRC: + case RTE_FLOW_FIELD_UDP_PORT_DST: + return 16; + case RTE_FLOW_FIELD_VXLAN_VNI: + case RTE_FLOW_FIELD_GENEVE_VNI: + return 24; + case RTE_FLOW_FIELD_GTP_TEID: + case RTE_FLOW_FIELD_TAG: + return 32; + case RTE_FLOW_FIELD_MARK: + return __builtin_popcount(priv->sh->dv_mark_mask); + case RTE_FLOW_FIELD_META: + return __builtin_popcount(priv->sh->dv_meta_mask); + case RTE_FLOW_FIELD_POINTER: + case RTE_FLOW_FIELD_VALUE: + return 64; + default: + MLX5_ASSERT(false); + } + return 0; +} + static void mlx5_flow_field_id_to_modify_info (const struct rte_flow_action_modify_data *data, struct field_modify_info *info, - uint32_t *mask, uint32_t *value, uint32_t width, - struct rte_eth_dev *dev, + uint32_t *mask, uint32_t *value, + uint32_t width, uint32_t dst_width, + uint32_t *shift, struct rte_eth_dev *dev, const struct rte_flow_attr *attr, struct rte_flow_error *error) { + struct mlx5_priv *priv = dev->data->dev_private; uint32_t idx = 0; + uint32_t off = 0; + uint64_t val = 0; switch (data->field) { case RTE_FLOW_FIELD_START: /* not supported yet */ MLX5_ASSERT(false); break; case RTE_FLOW_FIELD_MAC_DST: + off = data->offset > 16 ? data->offset - 16 : 0; if (mask) { - if (data->offset < 32) { - info[idx] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DMAC_47_16}; - mask[idx] = 0xffffffff; - if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + if (data->offset < 16) { + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_DMAC_15_0}; + if (width < 16) { + mask[idx] = rte_cpu_to_be_16(0xffff >> + (16 - width)); width = 0; } else { - width -= 32; + mask[idx] = RTE_BE16(0xffff); + width -= 16; } if (!width) break; ++idx; } - info[idx] = (struct field_modify_info){2, 4 * idx, - MLX5_MODI_OUT_DMAC_15_0}; - mask[idx] = (width) ? 0x0000ffff : 0x0; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } else { - if (data->offset < 32) - info[idx++] = (struct field_modify_info){4, 0, + info[idx] = (struct field_modify_info){4, 4 * idx, MLX5_MODI_OUT_DMAC_47_16}; - info[idx] = (struct field_modify_info){2, 0, + mask[idx] = rte_cpu_to_be_32((0xffffffff >> + (32 - width)) << off); + } else { + if (data->offset < 16) + info[idx++] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_DMAC_15_0}; + info[idx] = (struct field_modify_info){4, off, + MLX5_MODI_OUT_DMAC_47_16}; } break; case RTE_FLOW_FIELD_MAC_SRC: + off = data->offset > 16 ? data->offset - 16 : 0; if (mask) { - if (data->offset < 32) { - info[idx] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SMAC_47_16}; - mask[idx] = 0xffffffff; - if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + if (data->offset < 16) { + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_SMAC_15_0}; + if (width < 16) { + mask[idx] = rte_cpu_to_be_16(0xffff >> + (16 - width)); width = 0; } else { - width -= 32; + mask[idx] = RTE_BE16(0xffff); + width -= 16; } if (!width) break; ++idx; } - info[idx] = (struct field_modify_info){2, 4 * idx, - MLX5_MODI_OUT_SMAC_15_0}; - mask[idx] = (width) ? 0x0000ffff : 0x0; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } else { - if (data->offset < 32) - info[idx++] = (struct field_modify_info){4, 0, + info[idx] = (struct field_modify_info){4, 4 * idx, MLX5_MODI_OUT_SMAC_47_16}; - info[idx] = (struct field_modify_info){2, 0, + mask[idx] = rte_cpu_to_be_32((0xffffffff >> + (32 - width)) << off); + } else { + if (data->offset < 16) + info[idx++] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_SMAC_15_0}; + info[idx] = (struct field_modify_info){4, off, + MLX5_MODI_OUT_SMAC_47_16}; } break; case RTE_FLOW_FIELD_VLAN_TYPE: @@ -1406,91 +1534,66 @@ mlx5_flow_field_id_to_modify_info case RTE_FLOW_FIELD_VLAN_ID: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_FIRST_VID}; - if (mask) { - mask[idx] = 0x00000fff; - if (width < 12) - mask[idx] = (mask[idx] << (12 - width)) & - 0x00000fff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width)); break; case RTE_FLOW_FIELD_MAC_TYPE: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_ETHERTYPE}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_IPV4_DSCP: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_IP_DSCP}; - if (mask) { - mask[idx] = 0x0000003f; - if (width < 6) - mask[idx] = (mask[idx] << (6 - width)) & - 0x0000003f; - } + if (mask) + mask[idx] = 0x3f >> (6 - width); break; case RTE_FLOW_FIELD_IPV4_TTL: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_IPV4_TTL}; - if (mask) { - mask[idx] = 0x000000ff; - if (width < 8) - mask[idx] = (mask[idx] << (8 - width)) & - 0x000000ff; - } + if (mask) + mask[idx] = 0xff >> (8 - width); break; case RTE_FLOW_FIELD_IPV4_SRC: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_SIPV4}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_IPV4_DST: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_DIPV4}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_IPV6_DSCP: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_IP_DSCP}; - if (mask) { - mask[idx] = 0x0000003f; - if (width < 6) - mask[idx] = (mask[idx] << (6 - width)) & - 0x0000003f; - } + if (mask) + mask[idx] = 0x3f >> (6 - width); break; case RTE_FLOW_FIELD_IPV6_HOPLIMIT: info[idx] = (struct field_modify_info){1, 0, MLX5_MODI_OUT_IPV6_HOPLIMIT}; - if (mask) { - mask[idx] = 0x000000ff; - if (width < 8) - mask[idx] = (mask[idx] << (8 - width)) & - 0x000000ff; - } + if (mask) + mask[idx] = 0xff >> (8 - width); break; case RTE_FLOW_FIELD_IPV6_SRC: if (mask) { if (data->offset < 32) { - info[idx] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_127_96}; - mask[idx] = 0xffffffff; + info[idx] = (struct field_modify_info){4, + 4 * idx, + MLX5_MODI_OUT_SIPV6_31_0}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1500,12 +1603,14 @@ mlx5_flow_field_id_to_modify_info if (data->offset < 64) { info[idx] = (struct field_modify_info){4, 4 * idx, - MLX5_MODI_OUT_SIPV6_95_64}; - mask[idx] = 0xffffffff; + MLX5_MODI_OUT_SIPV6_63_32}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1514,49 +1619,53 @@ mlx5_flow_field_id_to_modify_info } if (data->offset < 96) { info[idx] = (struct field_modify_info){4, - 8 * idx, - MLX5_MODI_OUT_SIPV6_63_32}; - mask[idx] = 0xffffffff; + 4 * idx, + MLX5_MODI_OUT_SIPV6_95_64}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) break; ++idx; } - info[idx] = (struct field_modify_info){4, 12 * idx, - MLX5_MODI_OUT_SIPV6_31_0}; - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); + info[idx] = (struct field_modify_info){4, 4 * idx, + MLX5_MODI_OUT_SIPV6_127_96}; + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); } else { if (data->offset < 32) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_127_96}; + MLX5_MODI_OUT_SIPV6_31_0}; if (data->offset < 64) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_95_64}; + MLX5_MODI_OUT_SIPV6_63_32}; if (data->offset < 96) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_63_32}; + MLX5_MODI_OUT_SIPV6_95_64}; if (data->offset < 128) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_SIPV6_31_0}; + MLX5_MODI_OUT_SIPV6_127_96}; } break; case RTE_FLOW_FIELD_IPV6_DST: if (mask) { if (data->offset < 32) { - info[idx] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_127_96}; - mask[idx] = 0xffffffff; + info[idx] = (struct field_modify_info){4, + 4 * idx, + MLX5_MODI_OUT_DIPV6_31_0}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1566,12 +1675,14 @@ mlx5_flow_field_id_to_modify_info if (data->offset < 64) { info[idx] = (struct field_modify_info){4, 4 * idx, - MLX5_MODI_OUT_DIPV6_95_64}; - mask[idx] = 0xffffffff; + MLX5_MODI_OUT_DIPV6_63_32}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) @@ -1580,106 +1691,83 @@ mlx5_flow_field_id_to_modify_info } if (data->offset < 96) { info[idx] = (struct field_modify_info){4, - 8 * idx, - MLX5_MODI_OUT_DIPV6_63_32}; - mask[idx] = 0xffffffff; + 4 * idx, + MLX5_MODI_OUT_DIPV6_95_64}; if (width < 32) { - mask[idx] = mask[idx] << (32 - width); + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); width = 0; } else { + mask[idx] = RTE_BE32(0xffffffff); width -= 32; } if (!width) break; ++idx; } - info[idx] = (struct field_modify_info){4, 12 * idx, - MLX5_MODI_OUT_DIPV6_31_0}; - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); + info[idx] = (struct field_modify_info){4, 4 * idx, + MLX5_MODI_OUT_DIPV6_127_96}; + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); } else { if (data->offset < 32) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_127_96}; + MLX5_MODI_OUT_DIPV6_31_0}; if (data->offset < 64) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_95_64}; + MLX5_MODI_OUT_DIPV6_63_32}; if (data->offset < 96) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_63_32}; + MLX5_MODI_OUT_DIPV6_95_64}; if (data->offset < 128) info[idx++] = (struct field_modify_info){4, 0, - MLX5_MODI_OUT_DIPV6_31_0}; + MLX5_MODI_OUT_DIPV6_127_96}; } break; case RTE_FLOW_FIELD_TCP_PORT_SRC: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_TCP_SPORT}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_TCP_PORT_DST: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_TCP_DPORT}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_TCP_SEQ_NUM: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_TCP_SEQ_NUM}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = (mask[idx] << (32 - width)); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_TCP_ACK_NUM: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_OUT_TCP_ACK_NUM}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = (mask[idx] << (32 - width)); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_TCP_FLAGS: - info[idx] = (struct field_modify_info){1, 0, - MLX5_MODI_IN_TCP_FLAGS}; - if (mask) { - mask[idx] = 0x0000003f; - if (width < 6) - mask[idx] = (mask[idx] << (6 - width)) & - 0x0000003f; - } + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_TCP_FLAGS}; + if (mask) + mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width)); break; case RTE_FLOW_FIELD_UDP_PORT_SRC: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_UDP_SPORT}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_UDP_PORT_DST: info[idx] = (struct field_modify_info){2, 0, MLX5_MODI_OUT_UDP_DPORT}; - if (mask) { - mask[idx] = 0x0000ffff; - if (width < 16) - mask[idx] = (mask[idx] << (16 - width)) & - 0x0000ffff; - } + if (mask) + mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width)); break; case RTE_FLOW_FIELD_VXLAN_VNI: /* not supported yet */ @@ -1690,11 +1778,9 @@ mlx5_flow_field_id_to_modify_info case RTE_FLOW_FIELD_GTP_TEID: info[idx] = (struct field_modify_info){4, 0, MLX5_MODI_GTP_TEID}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); break; case RTE_FLOW_FIELD_TAG: { @@ -1706,15 +1792,16 @@ mlx5_flow_field_id_to_modify_info MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); info[idx] = (struct field_modify_info){4, 0, reg_to_field[reg]}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); } break; case RTE_FLOW_FIELD_MARK: { + uint32_t mark_mask = priv->sh->dv_mark_mask; + uint32_t mark_count = __builtin_popcount(mark_mask); int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); if (reg < 0) @@ -1723,44 +1810,60 @@ mlx5_flow_field_id_to_modify_info MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); info[idx] = (struct field_modify_info){4, 0, reg_to_field[reg]}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = rte_cpu_to_be_32((mark_mask >> + (mark_count - width)) & mark_mask); } break; case RTE_FLOW_FIELD_META: { + uint32_t meta_mask = priv->sh->dv_meta_mask; + uint32_t meta_count = __builtin_popcount(meta_mask); + uint32_t msk_c0 = + rte_cpu_to_be_32(priv->sh->dv_regc0_mask); + uint32_t shl_c0 = rte_bsf32(msk_c0); int reg = flow_dv_get_metadata_reg(dev, attr, error); if (reg < 0) return; MLX5_ASSERT(reg != REG_NON); MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); + if (reg == REG_C_0) + *shift = shl_c0; info[idx] = (struct field_modify_info){4, 0, reg_to_field[reg]}; - if (mask) { - mask[idx] = 0xffffffff; - if (width < 32) - mask[idx] = mask[idx] << (32 - width); - } + if (mask) + mask[idx] = rte_cpu_to_be_32((meta_mask >> + (meta_count - width)) & meta_mask); } break; case RTE_FLOW_FIELD_POINTER: - for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) { - if (mask[idx]) { - memcpy(&value[idx], - (void *)(uintptr_t)data->value, 32); - value[idx] = RTE_BE32(value[idx]); - break; - } - } - break; case RTE_FLOW_FIELD_VALUE: + if (data->field == RTE_FLOW_FIELD_POINTER) + memcpy(&val, (void *)(uintptr_t)data->value, + sizeof(uint64_t)); + else + val = data->value; for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) { if (mask[idx]) { - value[idx] = RTE_BE32((uint32_t)data->value); - break; + if (dst_width == 48) { + /*special case for MAC addresses */ + value[idx] = rte_cpu_to_be_16(val); + val >>= 16; + dst_width -= 16; + } else if (dst_width > 16) { + value[idx] = rte_cpu_to_be_32(val); + val >>= 32; + } else if (dst_width > 8) { + value[idx] = rte_cpu_to_be_16(val); + val >>= 16; + } else { + value[idx] = (uint8_t)val; + val >>= 8; + } + if (*shift) + value[idx] <<= *shift; + if (!val) + break; } } break; @@ -1795,6 +1898,7 @@ flow_dv_convert_action_modify_field const struct rte_flow_attr *attr, struct rte_flow_error *error) { + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_modify_field *conf = (const struct rte_flow_action_modify_field *)(action->conf); struct rte_flow_item item; @@ -1805,25 +1909,31 @@ flow_dv_convert_action_modify_field uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; uint32_t type; + uint32_t shift = 0; + uint32_t dst_width = mlx5_flow_item_field_width(priv, conf->dst.field); if (conf->src.field == RTE_FLOW_FIELD_POINTER || conf->src.field == RTE_FLOW_FIELD_VALUE) { type = MLX5_MODIFICATION_TYPE_SET; /** For SET fill the destination field (field) first. */ mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask, - value, conf->width, dev, attr, error); + value, conf->width, dst_width, + &shift, dev, attr, error); /** Then copy immediate value from source as per mask. */ mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask, - value, conf->width, dev, attr, error); + value, conf->width, dst_width, + &shift, dev, attr, error); item.spec = &value; } else { type = MLX5_MODIFICATION_TYPE_COPY; /** For COPY fill the destination field (dcopy) without mask. */ mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL, - value, conf->width, dev, attr, error); + value, conf->width, dst_width, + &shift, dev, attr, error); /** Then construct the source field (field) with mask. */ mlx5_flow_field_id_to_modify_info(&conf->src, field, mask, - value, conf->width, dev, attr, error); + value, conf->width, dst_width, + &shift, dev, attr, error); } item.mask = &mask; return flow_dv_convert_modify_action(&item, @@ -1960,13 +2070,20 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, "isn't supported"); if (reg != REG_A) nic_mask.data = priv->sh->dv_meta_mask; - } else if (attr->transfer) { - return rte_flow_error_set(error, ENOTSUP, + } else { + if (attr->transfer) + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, "extended metadata feature " "should be enabled when " "meta item is requested " "with e-switch mode "); + if (attr->ingress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "match on metadata for ingress " + "is not supported in legacy " + "metadata mode"); } if (!mask) mask = &rte_flow_item_meta_mask; @@ -2351,19 +2468,19 @@ flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_item_ipv4(const struct rte_flow_item *item, - uint64_t item_flags, - uint64_t last_item, - uint16_t ether_type, - struct rte_flow_error *error) +flow_dv_validate_item_ipv4(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + uint64_t item_flags, uint64_t last_item, + uint16_t ether_type, struct rte_flow_error *error) { int ret; + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_item_ipv4 *spec = item->spec; const struct rte_flow_item_ipv4 *last = item->last; const struct rte_flow_item_ipv4 *mask = item->mask; rte_be16_t fragment_offset_spec = 0; rte_be16_t fragment_offset_last = 0; - const struct rte_flow_item_ipv4 nic_ipv4_mask = { + struct rte_flow_item_ipv4 nic_ipv4_mask = { .hdr = { .src_addr = RTE_BE32(0xffffffff), .dst_addr = RTE_BE32(0xffffffff), @@ -2374,6 +2491,17 @@ flow_dv_validate_item_ipv4(const struct rte_flow_item *item, }, }; + if (mask && (mask->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK)) { + int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); + bool ihl_cap = !tunnel ? priv->config.hca_attr.outer_ipv4_ihl : + priv->config.hca_attr.inner_ipv4_ihl; + if (!ihl_cap) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "IPV4 ihl offload not supported"); + nic_ipv4_mask.hdr.version_ihl = mask->hdr.version_ihl; + } ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item, ether_type, &nic_ipv4_mask, MLX5_ITEM_RANGE_ACCEPTED, error); @@ -2565,19 +2693,15 @@ flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item, "specified range not supported"); } -/** - * Validate the pop VLAN action. +/* + * Validate ASO CT item. * * @param[in] dev * Pointer to the rte_eth_dev structure. - * @param[in] action_flags - * Holds the actions detected until now. - * @param[in] action - * Pointer to the pop vlan action. + * @param[in] item + * Item specification. * @param[in] item_flags - * The items found in this flow rule. - * @param[in] attr - * Pointer to flow attributes. + * Pointer to bit-fields that holds the items detected until now. * @param[out] error * Pointer to error structure. * @@ -2585,28 +2709,87 @@ flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, - uint64_t action_flags, - const struct rte_flow_action *action, - uint64_t item_flags, - const struct rte_flow_attr *attr, - struct rte_flow_error *error) -{ +flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + uint64_t *item_flags, + struct rte_flow_error *error) +{ + const struct rte_flow_item_conntrack *spec = item->spec; + const struct rte_flow_item_conntrack *mask = item->mask; + RTE_SET_USED(dev); + uint32_t flags; + + if (*item_flags & MLX5_FLOW_LAYER_ASO_CT) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "Only one CT is supported"); + if (!mask) + mask = &rte_flow_item_conntrack_mask; + flags = spec->flags & mask->flags; + if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) && + ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) || + (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) || + (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "Conflict status bits"); + /* State change also needs to be considered. */ + *item_flags |= MLX5_FLOW_LAYER_ASO_CT; + return 0; +} + +/** + * Validate the pop VLAN action. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the pop vlan action. + * @param[in] item_flags + * The items found in this flow rule. + * @param[in] attr + * Pointer to flow attributes. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, + uint64_t action_flags, + const struct rte_flow_action *action, + uint64_t item_flags, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ const struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + bool direction_error = false; - (void)action; - (void)attr; if (!priv->sh->pop_vlan_action) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "pop vlan action is not supported"); - if (attr->egress) + /* Pop VLAN is not supported in egress except for CX6 FDB mode. */ + if (attr->transfer) { + bool fdb_tx = priv->representor_id != UINT16_MAX; + bool is_cx5 = sh->steering_format_version == + MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5; + + if (fdb_tx && is_cx5) + direction_error = true; + } else if (attr->egress) { + direction_error = true; + } + if (direction_error) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, - "pop vlan action not supported for " - "egress"); + "pop vlan action not supported for egress"); if (action_flags & MLX5_FLOW_VLAN_ACTIONS) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, action, @@ -2730,6 +2913,8 @@ flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev, { const struct rte_flow_action_of_push_vlan *push_vlan = action->conf; const struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + bool direction_error = false; if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) && push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ)) @@ -2741,6 +2926,22 @@ flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_ACTION, action, "wrong action order, port_id should " "be after push VLAN"); + /* Push VLAN is not supported in ingress except for CX6 FDB mode. */ + if (attr->transfer) { + bool fdb_tx = priv->representor_id != UINT16_MAX; + bool is_cx5 = sh->steering_format_version == + MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5; + + if (!fdb_tx && is_cx5) + direction_error = true; + } else if (attr->ingress) { + direction_error = true; + } + if (direction_error) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + NULL, + "push vlan action not supported for ingress"); if (!attr->transfer && priv->representor) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -3113,11 +3314,33 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, return 0; } +/** + * Check if action counter is shared by either old or new mechanism. + * + * @param[in] action + * Pointer to the action structure. + * + * @return + * True when counter is shared, false otherwise. + */ +static inline bool +is_shared_action_count(const struct rte_flow_action *action) +{ + const struct rte_flow_action_count *count = + (const struct rte_flow_action_count *)action->conf; + + if ((int)action->type == MLX5_RTE_FLOW_ACTION_TYPE_COUNT) + return true; + return !!(count && count->shared); +} + /** * Validate count action. * * @param[in] dev * Pointer to rte_eth_dev structure. + * @param[in] shared + * Indicator if action is shared. * @param[in] action_flags * Holds the actions detected until now. * @param[out] error @@ -3127,18 +3350,23 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_count(struct rte_eth_dev *dev, +flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared, uint64_t action_flags, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + if (!priv->config.devx) + goto notsup_err; if (action_flags & MLX5_FLOW_ACTION_COUNT) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "duplicate count actions set"); - if (!priv->config.devx) - goto notsup_err; + if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) && + !priv->sh->flow_hit_aso_en) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "old age and shared count combination is not supported"); #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS return 0; #endif @@ -3357,90 +3585,108 @@ flow_dv_validate_action_raw_encap_decap return 0; } -/** - * Match encap_decap resource. +/* + * Validate the ASO CT action. * - * @param list - * Pointer to the hash list. - * @param entry - * Pointer to exist resource entry object. - * @param key - * Key of the new entry. - * @param ctx_cb - * Pointer to new encap_decap resource. + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] item_flags + * The items found in this flow rule. + * @param[in] attr + * Pointer to flow attributes. + * @param[out] error + * Pointer to error structure. * * @return - * 0 on matching, none-zero otherwise. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ +static int +flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev, + uint64_t action_flags, + uint64_t item_flags, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + RTE_SET_USED(dev); + + if (attr->group == 0 && !attr->transfer) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Only support non-root table"); + if (action_flags & MLX5_FLOW_FATE_ACTIONS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "CT cannot follow a fate action"); + if ((action_flags & MLX5_FLOW_ACTION_METER) || + (action_flags & MLX5_FLOW_ACTION_AGE)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Only one ASO action is supported"); + if (action_flags & MLX5_FLOW_ACTION_ENCAP) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Encap cannot exist before CT"); + if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Not a outer TCP packet"); + return 0; +} + int -flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused, - struct mlx5_hlist_entry *entry, - uint64_t key __rte_unused, void *cb_ctx) +flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; - struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data; - struct mlx5_flow_dv_encap_decap_resource *cache_resource; - - cache_resource = container_of(entry, - struct mlx5_flow_dv_encap_decap_resource, - entry); - if (resource->reformat_type == cache_resource->reformat_type && - resource->ft_type == cache_resource->ft_type && - resource->flags == cache_resource->flags && - resource->size == cache_resource->size && + struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data; + struct mlx5_flow_dv_encap_decap_resource *resource; + + resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource, + entry); + if (resource->reformat_type == ctx_resource->reformat_type && + resource->ft_type == ctx_resource->ft_type && + resource->flags == ctx_resource->flags && + resource->size == ctx_resource->size && !memcmp((const void *)resource->buf, - (const void *)cache_resource->buf, + (const void *)ctx_resource->buf, resource->size)) return 0; return -1; } -/** - * Allocate encap_decap resource. - * - * @param list - * Pointer to the hash list. - * @param entry - * Pointer to exist resource entry object. - * @param ctx_cb - * Pointer to new encap_decap resource. - * - * @return - * 0 on matching, none-zero otherwise. - */ -struct mlx5_hlist_entry * -flow_dv_encap_decap_create_cb(struct mlx5_hlist *list, - uint64_t key __rte_unused, - void *cb_ctx) +struct mlx5_list_entry * +flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5dv_dr_domain *domain; - struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data; - struct mlx5_flow_dv_encap_decap_resource *cache_resource; + struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data; + struct mlx5_flow_dv_encap_decap_resource *resource; uint32_t idx; int ret; - if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) domain = sh->fdb_domain; - else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) + else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) domain = sh->rx_domain; else domain = sh->tx_domain; /* Register new encap/decap resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], - &idx); - if (!cache_resource) { + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx); + if (!resource) { rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); return NULL; } - *cache_resource = *resource; - cache_resource->idx = idx; - ret = mlx5_flow_os_create_flow_action_packet_reformat - (sh->ctx, domain, cache_resource, - &cache_resource->action); + *resource = *ctx_resource; + resource->idx = idx; + ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain, + resource, + &resource->action); if (ret) { mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx); rte_flow_error_set(ctx->error, ENOMEM, @@ -3449,9 +3695,41 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list, return NULL; } + return &resource->entry; +} + +struct mlx5_list_entry * +flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_encap_decap_resource *cache_resource; + uint32_t idx; + + cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], + &idx); + if (!cache_resource) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + return NULL; + } + memcpy(cache_resource, oentry, sizeof(*cache_resource)); + cache_resource->idx = idx; return &cache_resource->entry; } +void +flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_dv_encap_decap_resource *res = + container_of(entry, typeof(*res), entry); + + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx); +} + /** * Find existing encap/decap resource or create and register a new one. * @@ -3476,7 +3754,7 @@ flow_dv_encap_decap_resource_register { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_hlist_entry *entry; + struct mlx5_list_entry *entry; union { struct { uint32_t ft_type:8; @@ -3502,8 +3780,20 @@ flow_dv_encap_decap_resource_register .error = error, .data = resource, }; + struct mlx5_hlist *encaps_decaps; uint64_t key64; + encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps, + "encaps_decaps", + MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ, + true, true, sh, + flow_dv_encap_decap_create_cb, + flow_dv_encap_decap_match_cb, + flow_dv_encap_decap_remove_cb, + flow_dv_encap_decap_clone_cb, + flow_dv_encap_decap_clone_free_cb); + if (unlikely(!encaps_decaps)) + return -rte_errno; resource->flags = dev_flow->dv.group ? 0 : 1; key64 = __rte_raw_cksum(&encap_decap_key.v32, sizeof(encap_decap_key.v32), 0); @@ -3511,7 +3801,7 @@ flow_dv_encap_decap_resource_register MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 && resource->size) key64 = __rte_raw_cksum(resource->buf, resource->size, key64); - entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx); + entry = mlx5_hlist_register(encaps_decaps, key64, &ctx); if (!entry) return -rte_errno; resource = container_of(entry, typeof(*resource), entry); @@ -3553,41 +3843,39 @@ flow_dv_jump_tbl_resource_register } int -flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry, void *cb_ctx) +flow_dv_port_id_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data; struct mlx5_flow_dv_port_id_action_resource *res = - container_of(entry, typeof(*res), entry); + container_of(entry, typeof(*res), entry); return ref->port_id != res->port_id; } -struct mlx5_cache_entry * -flow_dv_port_id_create_cb(struct mlx5_cache_list *list, - struct mlx5_cache_entry *entry __rte_unused, - void *cb_ctx) +struct mlx5_list_entry * +flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data; - struct mlx5_flow_dv_port_id_action_resource *cache; + struct mlx5_flow_dv_port_id_action_resource *resource; uint32_t idx; int ret; /* Register new port id action resource. */ - cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx); - if (!cache) { + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx); + if (!resource) { rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate port_id action cache memory"); + "cannot allocate port_id action memory"); return NULL; } - *cache = *ref; + *resource = *ref; ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain, ref->port_id, - &cache->action); + &resource->action); if (ret) { mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx); rte_flow_error_set(ctx->error, ENOMEM, @@ -3595,7 +3883,40 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list, "cannot create action"); return NULL; } - return &cache->entry; + resource->idx = idx; + return &resource->entry; +} + +struct mlx5_list_entry * +flow_dv_port_id_clone_cb(void *tool_ctx, + struct mlx5_list_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_port_id_action_resource *resource; + uint32_t idx; + + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx); + if (!resource) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate port_id action memory"); + return NULL; + } + memcpy(resource, entry, sizeof(*resource)); + resource->idx = idx; + return &resource->entry; +} + +void +flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_dv_port_id_action_resource *resource = + container_of(entry, typeof(*resource), entry); + + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx); } /** @@ -3603,8 +3924,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list, * * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in, out] resource - * Pointer to port ID action resource. + * @param[in, out] ref + * Pointer to port ID action resource reference. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -3616,61 +3937,59 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list, static int flow_dv_port_id_action_resource_register (struct rte_eth_dev *dev, - struct mlx5_flow_dv_port_id_action_resource *resource, + struct mlx5_flow_dv_port_id_action_resource *ref, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_cache_entry *entry; - struct mlx5_flow_dv_port_id_action_resource *cache; + struct mlx5_list_entry *entry; + struct mlx5_flow_dv_port_id_action_resource *resource; struct mlx5_flow_cb_ctx ctx = { .error = error, - .data = resource, + .data = ref, }; - entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx); + entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx); if (!entry) return -rte_errno; - cache = container_of(entry, typeof(*cache), entry); - dev_flow->dv.port_id_action = cache; - dev_flow->handle->rix_port_id_action = cache->idx; + resource = container_of(entry, typeof(*resource), entry); + dev_flow->dv.port_id_action = resource; + dev_flow->handle->rix_port_id_action = resource->idx; return 0; } int -flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry, void *cb_ctx) +flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data; struct mlx5_flow_dv_push_vlan_action_resource *res = - container_of(entry, typeof(*res), entry); + container_of(entry, typeof(*res), entry); return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type; } -struct mlx5_cache_entry * -flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list, - struct mlx5_cache_entry *entry __rte_unused, - void *cb_ctx) +struct mlx5_list_entry * +flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data; - struct mlx5_flow_dv_push_vlan_action_resource *cache; + struct mlx5_flow_dv_push_vlan_action_resource *resource; struct mlx5dv_dr_domain *domain; uint32_t idx; int ret; /* Register new port id action resource. */ - cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx); - if (!cache) { + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx); + if (!resource) { rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate push_vlan action cache memory"); + "cannot allocate push_vlan action memory"); return NULL; } - *cache = *ref; + *resource = *ref; if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) domain = sh->fdb_domain; else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) @@ -3678,7 +3997,7 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list, else domain = sh->tx_domain; ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag, - &cache->action); + &resource->action); if (ret) { mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); rte_flow_error_set(ctx->error, ENOMEM, @@ -3686,7 +4005,40 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list, "cannot create push vlan action"); return NULL; } - return &cache->entry; + resource->idx = idx; + return &resource->entry; +} + +struct mlx5_list_entry * +flow_dv_push_vlan_clone_cb(void *tool_ctx, + struct mlx5_list_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_push_vlan_action_resource *resource; + uint32_t idx; + + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx); + if (!resource) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate push_vlan action memory"); + return NULL; + } + memcpy(resource, entry, sizeof(*resource)); + resource->idx = idx; + return &resource->entry; +} + +void +flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_dv_push_vlan_action_resource *resource = + container_of(entry, typeof(*resource), entry); + + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx); } /** @@ -3694,8 +4046,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list, * * @param [in, out] dev * Pointer to rte_eth_dev structure. - * @param[in, out] resource - * Pointer to port ID action resource. + * @param[in, out] ref + * Pointer to port ID action resource reference. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -3707,25 +4059,25 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list, static int flow_dv_push_vlan_action_resource_register (struct rte_eth_dev *dev, - struct mlx5_flow_dv_push_vlan_action_resource *resource, + struct mlx5_flow_dv_push_vlan_action_resource *ref, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_push_vlan_action_resource *cache; - struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_push_vlan_action_resource *resource; + struct mlx5_list_entry *entry; struct mlx5_flow_cb_ctx ctx = { .error = error, - .data = resource, + .data = ref, }; - entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx); + entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx); if (!entry) return -rte_errno; - cache = container_of(entry, typeof(*cache), entry); + resource = container_of(entry, typeof(*resource), entry); - dev_flow->handle->dvh.rix_push_vlan = cache->idx; - dev_flow->dv.push_vlan_res = cache; + dev_flow->handle->dvh.rix_push_vlan = resource->idx; + dev_flow->dv.push_vlan_res = resource; return 0; } @@ -4498,73 +4850,16 @@ flow_dv_validate_action_modify_ttl(const uint64_t action_flags, return ret; } -static int -mlx5_flow_item_field_width(enum rte_flow_field_id field) -{ - switch (field) { - case RTE_FLOW_FIELD_START: - return 32; - case RTE_FLOW_FIELD_MAC_DST: - case RTE_FLOW_FIELD_MAC_SRC: - return 48; - case RTE_FLOW_FIELD_VLAN_TYPE: - return 16; - case RTE_FLOW_FIELD_VLAN_ID: - return 12; - case RTE_FLOW_FIELD_MAC_TYPE: - return 16; - case RTE_FLOW_FIELD_IPV4_DSCP: - return 6; - case RTE_FLOW_FIELD_IPV4_TTL: - return 8; - case RTE_FLOW_FIELD_IPV4_SRC: - case RTE_FLOW_FIELD_IPV4_DST: - return 32; - case RTE_FLOW_FIELD_IPV6_DSCP: - return 6; - case RTE_FLOW_FIELD_IPV6_HOPLIMIT: - return 8; - case RTE_FLOW_FIELD_IPV6_SRC: - case RTE_FLOW_FIELD_IPV6_DST: - return 128; - case RTE_FLOW_FIELD_TCP_PORT_SRC: - case RTE_FLOW_FIELD_TCP_PORT_DST: - return 16; - case RTE_FLOW_FIELD_TCP_SEQ_NUM: - case RTE_FLOW_FIELD_TCP_ACK_NUM: - return 32; - case RTE_FLOW_FIELD_TCP_FLAGS: - return 6; - case RTE_FLOW_FIELD_UDP_PORT_SRC: - case RTE_FLOW_FIELD_UDP_PORT_DST: - return 16; - case RTE_FLOW_FIELD_VXLAN_VNI: - case RTE_FLOW_FIELD_GENEVE_VNI: - return 24; - case RTE_FLOW_FIELD_GTP_TEID: - case RTE_FLOW_FIELD_TAG: - return 32; - case RTE_FLOW_FIELD_MARK: - return 24; - case RTE_FLOW_FIELD_META: - case RTE_FLOW_FIELD_POINTER: - case RTE_FLOW_FIELD_VALUE: - return 32; - default: - MLX5_ASSERT(false); - } - return 0; -} - /** * Validate the generic modify field actions. - * + * @param[in] dev + * Pointer to the rte_eth_dev structure. * @param[in] action_flags * Holds the actions detected until now. * @param[in] action * Pointer to the modify action. - * @param[in] item_flags - * Holds the items detected. + * @param[in] attr + * Pointer to the flow attributes. * @param[out] error * Pointer to error structure. * @@ -4573,84 +4868,126 @@ mlx5_flow_item_field_width(enum rte_flow_field_id field) * a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_modify_field(const uint64_t action_flags, +flow_dv_validate_action_modify_field(struct rte_eth_dev *dev, + const uint64_t action_flags, const struct rte_flow_action *action, + const struct rte_flow_attr *attr, struct rte_flow_error *error) { int ret = 0; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; const struct rte_flow_action_modify_field *action_modify_field = action->conf; - uint32_t dst_width = - mlx5_flow_item_field_width(action_modify_field->dst.field); - uint32_t src_width = - mlx5_flow_item_field_width(action_modify_field->src.field); + uint32_t dst_width = mlx5_flow_item_field_width(priv, + action_modify_field->dst.field); + uint32_t src_width = mlx5_flow_item_field_width(priv, + action_modify_field->src.field); ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); if (ret) return ret; + if (action_modify_field->width == 0) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "no bits are requested to be modified"); + else if (action_modify_field->width > dst_width || + action_modify_field->width > src_width) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "cannot modify more bits than" + " the width of a field"); if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE && action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) { - if (action_modify_field->dst.offset >= dst_width || + if ((action_modify_field->dst.offset + + action_modify_field->width > dst_width) || (action_modify_field->dst.offset % 32)) return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "destination offset is too big" - " or not aligned to 4 bytes"); + RTE_FLOW_ERROR_TYPE_ACTION, action, + "destination offset is too big" + " or not aligned to 4 bytes"); if (action_modify_field->dst.level && action_modify_field->dst.field != RTE_FLOW_FIELD_TAG) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot modify inner headers"); + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "inner header fields modification" + " is not supported"); } if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE && action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) { - if (action_modify_field->src.offset >= src_width || + if (!attr->transfer && !attr->group) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "modify field action is not" + " supported for group 0"); + if ((action_modify_field->src.offset + + action_modify_field->width > src_width) || (action_modify_field->src.offset % 32)) return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "source offset is too big" - " or not aligned to 4 bytes"); + RTE_FLOW_ERROR_TYPE_ACTION, action, + "source offset is too big" + " or not aligned to 4 bytes"); if (action_modify_field->src.level && action_modify_field->src.field != RTE_FLOW_FIELD_TAG) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot copy from inner headers"); + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "inner header fields modification" + " is not supported"); } - if (action_modify_field->width == 0) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "width is required for modify action"); - if (action_modify_field->dst.field == - action_modify_field->src.field) + if ((action_modify_field->dst.field == + action_modify_field->src.field) && + (action_modify_field->dst.level == + action_modify_field->src.level)) return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "source and destination fields" - " cannot be the same"); + RTE_FLOW_ERROR_TYPE_ACTION, action, + "source and destination fields" + " cannot be the same"); if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE || - action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER) + action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER || + action_modify_field->dst.field == RTE_FLOW_FIELD_MARK) return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "immediate value or a pointer to it" - " cannot be used as a destination"); + RTE_FLOW_ERROR_TYPE_ACTION, action, + "mark, immediate value or a pointer to it" + " cannot be used as a destination"); if (action_modify_field->dst.field == RTE_FLOW_FIELD_START || action_modify_field->src.field == RTE_FLOW_FIELD_START) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, "modifications of an arbitrary" " place in a packet is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE || + action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "modifications of the 802.1Q Tag" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI || + action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "modifications of the VXLAN Network" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI || + action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "modifications of the GENEVE Network" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK || + action_modify_field->src.field == RTE_FLOW_FIELD_MARK || + action_modify_field->dst.field == RTE_FLOW_FIELD_META || + action_modify_field->src.field == RTE_FLOW_FIELD_META) { + if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || + !mlx5_flow_ext_mreg_supported(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "cannot modify mark or metadata without" + " extended metadata register support"); + } if (action_modify_field->operation != RTE_FLOW_MODIFY_SET) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, "add and sub operations" " are not supported"); return (action_modify_field->width / 32) + @@ -4696,10 +5033,6 @@ flow_dv_validate_action_jump(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can't have 2 fate actions in" " same flow"); - if (action_flags & MLX5_FLOW_ACTION_METER) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "jump with meter not support"); if (!action->conf) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, @@ -4795,21 +5128,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev, * * @param dev * Pointer to rte_eth_dev structure. - * @param flags - * Flags bits to check if root level. + * @param root + * Whether action is on root table. * * @return * Max number of modify header actions device can support. */ static inline unsigned int flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused, - uint64_t flags) + bool root) { /* * There's no way to directly query the max capacity from FW. * The maximal value on root table should be assumed to be supported. */ - if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL)) + if (!root) return MLX5_MAX_MODIFY_NUM; else return MLX5_ROOT_TBL_MODIFY_NUM; @@ -4826,6 +5159,8 @@ flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused, * Pointer to the meter action. * @param[in] attr * Attributes of flow that includes this action. + * @param[in] port_id_item + * Pointer to item indicating port id. * @param[out] error * Pointer to error structure. * @@ -4837,11 +5172,15 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, uint64_t action_flags, const struct rte_flow_action *action, const struct rte_flow_attr *attr, + const struct rte_flow_item *port_id_item, + bool *def_policy, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_meter *am = action->conf; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; + struct mlx5_flow_meter_policy *mtr_policy; + struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng; if (!am) return rte_flow_error_set(error, EINVAL, @@ -4861,19 +5200,82 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "meter action not supported"); - fm = mlx5_flow_meter_find(priv, am->mtr_id); + fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL); if (!fm) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Meter not found"); - if (fm->ref_cnt && (!(fm->transfer == attr->transfer || + /* aso meter can always be shared by different domains */ + if (fm->ref_cnt && !priv->sh->meter_aso_en && + !(fm->transfer == attr->transfer || (!fm->ingress && !attr->ingress && attr->egress) || - (!fm->egress && !attr->egress && attr->ingress)))) + (!fm->egress && !attr->egress && attr->ingress))) return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Flow attributes domain are either invalid " + "or have a domain conflict with current " + "meter attributes"); + if (fm->def_policy) { + if (!((attr->transfer && + mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) || + (attr->egress && + mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) || + (attr->ingress && + mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS]))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Flow attributes domain " + "have a conflict with current " + "meter domain attributes"); + *def_policy = true; + } else { + mtr_policy = mlx5_flow_meter_policy_find(dev, + fm->policy_id, NULL); + if (!mtr_policy) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Invalid policy id for meter "); + if (!((attr->transfer && mtr_policy->transfer) || + (attr->egress && mtr_policy->egress) || + (attr->ingress && mtr_policy->ingress))) + return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "Flow attributes are either invalid " - "or have a conflict with current " - "meter attributes"); + "Flow attributes domain " + "have a conflict with current " + "meter domain attributes"); + if (attr->transfer && mtr_policy->dev) { + /** + * When policy has fate action of port_id, + * the flow should have the same src port as policy. + */ + struct mlx5_priv *policy_port_priv = + mtr_policy->dev->data->dev_private; + int32_t flow_src_port = priv->representor_id; + + if (port_id_item) { + const struct rte_flow_item_port_id *spec = + port_id_item->spec; + struct mlx5_priv *port_priv = + mlx5_port_to_eswitch_info(spec->id, + false); + if (!port_priv) + return rte_flow_error_set(error, + rte_errno, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, + spec, + "Failed to get port info."); + flow_src_port = port_priv->representor_id; + } + if (flow_src_port != policy_port_priv->representor_id) + return rte_flow_error_set(error, + rte_errno, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, + NULL, + "Flow and meter policy " + "have different src port."); + } + *def_policy = false; + } return 0; } @@ -4990,30 +5392,14 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags, return ret; } -/** - * Match modify-header resource. - * - * @param list - * Pointer to the hash list. - * @param entry - * Pointer to exist resource entry object. - * @param key - * Key of the new entry. - * @param ctx - * Pointer to new modify-header resource. - * - * @return - * 0 on matching, non-zero otherwise. - */ int -flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused, - struct mlx5_hlist_entry *entry, - uint64_t key __rte_unused, void *cb_ctx) +flow_dv_modify_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data; struct mlx5_flow_dv_modify_hdr_resource *resource = - container_of(entry, typeof(*resource), entry); + container_of(entry, typeof(*resource), entry); uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type); key_len += ref->actions_num * sizeof(ref->actions[0]); @@ -5021,21 +5407,68 @@ flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused, memcmp(&ref->ft_type, &resource->ft_type, key_len); } -struct mlx5_hlist_entry * -flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused, - void *cb_ctx) +static struct mlx5_indexed_pool * +flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index) +{ + struct mlx5_indexed_pool *ipool = __atomic_load_n + (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST); + + if (!ipool) { + struct mlx5_indexed_pool *expected = NULL; + struct mlx5_indexed_pool_config cfg = + (struct mlx5_indexed_pool_config) { + .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) + + (index + 1) * + sizeof(struct mlx5_modification_cmd), + .trunk_size = 64, + .grow_trunk = 3, + .grow_shift = 2, + .need_lock = 1, + .release_mem_en = !!sh->reclaim_mode, + .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16), + .malloc = mlx5_malloc, + .free = mlx5_free, + .type = "mlx5_modify_action_resource", + }; + + cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool)); + ipool = mlx5_ipool_create(&cfg); + if (!ipool) + return NULL; + if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index], + &expected, ipool, false, + __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST)) { + mlx5_ipool_destroy(ipool); + ipool = __atomic_load_n(&sh->mdh_ipools[index], + __ATOMIC_SEQ_CST); + } + } + return ipool; +} + +struct mlx5_list_entry * +flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5dv_dr_domain *ns; struct mlx5_flow_dv_modify_hdr_resource *entry; struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data; + struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh, + ref->actions_num - 1); int ret; uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]); uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type); + uint32_t idx; - entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0, - SOCKET_ID_ANY); + if (unlikely(!ipool)) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot allocate modify ipool"); + return NULL; + } + entry = mlx5_ipool_zmalloc(ipool, &idx); if (!entry) { rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -5055,15 +5488,50 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused, (sh->ctx, ns, entry, data_len, &entry->action); if (ret) { - mlx5_free(entry); + mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx); rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create modification action"); return NULL; } + entry->idx = idx; + return &entry->entry; +} + +struct mlx5_list_entry * +flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_modify_hdr_resource *entry; + struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data; + uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]); + uint32_t idx; + + entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1], + &idx); + if (!entry) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + return NULL; + } + memcpy(entry, oentry, sizeof(*entry) + data_len); + entry->idx = idx; return &entry->entry; } +void +flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_dv_modify_hdr_resource *res = + container_of(entry, typeof(*res), entry); + + mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx); +} + /** * Validate the sample action. * @@ -5081,6 +5549,10 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused, * Pointer to the RSS action. * @param[out] sample_rss * Pointer to the RSS action in sample action list. + * @param[out] count + * Pointer to the COUNT action in sample action list. + * @param[out] fdb_mirror_limit + * Pointer to the FDB mirror limitation flag. * @param[out] error * Pointer to error structure. * @@ -5095,6 +5567,8 @@ flow_dv_validate_action_sample(uint64_t *action_flags, uint64_t item_flags, const struct rte_flow_action_rss *rss, const struct rte_flow_action_rss **sample_rss, + const struct rte_flow_action_count **count, + int *fdb_mirror_limit, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; @@ -5189,16 +5663,13 @@ flow_dv_validate_action_sample(uint64_t *action_flags, ++actions_n; break; case RTE_FLOW_ACTION_TYPE_COUNT: - if (*action_flags & MLX5_FLOW_ACTION_COUNT) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - action, - "duplicate count action set"); - ret = flow_dv_validate_action_count(dev, - sub_action_flags, - error); + ret = flow_dv_validate_action_count + (dev, is_shared_action_count(act), + *action_flags | sub_action_flags, + error); if (ret < 0) return ret; + *count = act->conf; sub_action_flags |= MLX5_FLOW_ACTION_COUNT; *action_flags |= MLX5_FLOW_ACTION_COUNT; ++actions_n; @@ -5222,6 +5693,17 @@ flow_dv_validate_action_sample(uint64_t *action_flags, return ret; ++actions_n; break; + case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: + ret = flow_dv_validate_action_l2_encap(dev, + sub_action_flags, + act, attr, + error); + if (ret < 0) + return ret; + sub_action_flags |= MLX5_FLOW_ACTION_ENCAP; + ++actions_n; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -5269,6 +5751,9 @@ flow_dv_validate_action_sample(uint64_t *action_flags, NULL, "E-Switch must has a dest " "port for mirroring"); + if (!priv->config.hca_attr.reg_c_preserve && + priv->representor_id != UINT16_MAX) + *fdb_mirror_limit = 1; } /* Continue validation for Xcap actions.*/ if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) && @@ -5318,22 +5803,33 @@ flow_dv_modify_hdr_resource_register uint32_t key_len = sizeof(*resource) - offsetof(typeof(*resource), ft_type) + resource->actions_num * sizeof(resource->actions[0]); - struct mlx5_hlist_entry *entry; + struct mlx5_list_entry *entry; struct mlx5_flow_cb_ctx ctx = { .error = error, .data = resource, }; + struct mlx5_hlist *modify_cmds; uint64_t key64; - resource->flags = dev_flow->dv.group ? 0 : - MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; + modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds, + "hdr_modify", + MLX5_FLOW_HDR_MODIFY_HTABLE_SZ, + true, false, sh, + flow_dv_modify_create_cb, + flow_dv_modify_match_cb, + flow_dv_modify_remove_cb, + flow_dv_modify_clone_cb, + flow_dv_modify_clone_free_cb); + if (unlikely(!modify_cmds)) + return -rte_errno; + resource->root = !dev_flow->dv.group; if (resource->actions_num > flow_dv_modify_hdr_action_max(dev, - resource->flags)) + resource->root)) return rte_flow_error_set(error, EOVERFLOW, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many modify header items"); key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0); - entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx); + entry = mlx5_hlist_register(modify_cmds, key64, &ctx); if (!entry) return -rte_errno; resource = container_of(entry, typeof(*resource), entry); @@ -5349,7 +5845,7 @@ flow_dv_modify_hdr_resource_register * @param[in] idx * mlx5 flow counter index in the container. * @param[out] ppool - * mlx5 flow counter pool in the container, + * mlx5 flow counter pool in the container. * * @return * Pointer to the counter, NULL otherwise. @@ -5479,7 +5975,7 @@ flow_dv_container_resize(struct rte_eth_dev *dev) * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] cnt + * @param[in] counter * Index to the flow counter. * @param[out] pkts * The statistics value of packets. @@ -5720,6 +6216,13 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age) if (!fallback && !priv->sh->cmng.query_thread_on) /* Start the asynchronous batch query by the host thread. */ mlx5_set_query_alarm(priv->sh); + /* + * When the count action isn't shared (by ID), shared_info field is + * used for indirect action API's refcnt. + * When the counter action is not shared neither by ID nor by indirect + * action API, shared info must be 1. + */ + cnt_free->shared_info.refcnt = 1; return cnt_idx; err: if (cnt_free) { @@ -5866,11 +6369,33 @@ flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter) return; cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); MLX5_ASSERT(pool); - if (IS_SHARED_CNT(counter) && - mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id)) - return; - if (pool->is_aged) + if (pool->is_aged) { flow_dv_counter_remove_from_age(dev, counter, cnt); + } else { + /* + * If the counter action is shared by ID, the l3t_clear_entry + * function reduces its references counter. If after the + * reduction the action is still referenced, the function + * returns here and does not release it. + */ + if (IS_LEGACY_SHARED_CNT(counter) && + mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, + cnt->shared_info.id)) + return; + /* + * If the counter action is shared by indirect action API, + * the atomic function reduces its references counter. + * If after the reduction the action is still referenced, the + * function returns here and does not release it. + * When the counter action is not shared neither by ID nor by + * indirect action API, shared info is 1 before the reduction, + * so this condition is failed and function doesn't return here. + */ + if (!IS_LEGACY_SHARED_CNT(counter) && + __atomic_sub_fetch(&cnt->shared_info.refcnt, 1, + __ATOMIC_RELAXED)) + return; + } cnt->pool = pool; /* * Put the counter back to list to be updated in none fallback mode. @@ -5880,7 +6405,6 @@ flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter) * container counter list. The list changes while query starts. In * this case, lock will not be needed as query callback and release * function both operate with the different list. - * */ if (!priv->sh->cmng.counter_fallback) { rte_spinlock_lock(&pool->csl); @@ -5897,6 +6421,186 @@ flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter) } } +/** + * Resize a meter id container. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * 0 on success, otherwise negative errno value and rte_errno is set. + */ +static int +flow_dv_mtr_container_resize(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *pools_mng = + &priv->sh->mtrmng->pools_mng; + void *old_pools = pools_mng->pools; + uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE; + uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize; + void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); + + if (!pools) { + rte_errno = ENOMEM; + return -ENOMEM; + } + if (!pools_mng->n) + if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) { + mlx5_free(pools); + return -ENOMEM; + } + if (old_pools) + memcpy(pools, old_pools, pools_mng->n * + sizeof(struct mlx5_aso_mtr_pool *)); + pools_mng->n = resize; + pools_mng->pools = pools; + if (old_pools) + mlx5_free(old_pools); + return 0; +} + +/** + * Prepare a new meter and/or a new meter pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] mtr_free + * Where to put the pointer of a new meter.g. + * + * @return + * The meter pool pointer and @mtr_free is set on success, + * NULL otherwise and rte_errno is set. + */ +static struct mlx5_aso_mtr_pool * +flow_dv_mtr_pool_create(struct rte_eth_dev *dev, + struct mlx5_aso_mtr **mtr_free) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *pools_mng = + &priv->sh->mtrmng->pools_mng; + struct mlx5_aso_mtr_pool *pool = NULL; + struct mlx5_devx_obj *dcs = NULL; + uint32_t i; + uint32_t log_obj_size; + + log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); + dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx, + priv->sh->pdn, log_obj_size); + if (!dcs) { + rte_errno = ENODATA; + return NULL; + } + pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); + if (!pool) { + rte_errno = ENOMEM; + claim_zero(mlx5_devx_cmd_destroy(dcs)); + return NULL; + } + pool->devx_obj = dcs; + pool->index = pools_mng->n_valid; + if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) { + mlx5_free(pool); + claim_zero(mlx5_devx_cmd_destroy(dcs)); + return NULL; + } + pools_mng->pools[pool->index] = pool; + pools_mng->n_valid++; + for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) { + pool->mtrs[i].offset = i; + LIST_INSERT_HEAD(&pools_mng->meters, + &pool->mtrs[i], next); + } + pool->mtrs[0].offset = 0; + *mtr_free = &pool->mtrs[0]; + return pool; +} + +/** + * Release a flow meter into pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_idx + * Index to aso flow meter. + */ +static void +flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *pools_mng = + &priv->sh->mtrmng->pools_mng; + struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); + + MLX5_ASSERT(aso_mtr); + rte_spinlock_lock(&pools_mng->mtrsl); + memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info)); + aso_mtr->state = ASO_METER_FREE; + LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next); + rte_spinlock_unlock(&pools_mng->mtrsl); +} + +/** + * Allocate a aso flow meter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * Index to aso flow meter on success, 0 otherwise and rte_errno is set. + */ +static uint32_t +flow_dv_mtr_alloc(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr *mtr_free = NULL; + struct mlx5_aso_mtr_pools_mng *pools_mng = + &priv->sh->mtrmng->pools_mng; + struct mlx5_aso_mtr_pool *pool; + uint32_t mtr_idx = 0; + + if (!priv->config.devx) { + rte_errno = ENOTSUP; + return 0; + } + /* Allocate the flow meter memory. */ + /* Get free meters from management. */ + rte_spinlock_lock(&pools_mng->mtrsl); + mtr_free = LIST_FIRST(&pools_mng->meters); + if (mtr_free) + LIST_REMOVE(mtr_free, next); + if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) { + rte_spinlock_unlock(&pools_mng->mtrsl); + return 0; + } + mtr_free->state = ASO_METER_WAIT; + rte_spinlock_unlock(&pools_mng->mtrsl); + pool = container_of(mtr_free, + struct mlx5_aso_mtr_pool, + mtrs[mtr_free->offset]); + mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset); + if (!mtr_free->fm.meter_action) { +#ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO + struct rte_flow_error error; + uint8_t reg_id; + + reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error); + mtr_free->fm.meter_action = + mlx5_glue->dv_create_flow_action_aso + (priv->sh->rx_domain, + pool->devx_obj->obj, + mtr_free->offset, + (1 << MLX5_FLOW_COLOR_GREEN), + reg_id - REG_C_0); +#endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ + if (!mtr_free->fm.meter_action) { + flow_dv_aso_mtr_release_to_pool(dev, mtr_idx); + return 0; + } + } + return mtr_idx; +} + /** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. @@ -5975,62 +6679,217 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, return ret; } -/** - * Internal validation function. For validating both actions and items. - * - * @param[in] dev - * Pointer to the rte_eth_dev structure. - * @param[in] attr - * Pointer to the flow attributes. - * @param[in] items - * Pointer to the list of items. - * @param[in] actions - * Pointer to the list of actions. - * @param[in] external - * This flow rule is created by request external to PMD. - * @param[in] hairpin - * Number of hairpin TX actions, 0 means classic flow. - * @param[out] error - * Pointer to the error structure. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -static int -flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - bool external, int hairpin, struct rte_flow_error *error) +static uint16_t +mlx5_flow_locate_proto_l3(const struct rte_flow_item **head, + const struct rte_flow_item *end) { - int ret; - uint64_t action_flags = 0; - uint64_t item_flags = 0; - uint64_t last_item = 0; - uint8_t next_protocol = 0xff; - uint16_t ether_type = 0; - int actions_n = 0; - uint8_t item_ipv6_proto = 0; - const struct rte_flow_item *geneve_item = NULL; - const struct rte_flow_item *gre_item = NULL; - const struct rte_flow_item *gtp_item = NULL; - const struct rte_flow_action_raw_decap *decap; - const struct rte_flow_action_raw_encap *encap; - const struct rte_flow_action_rss *rss = NULL; - const struct rte_flow_action_rss *sample_rss = NULL; - const struct rte_flow_item_tcp nic_tcp_mask = { - .hdr = { - .tcp_flags = 0xFF, - .src_port = RTE_BE16(UINT16_MAX), - .dst_port = RTE_BE16(UINT16_MAX), - } - }; - const struct rte_flow_item_ipv6 nic_ipv6_mask = { - .hdr = { - .src_addr = - "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff", - .dst_addr = - "\xff\xff\xff\xff\xff\xff\xff\xff" + const struct rte_flow_item *item = *head; + uint16_t l3_protocol; + + for (; item != end; item++) { + switch (item->type) { + default: + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + l3_protocol = RTE_ETHER_TYPE_IPV4; + goto l3_ok; + case RTE_FLOW_ITEM_TYPE_IPV6: + l3_protocol = RTE_ETHER_TYPE_IPV6; + goto l3_ok; + case RTE_FLOW_ITEM_TYPE_ETH: + if (item->mask && item->spec) { + MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_eth, + type, item, + l3_protocol); + if (l3_protocol == RTE_ETHER_TYPE_IPV4 || + l3_protocol == RTE_ETHER_TYPE_IPV6) + goto l3_ok; + } + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + if (item->mask && item->spec) { + MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_vlan, + inner_type, item, + l3_protocol); + if (l3_protocol == RTE_ETHER_TYPE_IPV4 || + l3_protocol == RTE_ETHER_TYPE_IPV6) + goto l3_ok; + } + break; + } + } + return 0; +l3_ok: + *head = item; + return l3_protocol; +} + +static uint8_t +mlx5_flow_locate_proto_l4(const struct rte_flow_item **head, + const struct rte_flow_item *end) +{ + const struct rte_flow_item *item = *head; + uint8_t l4_protocol; + + for (; item != end; item++) { + switch (item->type) { + default: + break; + case RTE_FLOW_ITEM_TYPE_TCP: + l4_protocol = IPPROTO_TCP; + goto l4_ok; + case RTE_FLOW_ITEM_TYPE_UDP: + l4_protocol = IPPROTO_UDP; + goto l4_ok; + case RTE_FLOW_ITEM_TYPE_IPV4: + if (item->mask && item->spec) { + const struct rte_flow_item_ipv4 *mask, *spec; + + mask = (typeof(mask))item->mask; + spec = (typeof(spec))item->spec; + l4_protocol = mask->hdr.next_proto_id & + spec->hdr.next_proto_id; + if (l4_protocol == IPPROTO_TCP || + l4_protocol == IPPROTO_UDP) + goto l4_ok; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + if (item->mask && item->spec) { + const struct rte_flow_item_ipv6 *mask, *spec; + mask = (typeof(mask))item->mask; + spec = (typeof(spec))item->spec; + l4_protocol = mask->hdr.proto & spec->hdr.proto; + if (l4_protocol == IPPROTO_TCP || + l4_protocol == IPPROTO_UDP) + goto l4_ok; + } + break; + } + } + return 0; +l4_ok: + *head = item; + return l4_protocol; +} + +static int +flow_dv_validate_item_integrity(struct rte_eth_dev *dev, + const struct rte_flow_item *rule_items, + const struct rte_flow_item *integrity_item, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_item *tunnel_item, *end_item, *item = rule_items; + const struct rte_flow_item_integrity *mask = (typeof(mask)) + integrity_item->mask; + const struct rte_flow_item_integrity *spec = (typeof(spec)) + integrity_item->spec; + uint32_t protocol; + + if (!priv->config.hca_attr.pkt_integrity_match) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + integrity_item, + "packet integrity integrity_item not supported"); + if (!mask) + mask = &rte_flow_item_integrity_mask; + if (!mlx5_validate_integrity_item(mask)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + integrity_item, + "unsupported integrity filter"); + tunnel_item = mlx5_flow_find_tunnel_item(rule_items); + if (spec->level > 1) { + if (!tunnel_item) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + integrity_item, + "missing tunnel item"); + item = tunnel_item; + end_item = mlx5_find_end_item(tunnel_item); + } else { + end_item = tunnel_item ? tunnel_item : + mlx5_find_end_item(integrity_item); + } + if (mask->l3_ok || mask->ipv4_csum_ok) { + protocol = mlx5_flow_locate_proto_l3(&item, end_item); + if (!protocol) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + integrity_item, + "missing L3 protocol"); + } + if (mask->l4_ok || mask->l4_csum_ok) { + protocol = mlx5_flow_locate_proto_l4(&item, end_item); + if (!protocol) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + integrity_item, + "missing L4 protocol"); + } + return 0; +} + +/** + * Internal validation function. For validating both actions and items. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] attr + * Pointer to the flow attributes. + * @param[in] items + * Pointer to the list of items. + * @param[in] actions + * Pointer to the list of actions. + * @param[in] external + * This flow rule is created by request external to PMD. + * @param[in] hairpin + * Number of hairpin TX actions, 0 means classic flow. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + bool external, int hairpin, struct rte_flow_error *error) +{ + int ret; + uint64_t action_flags = 0; + uint64_t item_flags = 0; + uint64_t last_item = 0; + uint8_t next_protocol = 0xff; + uint16_t ether_type = 0; + int actions_n = 0; + uint8_t item_ipv6_proto = 0; + int fdb_mirror_limit = 0; + int modify_after_mirror = 0; + const struct rte_flow_item *geneve_item = NULL; + const struct rte_flow_item *gre_item = NULL; + const struct rte_flow_item *gtp_item = NULL; + const struct rte_flow_action_raw_decap *decap; + const struct rte_flow_action_raw_encap *encap; + const struct rte_flow_action_rss *rss = NULL; + const struct rte_flow_action_rss *sample_rss = NULL; + const struct rte_flow_action_count *sample_count = NULL; + const struct rte_flow_item_tcp nic_tcp_mask = { + .hdr = { + .tcp_flags = 0xFF, + .src_port = RTE_BE16(UINT16_MAX), + .dst_port = RTE_BE16(UINT16_MAX), + } + }; + const struct rte_flow_item_ipv6 nic_ipv6_mask = { + .hdr = { + .src_addr = + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + .dst_addr = + "\xff\xff\xff\xff\xff\xff\xff\xff" "\xff\xff\xff\xff\xff\xff\xff\xff", .vtc_flow = RTE_BE32(0xffffffff), .proto = 0xff, @@ -6056,32 +6915,36 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, uint32_t rw_act_num = 0; uint64_t is_root; const struct mlx5_flow_tunnel *tunnel; + enum mlx5_tof_rule_type tof_rule_type; struct flow_grp_info grp_info = { .external = !!external, .transfer = !!attr->transfer, .fdb_def_rule = !!priv->fdb_def_rule, + .std_tbl_fix = true, }; const struct rte_eth_hairpin_conf *conf; + const struct rte_flow_item *rule_items = items; + const struct rte_flow_item *port_id_item = NULL; + bool def_policy = false; if (items == NULL) return -1; - if (is_flow_tunnel_match_rule(dev, attr, items, actions)) { - tunnel = flow_items_to_tunnel(items); - action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH | - MLX5_FLOW_ACTION_DECAP; - } else if (is_flow_tunnel_steer_rule(dev, attr, items, actions)) { - tunnel = flow_actions_to_tunnel(actions); - action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET; - } else { - tunnel = NULL; + tunnel = is_tunnel_offload_active(dev) ? + mlx5_get_tof(items, actions, &tof_rule_type) : NULL; + if (tunnel) { + if (priv->representor) + return rte_flow_error_set + (error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "decap not supported for VF representor"); + if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE) + action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET; + else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE) + action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH | + MLX5_FLOW_ACTION_DECAP; + grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate + (dev, attr, tunnel, tof_rule_type); } - if (tunnel && priv->representor) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "decap not supported " - "for VF representor"); - grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate - (dev, tunnel, attr, items, actions); ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error); if (ret < 0) return ret; @@ -6095,15 +6958,6 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "item not supported"); switch (type) { - case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL: - if (items[0].type != (typeof(items[0].type)) - MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) - return rte_flow_error_set - (error, EINVAL, - RTE_FLOW_ERROR_TYPE_ITEM, - NULL, "MLX5 private items " - "must be the first"); - break; case RTE_FLOW_ITEM_TYPE_VOID: break; case RTE_FLOW_ITEM_TYPE_PORT_ID: @@ -6112,6 +6966,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (ret < 0) return ret; last_item = MLX5_FLOW_ITEM_PORT_ID; + port_id_item = items; break; case RTE_FLOW_ITEM_TYPE_ETH: ret = mlx5_flow_validate_item_eth(items, item_flags, @@ -6157,7 +7012,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case RTE_FLOW_ITEM_TYPE_IPV4: mlx5_flow_tunnel_ip_check(items, next_protocol, &item_flags, &tunnel); - ret = flow_dv_validate_item_ipv4(items, item_flags, + ret = flow_dv_validate_item_ipv4(dev, items, item_flags, last_item, ether_type, error); if (ret < 0) @@ -6274,7 +7129,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, last_item = MLX5_FLOW_LAYER_GRE_KEY; break; case RTE_FLOW_ITEM_TYPE_VXLAN: - ret = mlx5_flow_validate_item_vxlan(items, item_flags, + ret = mlx5_flow_validate_item_vxlan(dev, items, + item_flags, attr, error); if (ret < 0) return ret; @@ -6384,6 +7240,29 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, return ret; last_item = MLX5_FLOW_LAYER_ECPRI; break; + case RTE_FLOW_ITEM_TYPE_INTEGRITY: + if (item_flags & MLX5_FLOW_ITEM_INTEGRITY) + return rte_flow_error_set + (error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "multiple integrity items not supported"); + ret = flow_dv_validate_item_integrity(dev, rule_items, + items, error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_ITEM_INTEGRITY; + break; + case RTE_FLOW_ITEM_TYPE_CONNTRACK: + ret = flow_dv_validate_item_aso_ct(dev, items, + &item_flags, error); + if (ret < 0) + return ret; + break; + case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL: + /* tunnel offload item was processed before + * list it here as a supported type + */ + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, @@ -6393,6 +7272,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, } for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { int type = actions->type; + bool shared_count = false; if (!mlx5_flow_os_action_supported(type)) return rte_flow_error_set(error, ENOTSUP, @@ -6403,6 +7283,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, actions, "too many actions"); + if (action_flags & + MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "meter action with policy " + "must be the last action"); switch (type) { case RTE_FLOW_ACTION_TYPE_VOID: break; @@ -6429,6 +7315,9 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; action_flags |= MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK_EXT; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; + } else { action_flags |= MLX5_FLOW_ACTION_FLAG; ++actions_n; @@ -6448,6 +7337,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; action_flags |= MLX5_FLOW_ACTION_MARK | MLX5_FLOW_ACTION_MARK_EXT; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; } else { action_flags |= MLX5_FLOW_ACTION_MARK; ++actions_n; @@ -6463,6 +7354,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_META; rw_act_num += MLX5_ACT_NUM_SET_META; break; @@ -6475,6 +7368,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_TAG; rw_act_num += MLX5_ACT_NUM_SET_TAG; break; @@ -6527,8 +7422,11 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; ++actions_n; break; + case MLX5_RTE_FLOW_ACTION_TYPE_COUNT: case RTE_FLOW_ACTION_TYPE_COUNT: - ret = flow_dv_validate_action_count(dev, action_flags, + shared_count = is_shared_action_count(actions); + ret = flow_dv_validate_action_count(dev, shared_count, + action_flags, error); if (ret < 0) return ret; @@ -6542,6 +7440,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, item_flags, attr, error)) return -rte_errno; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; ++actions_n; break; @@ -6553,6 +7453,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; ++actions_n; break; @@ -6561,6 +7463,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, (action_flags, actions, error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* Count PCP with push_vlan command. */ action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP; break; @@ -6570,6 +7474,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, actions, error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* Count VID with push_vlan command. */ action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; rw_act_num += MLX5_ACT_NUM_MDF_VID; @@ -6592,6 +7498,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, attr, error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_DECAP; ++actions_n; break; @@ -6619,6 +7527,9 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, actions, item_flags, error); if (ret < 0) return ret; + if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && + (action_flags & MLX5_FLOW_ACTION_DECAP)) + modify_after_mirror = 1; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: @@ -6635,6 +7546,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? MLX5_FLOW_ACTION_SET_MAC_SRC : MLX5_FLOW_ACTION_SET_MAC_DST; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* * Even if the source and destination MAC addresses have * overlap in the header with 4B alignment, the convert @@ -6655,6 +7568,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? MLX5_FLOW_ACTION_SET_IPV4_SRC : @@ -6678,6 +7593,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? MLX5_FLOW_ACTION_SET_IPV6_SRC : @@ -6695,6 +7612,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? MLX5_FLOW_ACTION_SET_TP_SRC : @@ -6712,6 +7631,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_TTL ? MLX5_FLOW_ACTION_SET_TTL : @@ -6725,6 +7646,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret) return ret; + if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && + fdb_mirror_limit) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "sample and jump action combination is not supported"); ++actions_n; action_flags |= MLX5_FLOW_ACTION_JUMP; break; @@ -6740,6 +7667,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? MLX5_FLOW_ACTION_INC_TCP_SEQ : @@ -6758,6 +7687,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? MLX5_FLOW_ACTION_INC_TCP_ACK : @@ -6774,10 +7705,15 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ret = mlx5_flow_validate_action_meter(dev, action_flags, actions, attr, + port_id_item, + &def_policy, error); if (ret < 0) return ret; action_flags |= MLX5_FLOW_ACTION_METER; + if (!def_policy) + action_flags |= + MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY; ++actions_n; /* Meter action will add one more TAG action. */ rw_act_num += MLX5_ACT_NUM_SET_TAG; @@ -6788,6 +7724,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "Shared ASO age action is not supported for group 0"); + if (action_flags & MLX5_FLOW_ACTION_AGE) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "duplicate age actions set"); action_flags |= MLX5_FLOW_ACTION_AGE; ++actions_n; break; @@ -6797,6 +7739,24 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + /* + * Validate the regular AGE action (using counter) + * mutual exclusion with share counter actions. + */ + if (!priv->sh->flow_hit_aso_en) { + if (shared_count) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "old age and shared count combination is not supported"); + if (sample_count) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "old age action and count must be in the same sub flow"); + } action_flags |= MLX5_FLOW_ACTION_AGE; ++actions_n; break; @@ -6811,6 +7771,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; rw_act_num += MLX5_ACT_NUM_SET_DSCP; break; @@ -6825,6 +7787,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; rw_act_num += MLX5_ACT_NUM_SET_DSCP; break; @@ -6833,40 +7797,43 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, actions, dev, attr, item_flags, rss, &sample_rss, + &sample_count, + &fdb_mirror_limit, error); if (ret < 0) return ret; action_flags |= MLX5_FLOW_ACTION_SAMPLE; ++actions_n; break; - case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET: - if (actions[0].type != (typeof(actions[0].type)) - MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) - return rte_flow_error_set - (error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, "MLX5 private action " - "must be the first"); - - action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET; - break; case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: - if (!attr->transfer && !attr->group) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, "modify field action " - "is not supported for group 0"); - ret = flow_dv_validate_action_modify_field(action_flags, - actions, - error); + ret = flow_dv_validate_action_modify_field(dev, + action_flags, + actions, + attr, + error); if (ret < 0) return ret; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* Count all modify-header actions as one action. */ - if (!(action_flags & MLX5_FLOW_ACTION_MODIFY_FIELD)) + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD; rw_act_num += ret; break; + case RTE_FLOW_ACTION_TYPE_CONNTRACK: + ret = flow_dv_validate_action_aso_ct(dev, action_flags, + item_flags, attr, + error); + if (ret < 0) + return ret; + action_flags |= MLX5_FLOW_ACTION_CT; + break; + case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET: + /* tunnel offload action was processed before + * list it here as a supported type + */ + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -7013,6 +7980,36 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, "multiple VLAN actions"); } } + if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) { + if ((action_flags & (MLX5_FLOW_FATE_ACTIONS & + ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) && + attr->ingress) + return rte_flow_error_set + (error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "fate action not supported for " + "meter with policy"); + if (attr->egress) { + if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) + return rte_flow_error_set + (error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "modify header action in egress " + "cannot be done before meter action"); + if (action_flags & MLX5_FLOW_ACTION_ENCAP) + return rte_flow_error_set + (error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "encap action in egress " + "cannot be done before meter action"); + if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) + return rte_flow_error_set + (error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "push vlan action in egress " + "cannot be done before meter action"); + } + } /* * Hairpin flow will add one more TAG action in TX implicit mode. * In TX explicit mode, there will be no hairpin flow ID. @@ -7031,6 +8028,11 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, NULL, "too many header modify" " actions to support"); } + /* Eswitch egress mirror and modify flow has limitation on CX5 */ + if (fdb_mirror_limit && modify_after_mirror) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "sample before modify action is not supported"); return 0; } @@ -7067,6 +8069,9 @@ flow_dv_prepare(struct rte_eth_dev *dev, struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); MLX5_ASSERT(wks); + wks->skip_matcher_reg = 0; + wks->policy = NULL; + wks->final_policy = NULL; /* In case of corrupting the memory. */ if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { rte_flow_error_set(error, ENOSPC, @@ -7087,15 +8092,7 @@ flow_dv_prepare(struct rte_eth_dev *dev, memset(dev_flow, 0, sizeof(*dev_flow)); dev_flow->handle = dev_handle; dev_flow->handle_idx = handle_idx; - /* - * In some old rdma-core releases, before continuing, a check of the - * length of matching parameter will be done at first. It needs to use - * the length without misc4 param. If the flow has misc4 support, then - * the length needs to be adjusted accordingly. Each param member is - * aligned with a 64B boundary naturally. - */ - dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) - - MLX5_ST_SZ_BYTES(fte_match_set_misc4); + dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); dev_flow->ingress = attr->ingress; dev_flow->dv.transfer = attr->transfer; return dev_flow; @@ -7394,7 +8391,7 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, void *headers_v; char *l24_m; char *l24_v; - uint8_t tos; + uint8_t tos, ihl_m, ihl_v; if (inner) { headers_m = MLX5_ADDR_OF(fte_match_param, matcher, @@ -7423,6 +8420,10 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, *(uint32_t *)l24_m = ipv4_m->hdr.src_addr; *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr; tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service; + ihl_m = ipv4_m->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK; + ihl_v = ipv4_v->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK; + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_ihl, ihl_m); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_ihl, ihl_m & ihl_v); MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, ipv4_m->hdr.type_of_service); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos); @@ -7876,6 +8877,10 @@ flow_dv_translate_item_nvgre(void *matcher, void *key, /** * Add VXLAN item to matcher and to the value. * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] attr + * Flow rule attributes. * @param[in, out] matcher * Flow matcher. * @param[in, out] key @@ -7886,7 +8891,9 @@ flow_dv_translate_item_nvgre(void *matcher, void *key, * Item is inner pattern. */ static void -flow_dv_translate_item_vxlan(void *matcher, void *key, +flow_dv_translate_item_vxlan(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + void *matcher, void *key, const struct rte_flow_item *item, int inner) { @@ -7894,13 +8901,16 @@ flow_dv_translate_item_vxlan(void *matcher, void *key, const struct rte_flow_item_vxlan *vxlan_v = item->spec; void *headers_m; void *headers_v; - void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); - void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); - char *vni_m; - char *vni_v; + void *misc5_m; + void *misc5_v; + uint32_t *tunnel_header_v; + uint32_t *tunnel_header_m; uint16_t dport; - int size; - int i; + struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_item_vxlan nic_mask = { + .vni = "\xff\xff\xff", + .rsvd1 = 0xff, + }; if (inner) { headers_m = MLX5_ADDR_OF(fte_match_param, matcher, @@ -7919,14 +8929,52 @@ flow_dv_translate_item_vxlan(void *matcher, void *key, } if (!vxlan_v) return; - if (!vxlan_m) - vxlan_m = &rte_flow_item_vxlan_mask; - size = sizeof(vxlan_m->vni); - vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni); - vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni); - memcpy(vni_m, vxlan_m->vni, size); - for (i = 0; i < size; ++i) - vni_v[i] = vni_m[i] & vxlan_v->vni[i]; + if (!vxlan_m) { + if ((!attr->group && !priv->sh->tunnel_header_0_1) || + (attr->group && !priv->sh->misc5_cap)) + vxlan_m = &rte_flow_item_vxlan_mask; + else + vxlan_m = &nic_mask; + } + if ((!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) || + ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) { + void *misc_m; + void *misc_v; + char *vni_m; + char *vni_v; + int size; + int i; + misc_m = MLX5_ADDR_OF(fte_match_param, + matcher, misc_parameters); + misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); + size = sizeof(vxlan_m->vni); + vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni); + vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni); + memcpy(vni_m, vxlan_m->vni, size); + for (i = 0; i < size; ++i) + vni_v[i] = vni_m[i] & vxlan_v->vni[i]; + return; + } + misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5); + misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5); + tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5, + misc5_v, + tunnel_header_1); + tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5, + misc5_m, + tunnel_header_1); + *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) | + (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 | + (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16; + if (*tunnel_header_v) + *tunnel_header_m = vxlan_m->vni[0] | + vxlan_m->vni[1] << 8 | + vxlan_m->vni[2] << 16; + else + *tunnel_header_m = 0x0; + *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24; + if (vxlan_v->rsvd1 & vxlan_m->rsvd1) + *tunnel_header_m |= vxlan_m->rsvd1 << 24; } /** @@ -8267,14 +9315,13 @@ flow_dv_translate_item_mpls(void *matcher, void *key, MLX5_UDP_PORT_MPLS); break; case MLX5_FLOW_LAYER_GRE: + /* Fall-through. */ + case MLX5_FLOW_LAYER_GRE_KEY: MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff); MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, RTE_ETHER_TYPE_MPLS); break; default: - MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); - MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, - IPPROTO_MPLS); break; } if (!in_mpls_v) @@ -8484,27 +9531,14 @@ flow_dv_translate_item_meta(struct rte_eth_dev *dev, if (reg < 0) return; MLX5_ASSERT(reg != REG_NON); - /* - * In datapath code there is no endianness - * coversions for perfromance reasons, all - * pattern conversions are done in rte_flow. - */ - value = rte_cpu_to_be_32(value); - mask = rte_cpu_to_be_32(mask); if (reg == REG_C_0) { struct mlx5_priv *priv = dev->data->dev_private; uint32_t msk_c0 = priv->sh->dv_regc0_mask; uint32_t shl_c0 = rte_bsf32(msk_c0); -#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask); - value >>= shr_c0; - mask >>= shr_c0; -#endif - value <<= shl_c0; + mask &= msk_c0; mask <<= shl_c0; - MLX5_ASSERT(msk_c0); - MLX5_ASSERT(!(~msk_c0 & mask)); + value <<= shl_c0; } flow_dv_match_meta_reg(matcher, key, reg, value, mask); } @@ -9005,22 +10039,80 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, } } -static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; - -#define HEADER_IS_ZERO(match_criteria, headers) \ - !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ - matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ - -/** - * Calculate flow matcher enable bitmap. - * - * @param match_criteria - * Pointer to flow matcher criteria. +/* + * Add connection tracking status item to matcher * - * @return - * Bitmap of enabled fields. - */ -static uint8_t + * @param[in] dev + * The devich to configure through. + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + */ +static void +flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev, + void *matcher, void *key, + const struct rte_flow_item *item) +{ + uint32_t reg_value = 0; + int reg_id; + /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */ + uint32_t reg_mask = 0; + const struct rte_flow_item_conntrack *spec = item->spec; + const struct rte_flow_item_conntrack *mask = item->mask; + uint32_t flags; + struct rte_flow_error error; + + if (!mask) + mask = &rte_flow_item_conntrack_mask; + if (!spec || !mask->flags) + return; + flags = spec->flags & mask->flags; + /* The conflict should be checked in the validation. */ + if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) + reg_value |= MLX5_CT_SYNDROME_VALID; + if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED) + reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE; + if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) + reg_value |= MLX5_CT_SYNDROME_INVALID; + if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED) + reg_value |= MLX5_CT_SYNDROME_TRAP; + if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) + reg_value |= MLX5_CT_SYNDROME_BAD_PACKET; + if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID | + RTE_FLOW_CONNTRACK_PKT_STATE_INVALID | + RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)) + reg_mask |= 0xc0; + if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED) + reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE; + if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) + reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET; + /* The REG_C_x value could be saved during startup. */ + reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error); + if (reg_id == REG_NON) + return; + flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id, + reg_value, reg_mask); +} + +static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; + +#define HEADER_IS_ZERO(match_criteria, headers) \ + !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ + matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ + +/** + * Calculate flow matcher enable bitmap. + * + * @param match_criteria + * Pointer to flow matcher criteria. + * + * @return + * Bitmap of enabled fields. + */ +static uint8_t flow_dv_matcher_enable(uint32_t *match_criteria) { uint8_t match_criteria_enable; @@ -9043,19 +10135,72 @@ flow_dv_matcher_enable(uint32_t *match_criteria) match_criteria_enable |= (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) << MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT; + match_criteria_enable |= + (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) << + MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT; return match_criteria_enable; } -struct mlx5_hlist_entry * -flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) +static void +__flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria) +{ + /* + * Check flow matching criteria first, subtract misc5/4 length if flow + * doesn't own misc5/4 parameters. In some old rdma-core releases, + * misc5/4 are not supported, and matcher creation failure is expected + * w/o subtration. If misc5 is provided, misc4 must be counted in since + * misc5 is right after misc4. + */ + if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) { + *size = MLX5_ST_SZ_BYTES(fte_match_param) - + MLX5_ST_SZ_BYTES(fte_match_set_misc5); + if (!(match_criteria & (1 << + MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) { + *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4); + } + } +} + +static struct mlx5_list_entry * +flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_matcher *ref = ctx->data; + struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl, + typeof(*tbl), tbl); + struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY, + sizeof(*resource), + 0, SOCKET_ID_ANY); + + if (!resource) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot create matcher"); + return NULL; + } + memcpy(resource, entry, sizeof(*resource)); + resource->tbl = &tbl->tbl; + return &resource->entry; +} + +static void +flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry) +{ + mlx5_free(entry); +} + +struct mlx5_list_entry * +flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct rte_eth_dev *dev = ctx->dev; struct mlx5_flow_tbl_data_entry *tbl_data; - struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data; + struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2; struct rte_flow_error *error = ctx->error; - union mlx5_flow_tbl_key key = { .v64 = key64 }; + union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) }; struct mlx5_flow_tbl_resource *tbl; void *domain; uint32_t idx = 0; @@ -9074,20 +10219,21 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) tbl_data->group_id = tt_prm->group_id; tbl_data->external = !!tt_prm->external; tbl_data->tunnel_offload = is_tunnel_offload_active(dev); - tbl_data->is_egress = !!key.direction; - tbl_data->is_transfer = !!key.domain; + tbl_data->is_egress = !!key.is_egress; + tbl_data->is_transfer = !!key.is_fdb; tbl_data->dummy = !!key.dummy; - tbl_data->table_id = key.table_id; + tbl_data->level = key.level; + tbl_data->id = key.id; tbl = &tbl_data->tbl; if (key.dummy) return &tbl_data->entry; - if (key.domain) + if (key.is_fdb) domain = sh->fdb_domain; - else if (key.direction) + else if (key.is_egress) domain = sh->tx_domain; else domain = sh->rx_domain; - ret = mlx5_flow_os_create_flow_tbl(domain, key.table_id, &tbl->obj); + ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj); if (ret) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -9095,7 +10241,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); return NULL; } - if (key.table_id) { + if (key.level != 0) { ret = mlx5_flow_os_create_flow_action_dest_flow_tbl (tbl->obj, &tbl_data->jump.action); if (ret) { @@ -9108,29 +10254,75 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) return NULL; } } - MKSTR(matcher_name, "%s_%s_%u_matcher_cache", - key.domain ? "FDB" : "NIC", key.direction ? "egress" : "ingress", - key.table_id); - mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh, - flow_dv_matcher_create_cb, - flow_dv_matcher_match_cb, - flow_dv_matcher_remove_cb); + MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list", + key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress", + key.level, key.id); + tbl_data->matchers = mlx5_list_create(matcher_name, sh, true, + flow_dv_matcher_create_cb, + flow_dv_matcher_match_cb, + flow_dv_matcher_remove_cb, + flow_dv_matcher_clone_cb, + flow_dv_matcher_clone_free_cb); + if (!tbl_data->matchers) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot create tbl matcher list"); + mlx5_flow_os_destroy_flow_action(tbl_data->jump.action); + mlx5_flow_os_destroy_flow_tbl(tbl->obj); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); + return NULL; + } return &tbl_data->entry; } int -flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused, - struct mlx5_hlist_entry *entry, uint64_t key64, - void *cb_ctx __rte_unused) +flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry, + void *cb_ctx) { + struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_tbl_data_entry *tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry); - union mlx5_flow_tbl_key key = { .v64 = key64 }; + union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) }; - return tbl_data->table_id != key.table_id || + return tbl_data->level != key.level || + tbl_data->id != key.id || tbl_data->dummy != key.dummy || - tbl_data->is_transfer != key.domain || - tbl_data->is_egress != key.direction; + tbl_data->is_transfer != !!key.is_fdb || + tbl_data->is_egress != !!key.is_egress; +} + +struct mlx5_list_entry * +flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct rte_flow_error *error = ctx->error; + uint32_t idx = 0; + + tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); + if (!tbl_data) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot allocate flow table data entry"); + return NULL; + } + memcpy(tbl_data, oentry, sizeof(*tbl_data)); + tbl_data->idx = idx; + return &tbl_data->entry; +} + +void +flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx); } /** @@ -9138,14 +10330,16 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused, * * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in] table_id - * Table id to use. + * @param[in] table_level + * Table level to use. * @param[in] egress * Direction of the table. * @param[in] transfer * E-Switch or NIC flow. * @param[in] dummy * Dummy entry for dv API. + * @param[in] table_id + * Table id to use. * @param[out] error * pointer to error structure. * @@ -9154,20 +10348,23 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused, */ struct mlx5_flow_tbl_resource * flow_dv_tbl_resource_get(struct rte_eth_dev *dev, - uint32_t table_id, uint8_t egress, + uint32_t table_level, uint8_t egress, uint8_t transfer, bool external, const struct mlx5_flow_tunnel *tunnel, uint32_t group_id, uint8_t dummy, + uint32_t table_id, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; union mlx5_flow_tbl_key table_key = { { - .table_id = table_id, - .dummy = dummy, - .domain = !!transfer, - .direction = !!egress, + .level = table_level, + .id = table_id, + .reserved = 0, + .dummy = !!dummy, + .is_fdb = !!transfer, + .is_egress = !!egress, } }; struct mlx5_flow_tbl_tunnel_prm tt_prm = { @@ -9178,9 +10375,10 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, struct mlx5_flow_cb_ctx ctx = { .dev = dev, .error = error, - .data = &tt_prm, + .data = &table_key.v64, + .data2 = &tt_prm, }; - struct mlx5_hlist_entry *entry; + struct mlx5_list_entry *entry; struct mlx5_flow_tbl_data_entry *tbl_data; entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx); @@ -9190,19 +10388,20 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, "cannot get table"); return NULL; } - DRV_LOG(DEBUG, "Table_id %u tunnel %u group %u registered.", - table_id, tunnel ? tunnel->tunnel_id : 0, group_id); + DRV_LOG(DEBUG, "table_level %u table_id %u " + "tunnel %u group %u registered.", + table_level, table_id, + tunnel ? tunnel->tunnel_id : 0, group_id); tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry); return &tbl_data->tbl; } void -flow_dv_tbl_remove_cb(struct mlx5_hlist *list, - struct mlx5_hlist_entry *entry) +flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_tbl_data_entry *tbl_data = - container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + container_of(entry, struct mlx5_flow_tbl_data_entry, entry); MLX5_ASSERT(entry && sh); if (tbl_data->jump.action) @@ -9210,7 +10409,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list, if (tbl_data->tbl.obj) mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj); if (tbl_data->tunnel_offload && tbl_data->external) { - struct mlx5_hlist_entry *he; + struct mlx5_list_entry *he; struct mlx5_hlist *tunnel_grp_hash; struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; union tunnel_tbl_key tunnel_key = { @@ -9218,22 +10417,26 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list, tbl_data->tunnel->tunnel_id : 0, .group = tbl_data->group_id }; - uint32_t table_id = tbl_data->table_id; + uint32_t table_level = tbl_data->level; + struct mlx5_flow_cb_ctx ctx = { + .data = (void *)&tunnel_key.val, + }; tunnel_grp_hash = tbl_data->tunnel ? tbl_data->tunnel->groups : thub->groups; - he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL); + he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx); if (he) mlx5_hlist_unregister(tunnel_grp_hash, he); DRV_LOG(DEBUG, - "Table_id %u tunnel %u group %u released.", - table_id, + "table_level %u id %u tunnel %u group %u released.", + table_level, + tbl_data->id, tbl_data->tunnel ? tbl_data->tunnel->tunnel_id : 0, tbl_data->group_id); } - mlx5_cache_list_destroy(&tbl_data->matchers); + mlx5_list_destroy(tbl_data->matchers); mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx); } @@ -9261,8 +10464,8 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh, } int -flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry, void *cb_ctx) +flow_dv_matcher_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_dv_matcher *ref = ctx->data; @@ -9275,15 +10478,13 @@ flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused, (const void *)ref->mask.buf, ref->mask.size); } -struct mlx5_cache_entry * -flow_dv_matcher_create_cb(struct mlx5_cache_list *list, - struct mlx5_cache_entry *entry __rte_unused, - void *cb_ctx) +struct mlx5_list_entry * +flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_dv_matcher *ref = ctx->data; - struct mlx5_flow_dv_matcher *cache; + struct mlx5_flow_dv_matcher *resource; struct mlx5dv_flow_matcher_attr dv_attr = { .type = IBV_FLOW_ATTR_NORMAL, .match_mask = (void *)&ref->mask, @@ -9292,29 +10493,32 @@ flow_dv_matcher_create_cb(struct mlx5_cache_list *list, typeof(*tbl), tbl); int ret; - cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY); - if (!cache) { + resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0, + SOCKET_ID_ANY); + if (!resource) { rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create matcher"); return NULL; } - *cache = *ref; + *resource = *ref; dv_attr.match_criteria_enable = - flow_dv_matcher_enable(cache->mask.buf); + flow_dv_matcher_enable(resource->mask.buf); + __flow_dv_adjust_buf_size(&ref->mask.size, + dv_attr.match_criteria_enable); dv_attr.priority = ref->priority; if (tbl->is_egress) dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj, - &cache->matcher_object); + &resource->matcher_object); if (ret) { - mlx5_free(cache); + mlx5_free(resource); rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create matcher"); return NULL; } - return &cache->entry; + return &resource->entry; } /** @@ -9343,62 +10547,61 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, uint32_t group_id, struct rte_flow_error *error) { - struct mlx5_cache_entry *entry; - struct mlx5_flow_dv_matcher *cache; + struct mlx5_list_entry *entry; + struct mlx5_flow_dv_matcher *resource; struct mlx5_flow_tbl_resource *tbl; struct mlx5_flow_tbl_data_entry *tbl_data; struct mlx5_flow_cb_ctx ctx = { .error = error, .data = ref, }; - /** * tunnel offload API requires this registration for cases when * tunnel match rule was inserted before tunnel set rule. */ - tbl = flow_dv_tbl_resource_get(dev, key->table_id, - key->direction, key->domain, + tbl = flow_dv_tbl_resource_get(dev, key->level, + key->is_egress, key->is_fdb, dev_flow->external, tunnel, - group_id, 0, error); + group_id, 0, key->id, error); if (!tbl) return -rte_errno; /* No need to refill the error info */ tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); ref->tbl = tbl; - entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + entry = mlx5_list_register(tbl_data->matchers, &ctx); if (!entry) { flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate ref memory"); } - cache = container_of(entry, typeof(*cache), entry); - dev_flow->handle->dvh.matcher = cache; + resource = container_of(entry, typeof(*resource), entry); + dev_flow->handle->dvh.matcher = resource; return 0; } -struct mlx5_hlist_entry * -flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx) +struct mlx5_list_entry * +flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx) { - struct mlx5_dev_ctx_shared *sh = list->ctx; - struct rte_flow_error *error = ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct mlx5_flow_dv_tag_resource *entry; uint32_t idx = 0; int ret; entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx); if (!entry) { - rte_flow_error_set(error, ENOMEM, + rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); return NULL; } entry->idx = idx; - entry->tag_id = key; - ret = mlx5_flow_os_create_flow_action_tag(key, + entry->tag_id = *(uint32_t *)(ctx->data); + ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id, &entry->action); if (ret) { mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx); - rte_flow_error_set(error, ENOMEM, + rte_flow_error_set(ctx->error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create action"); return NULL; @@ -9407,14 +10610,45 @@ flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx) } int -flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused, - struct mlx5_hlist_entry *entry, uint64_t key, - void *cb_ctx __rte_unused) +flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry, + void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_tag_resource *tag = + container_of(entry, struct mlx5_flow_dv_tag_resource, entry); + + return *(uint32_t *)(ctx->data) != tag->tag_id; +} + +struct mlx5_list_entry * +flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, + void *cb_ctx) { + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_tag_resource *entry; + uint32_t idx = 0; + + entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx); + if (!entry) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate tag resource memory"); + return NULL; + } + memcpy(entry, oentry, sizeof(*entry)); + entry->idx = idx; + return &entry->entry; +} + +void +flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_dv_tag_resource *tag = - container_of(entry, struct mlx5_flow_dv_tag_resource, entry); + container_of(entry, struct mlx5_flow_dv_tag_resource, entry); - return key != tag->tag_id; + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx); } /** @@ -9440,27 +10674,42 @@ flow_dv_tag_resource_register struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_tag_resource *cache_resource; - struct mlx5_hlist_entry *entry; - - entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error); + struct mlx5_flow_dv_tag_resource *resource; + struct mlx5_list_entry *entry; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &tag_be24, + }; + struct mlx5_hlist *tag_table; + + tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table, + "tags", + MLX5_TAGS_HLIST_ARRAY_SIZE, + false, false, priv->sh, + flow_dv_tag_create_cb, + flow_dv_tag_match_cb, + flow_dv_tag_remove_cb, + flow_dv_tag_clone_cb, + flow_dv_tag_clone_free_cb); + if (unlikely(!tag_table)) + return -rte_errno; + entry = mlx5_hlist_register(tag_table, tag_be24, &ctx); if (entry) { - cache_resource = container_of - (entry, struct mlx5_flow_dv_tag_resource, entry); - dev_flow->handle->dvh.rix_tag = cache_resource->idx; - dev_flow->dv.tag_resource = cache_resource; + resource = container_of(entry, struct mlx5_flow_dv_tag_resource, + entry); + dev_flow->handle->dvh.rix_tag = resource->idx; + dev_flow->dv.tag_resource = resource; return 0; } return -rte_errno; } void -flow_dv_tag_remove_cb(struct mlx5_hlist *list, - struct mlx5_hlist_entry *entry) +flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_dv_tag_resource *tag = - container_of(entry, struct mlx5_flow_dv_tag_resource, entry); + container_of(entry, struct mlx5_flow_dv_tag_resource, entry); MLX5_ASSERT(tag && sh && tag->action); claim_zero(mlx5_flow_os_destroy_flow_action(tag->action)); @@ -9527,7 +10776,7 @@ flow_dv_translate_action_port_id(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "No eswitch info was found for port"); -#ifdef HAVE_MLX5DV_DR_DEVX_PORT +#ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT /* * This parameter is transferred to * mlx5dv_dr_action_create_dest_ib_port(). @@ -9549,6 +10798,8 @@ flow_dv_translate_action_port_id(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to rte_eth_dev structure. + * @param[in] dev_flow + * Pointer to the mlx5_flow. * @param[out] count * Pointer to the counter action configuration. * @param[in] age @@ -9572,7 +10823,7 @@ flow_dv_translate_create_counter(struct rte_eth_dev *dev, counter = flow_dv_counter_alloc(dev, !!age); if (!counter || age == NULL) return counter; - age_param = flow_dv_counter_idx_get_age(dev, counter); + age_param = flow_dv_counter_idx_get_age(dev, counter); age_param->context = age->context ? age->context : (void *)(uintptr_t)(dev_flow->flow_idx); age_param->timeout = age->timeout; @@ -9645,10 +10896,8 @@ flow_dv_hashfields_set(struct mlx5_flow *dev_flow, dev_flow->hash_fields = 0; #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT - if (rss_desc->level >= 2) { - dev_flow->hash_fields |= IBV_RX_HASH_INNER; + if (rss_desc->level >= 2) rss_inner = 1; - } #endif if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) || (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) { @@ -9671,6 +10920,12 @@ flow_dv_hashfields_set(struct mlx5_flow *dev_flow, dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH; } } + if (dev_flow->hash_fields == 0) + /* + * There is no match between the RSS types and the + * L3 protocol (IPv4/IPv6) defined in the flow rule. + */ + return; if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) || (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) { if (rss_types & ETH_RSS_UDP) { @@ -9696,6 +10951,8 @@ flow_dv_hashfields_set(struct mlx5_flow *dev_flow, dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH; } } + if (rss_inner) + dev_flow->hash_fields |= IBV_RX_HASH_INNER; } /** @@ -9728,6 +10985,8 @@ flow_dv_hrxq_prepare(struct rte_eth_dev *dev, rss_desc->hash_fields = dev_flow->hash_fields; rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL); rss_desc->shared_rss = 0; + if (rss_desc->hash_fields == 0) + rss_desc->queue_num = 1; *hrxq_idx = mlx5_hrxq_get(dev, rss_desc); if (!*hrxq_idx) return NULL; @@ -9773,72 +11032,71 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev, } int -flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry, void *cb_ctx) +flow_dv_sample_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct rte_eth_dev *dev = ctx->dev; - struct mlx5_flow_dv_sample_resource *resource = ctx->data; - struct mlx5_flow_dv_sample_resource *cache_resource = - container_of(entry, typeof(*cache_resource), entry); - - if (resource->ratio == cache_resource->ratio && - resource->ft_type == cache_resource->ft_type && - resource->ft_id == cache_resource->ft_id && - resource->set_action == cache_resource->set_action && - !memcmp((void *)&resource->sample_act, - (void *)&cache_resource->sample_act, + struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data; + struct mlx5_flow_dv_sample_resource *resource = container_of(entry, + typeof(*resource), + entry); + + if (ctx_resource->ratio == resource->ratio && + ctx_resource->ft_type == resource->ft_type && + ctx_resource->ft_id == resource->ft_id && + ctx_resource->set_action == resource->set_action && + !memcmp((void *)&ctx_resource->sample_act, + (void *)&resource->sample_act, sizeof(struct mlx5_flow_sub_actions_list))) { /* * Existing sample action should release the prepared * sub-actions reference counter. */ flow_dv_sample_sub_actions_release(dev, - &resource->sample_idx); + &ctx_resource->sample_idx); return 0; } return 1; } -struct mlx5_cache_entry * -flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry __rte_unused, - void *cb_ctx) +struct mlx5_list_entry * +flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct rte_eth_dev *dev = ctx->dev; - struct mlx5_flow_dv_sample_resource *resource = ctx->data; - void **sample_dv_actions = resource->sub_actions; - struct mlx5_flow_dv_sample_resource *cache_resource; + struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data; + void **sample_dv_actions = ctx_resource->sub_actions; + struct mlx5_flow_dv_sample_resource *resource; struct mlx5dv_dr_flow_sampler_attr sampler_attr; struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_flow_tbl_resource *tbl; uint32_t idx = 0; const uint32_t next_ft_step = 1; - uint32_t next_ft_id = resource->ft_id + next_ft_step; + uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step; uint8_t is_egress = 0; uint8_t is_transfer = 0; struct rte_flow_error *error = ctx->error; /* Register new sample resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx); - if (!cache_resource) { + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx); + if (!resource) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); return NULL; } - *cache_resource = *resource; + *resource = *ctx_resource; /* Create normal path table level */ - if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) is_transfer = 1; - else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) + else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) is_egress = 1; tbl = flow_dv_tbl_resource_get(dev, next_ft_id, is_egress, is_transfer, - true, NULL, 0, 0, error); + true, NULL, 0, 0, 0, error); if (!tbl) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -9847,8 +11105,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused, "for sample"); goto error; } - cache_resource->normal_path_tbl = tbl; - if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) { + resource->normal_path_tbl = tbl; + if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) { if (!sh->default_miss_action) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -9857,45 +11115,83 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused, "created"); goto error; } - sample_dv_actions[resource->sample_act.actions_num++] = + sample_dv_actions[ctx_resource->sample_act.actions_num++] = sh->default_miss_action; } /* Create a DR sample action */ - sampler_attr.sample_ratio = cache_resource->ratio; + sampler_attr.sample_ratio = resource->ratio; sampler_attr.default_next_table = tbl->obj; - sampler_attr.num_sample_actions = resource->sample_act.actions_num; + sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num; sampler_attr.sample_actions = (struct mlx5dv_dr_action **) &sample_dv_actions[0]; - sampler_attr.action = cache_resource->set_action; + sampler_attr.action = resource->set_action; if (mlx5_os_flow_dr_create_flow_action_sampler - (&sampler_attr, &cache_resource->verbs_action)) { + (&sampler_attr, &resource->verbs_action)) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create sample action"); goto error; } - cache_resource->idx = idx; - cache_resource->dev = dev; - return &cache_resource->entry; + resource->idx = idx; + resource->dev = dev; + return &resource->entry; error: - if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB) + if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB) flow_dv_sample_sub_actions_release(dev, - &cache_resource->sample_idx); - if (cache_resource->normal_path_tbl) + &resource->sample_idx); + if (resource->normal_path_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), - cache_resource->normal_path_tbl); + resource->normal_path_tbl); mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx); return NULL; } +struct mlx5_list_entry * +flow_dv_sample_clone_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; + struct mlx5_flow_dv_sample_resource *resource; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + uint32_t idx = 0; + + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx); + if (!resource) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot allocate resource memory"); + return NULL; + } + memcpy(resource, entry, sizeof(*resource)); + resource->idx = idx; + resource->dev = dev; + return &resource->entry; +} + +void +flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry) +{ + struct mlx5_flow_dv_sample_resource *resource = + container_of(entry, typeof(*resource), entry); + struct rte_eth_dev *dev = resource->dev; + struct mlx5_priv *priv = dev->data->dev_private; + + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx); +} + /** * Find existing sample resource or create and register a new one. * * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in] resource - * Pointer to sample resource. + * @param[in] ref + * Pointer to sample resource reference. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -9906,66 +11202,64 @@ error: */ static int flow_dv_sample_resource_register(struct rte_eth_dev *dev, - struct mlx5_flow_dv_sample_resource *resource, + struct mlx5_flow_dv_sample_resource *ref, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { - struct mlx5_flow_dv_sample_resource *cache_resource; - struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_sample_resource *resource; + struct mlx5_list_entry *entry; struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_cb_ctx ctx = { .dev = dev, .error = error, - .data = resource, + .data = ref, }; - entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx); + entry = mlx5_list_register(priv->sh->sample_action_list, &ctx); if (!entry) return -rte_errno; - cache_resource = container_of(entry, typeof(*cache_resource), entry); - dev_flow->handle->dvh.rix_sample = cache_resource->idx; - dev_flow->dv.sample_res = cache_resource; + resource = container_of(entry, typeof(*resource), entry); + dev_flow->handle->dvh.rix_sample = resource->idx; + dev_flow->dv.sample_res = resource; return 0; } int -flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry, void *cb_ctx) +flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; - struct mlx5_flow_dv_dest_array_resource *resource = ctx->data; + struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data; struct rte_eth_dev *dev = ctx->dev; - struct mlx5_flow_dv_dest_array_resource *cache_resource = - container_of(entry, typeof(*cache_resource), entry); + struct mlx5_flow_dv_dest_array_resource *resource = + container_of(entry, typeof(*resource), entry); uint32_t idx = 0; - if (resource->num_of_dest == cache_resource->num_of_dest && - resource->ft_type == cache_resource->ft_type && - !memcmp((void *)cache_resource->sample_act, - (void *)resource->sample_act, - (resource->num_of_dest * + if (ctx_resource->num_of_dest == resource->num_of_dest && + ctx_resource->ft_type == resource->ft_type && + !memcmp((void *)resource->sample_act, + (void *)ctx_resource->sample_act, + (ctx_resource->num_of_dest * sizeof(struct mlx5_flow_sub_actions_list)))) { /* * Existing sample action should release the prepared * sub-actions reference counter. */ - for (idx = 0; idx < resource->num_of_dest; idx++) + for (idx = 0; idx < ctx_resource->num_of_dest; idx++) flow_dv_sample_sub_actions_release(dev, - &resource->sample_idx[idx]); + &ctx_resource->sample_idx[idx]); return 0; } return 1; } -struct mlx5_cache_entry * -flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry __rte_unused, - void *cb_ctx) +struct mlx5_list_entry * +flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx) { struct mlx5_flow_cb_ctx *ctx = cb_ctx; struct rte_eth_dev *dev = ctx->dev; - struct mlx5_flow_dv_dest_array_resource *cache_resource; - struct mlx5_flow_dv_dest_array_resource *resource = ctx->data; + struct mlx5_flow_dv_dest_array_resource *resource; + struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data; struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 }; struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM]; struct mlx5_priv *priv = dev->data->dev_private; @@ -9978,23 +11272,23 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused, int ret; /* Register new destination array resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY], + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY], &res_idx); - if (!cache_resource) { + if (!resource) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); return NULL; } - *cache_resource = *resource; + *resource = *ctx_resource; if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) domain = sh->fdb_domain; else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) domain = sh->rx_domain; else domain = sh->tx_domain; - for (idx = 0; idx < resource->num_of_dest; idx++) { + for (idx = 0; idx < ctx_resource->num_of_dest; idx++) { dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *) mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5dv_dr_action_dest_attr), @@ -10007,7 +11301,7 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused, goto error; } dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST; - sample_act = &resource->sample_act[idx]; + sample_act = &ctx_resource->sample_act[idx]; action_flags = sample_act->action_flags; switch (action_flags) { case MLX5_FLOW_ACTION_QUEUE: @@ -10038,9 +11332,9 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused, /* create a dest array actioin */ ret = mlx5_os_flow_dr_create_flow_action_dest_array (domain, - cache_resource->num_of_dest, + resource->num_of_dest, dest_attr, - &cache_resource->action); + &resource->action); if (ret) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -10048,46 +11342,69 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused, "cannot create destination array action"); goto error; } - cache_resource->idx = res_idx; - cache_resource->dev = dev; - for (idx = 0; idx < resource->num_of_dest; idx++) + resource->idx = res_idx; + resource->dev = dev; + for (idx = 0; idx < ctx_resource->num_of_dest; idx++) mlx5_free(dest_attr[idx]); - return &cache_resource->entry; + return &resource->entry; error: - for (idx = 0; idx < resource->num_of_dest; idx++) { - struct mlx5_flow_sub_actions_idx *act_res = - &cache_resource->sample_idx[idx]; - if (act_res->rix_hrxq && - !mlx5_hrxq_release(dev, - act_res->rix_hrxq)) - act_res->rix_hrxq = 0; - if (act_res->rix_encap_decap && - !flow_dv_encap_decap_resource_release(dev, - act_res->rix_encap_decap)) - act_res->rix_encap_decap = 0; - if (act_res->rix_port_id_action && - !flow_dv_port_id_action_resource_release(dev, - act_res->rix_port_id_action)) - act_res->rix_port_id_action = 0; - if (act_res->rix_jump && - !flow_dv_jump_tbl_resource_release(dev, - act_res->rix_jump)) - act_res->rix_jump = 0; + for (idx = 0; idx < ctx_resource->num_of_dest; idx++) { + flow_dv_sample_sub_actions_release(dev, + &resource->sample_idx[idx]); if (dest_attr[idx]) mlx5_free(dest_attr[idx]); } - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx); return NULL; } +struct mlx5_list_entry * +flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; + struct mlx5_flow_dv_dest_array_resource *resource; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + uint32_t res_idx = 0; + struct rte_flow_error *error = ctx->error; + + resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY], + &res_idx); + if (!resource) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot allocate dest-array memory"); + return NULL; + } + memcpy(resource, entry, sizeof(*resource)); + resource->idx = res_idx; + resource->dev = dev; + return &resource->entry; +} + +void +flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry) +{ + struct mlx5_flow_dv_dest_array_resource *resource = + container_of(entry, typeof(*resource), entry); + struct rte_eth_dev *dev = resource->dev; + struct mlx5_priv *priv = dev->data->dev_private; + + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx); +} + /** * Find existing destination array resource or create and register a new one. * * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in] resource - * Pointer to destination array resource. + * @param[in] ref + * Pointer to destination array resource reference. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -10098,25 +11415,25 @@ error: */ static int flow_dv_dest_array_resource_register(struct rte_eth_dev *dev, - struct mlx5_flow_dv_dest_array_resource *resource, + struct mlx5_flow_dv_dest_array_resource *ref, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { - struct mlx5_flow_dv_dest_array_resource *cache_resource; + struct mlx5_flow_dv_dest_array_resource *resource; struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_cache_entry *entry; + struct mlx5_list_entry *entry; struct mlx5_flow_cb_ctx ctx = { .dev = dev, .error = error, - .data = resource, + .data = ref, }; - entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx); + entry = mlx5_list_register(priv->sh->dest_array_list, &ctx); if (!entry) return -rte_errno; - cache_resource = container_of(entry, typeof(*cache_resource), entry); - dev_flow->handle->dvh.rix_dest_array = cache_resource->idx; - dev_flow->dv.dest_array_res = cache_resource; + resource = container_of(entry, typeof(*resource), entry); + dev_flow->handle->dvh.rix_dest_array = resource->idx; + dev_flow->dv.dest_array_res = resource; return 0; } @@ -10320,6 +11637,8 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, action_flags |= MLX5_FLOW_ACTION_PORT_ID; break; } + case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: + case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: /* Save the encap resource before sample */ pre_rix = dev_flow->handle->dvh.rix_encap_decap; @@ -10440,6 +11759,7 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, dev_flow->handle->dvh.rix_encap_decap; sample_act->dr_encap_action = dev_flow->dv.encap_decap->action; + dev_flow->handle->dvh.rix_encap_decap = 0; } if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) { normal_idx++; @@ -10447,6 +11767,7 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, dev_flow->handle->rix_port_id_action; sample_act->dr_port_id_action = dev_flow->dv.port_id_action->action; + dev_flow->handle->rix_port_id_action = 0; } if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) { normal_idx++; @@ -10573,7 +11894,7 @@ flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev) mlx5_free(old_pools); } else { /* First ASO flow hit allocation - starting ASO data-path. */ - int ret = mlx5_aso_queue_start(priv->sh); + int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh); if (ret) { mlx5_free(pools); @@ -10714,47 +12035,480 @@ flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error) } /** - * Create a age action using ASO mechanism. + * Initialize flow ASO age parameters. * * @param[in] dev * Pointer to rte_eth_dev structure. - * @param[in] age - * Pointer to the aging action configuration. - * @param[out] error - * Pointer to the error structure. + * @param[in] age_idx + * Index of ASO age action. + * @param[in] context + * Pointer to flow counter age context. + * @param[in] timeout + * Aging timeout in seconds. * - * @return - * Index to flow counter on success, 0 otherwise. */ -static uint32_t -flow_dv_translate_create_aso_age(struct rte_eth_dev *dev, - const struct rte_flow_action_age *age, - struct rte_flow_error *error) +static void +flow_dv_aso_age_params_init(struct rte_eth_dev *dev, + uint32_t age_idx, + void *context, + uint32_t timeout) { - uint32_t age_idx = 0; struct mlx5_aso_age_action *aso_age; - age_idx = flow_dv_aso_age_alloc(dev, error); - if (!age_idx) - return 0; aso_age = flow_aso_age_get_by_idx(dev, age_idx); - aso_age->age_params.context = age->context; - aso_age->age_params.timeout = age->timeout; + MLX5_ASSERT(aso_age); + aso_age->age_params.context = context; + aso_age->age_params.timeout = timeout; aso_age->age_params.port_id = dev->data->port_id; __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0, __ATOMIC_RELAXED); __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE, __ATOMIC_RELAXED); - return age_idx; +} + +static void +flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask, + const struct rte_flow_item_integrity *value, + void *headers_m, void *headers_v) +{ + if (mask->l4_ok) { + /* application l4_ok filter aggregates all hardware l4 filters + * therefore hw l4_checksum_ok must be implicitly added here. + */ + struct rte_flow_item_integrity local_item; + + local_item.l4_csum_ok = 1; + MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok, + local_item.l4_csum_ok); + if (value->l4_ok) { + /* application l4_ok = 1 matches sets both hw flags + * l4_ok and l4_checksum_ok flags to 1. + */ + MLX5_SET(fte_match_set_lyr_2_4, headers_v, + l4_checksum_ok, local_item.l4_csum_ok); + MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok, + mask->l4_ok); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok, + value->l4_ok); + } else { + /* application l4_ok = 0 matches on hw flag + * l4_checksum_ok = 0 only. + */ + MLX5_SET(fte_match_set_lyr_2_4, headers_v, + l4_checksum_ok, 0); + } + } else if (mask->l4_csum_ok) { + MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok, + mask->l4_csum_ok); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok, + value->l4_csum_ok); + } +} + +static void +flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask, + const struct rte_flow_item_integrity *value, + void *headers_m, void *headers_v, + bool is_ipv4) +{ + if (mask->l3_ok) { + /* application l3_ok filter aggregates all hardware l3 filters + * therefore hw ipv4_checksum_ok must be implicitly added here. + */ + struct rte_flow_item_integrity local_item; + + local_item.ipv4_csum_ok = !!is_ipv4; + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok, + local_item.ipv4_csum_ok); + if (value->l3_ok) { + MLX5_SET(fte_match_set_lyr_2_4, headers_v, + ipv4_checksum_ok, local_item.ipv4_csum_ok); + MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok, + mask->l3_ok); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok, + value->l3_ok); + } else { + MLX5_SET(fte_match_set_lyr_2_4, headers_v, + ipv4_checksum_ok, 0); + } + } else if (mask->ipv4_csum_ok) { + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok, + mask->ipv4_csum_ok); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok, + value->ipv4_csum_ok); + } +} + +static void +flow_dv_translate_item_integrity(void *matcher, void *key, + const struct rte_flow_item *head_item, + const struct rte_flow_item *integrity_item) +{ + const struct rte_flow_item_integrity *mask = integrity_item->mask; + const struct rte_flow_item_integrity *value = integrity_item->spec; + const struct rte_flow_item *tunnel_item, *end_item, *item; + void *headers_m; + void *headers_v; + uint32_t l3_protocol; + + if (!value) + return; + if (!mask) + mask = &rte_flow_item_integrity_mask; + if (value->level > 1) { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + inner_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); + } else { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + outer_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); + } + tunnel_item = mlx5_flow_find_tunnel_item(head_item); + if (value->level > 1) { + /* tunnel item was verified during the item validation */ + item = tunnel_item; + end_item = mlx5_find_end_item(tunnel_item); + } else { + item = head_item; + end_item = tunnel_item ? tunnel_item : + mlx5_find_end_item(integrity_item); + } + l3_protocol = mask->l3_ok ? + mlx5_flow_locate_proto_l3(&item, end_item) : 0; + flow_dv_translate_integrity_l3(mask, value, headers_m, headers_v, + l3_protocol == RTE_ETHER_TYPE_IPV4); + flow_dv_translate_integrity_l4(mask, value, headers_m, headers_v); } /** - * Fill the flow with DV spec, lock free - * (mutex should be acquired by caller). + * Prepares DV flow counter with aging configuration. + * Gets it by index when exists, creates a new one when doesn't. * * @param[in] dev * Pointer to rte_eth_dev structure. - * @param[in, out] dev_flow + * @param[in] dev_flow + * Pointer to the mlx5_flow. + * @param[in, out] flow + * Pointer to the sub flow. + * @param[in] count + * Pointer to the counter action configuration. + * @param[in] age + * Pointer to the aging action configuration. + * @param[out] error + * Pointer to the error structure. + * + * @return + * Pointer to the counter, NULL otherwise. + */ +static struct mlx5_flow_counter * +flow_dv_prepare_counter(struct rte_eth_dev *dev, + struct mlx5_flow *dev_flow, + struct rte_flow *flow, + const struct rte_flow_action_count *count, + const struct rte_flow_action_age *age, + struct rte_flow_error *error) +{ + if (!flow->counter) { + flow->counter = flow_dv_translate_create_counter(dev, dev_flow, + count, age); + if (!flow->counter) { + rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "cannot create counter object."); + return NULL; + } + } + return flow_dv_counter_get_by_idx(dev, flow->counter, NULL); +} + +/* + * Release an ASO CT action by its own device. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] idx + * Index of ASO CT action to release. + * + * @return + * 0 when CT action was removed, otherwise the number of references. + */ +static inline int +flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng; + uint32_t ret; + struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx); + enum mlx5_aso_ct_state state = + __atomic_load_n(&ct->state, __ATOMIC_RELAXED); + + /* Cannot release when CT is in the ASO SQ. */ + if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY) + return -1; + ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED); + if (!ret) { + if (ct->dr_action_orig) { +#ifdef HAVE_MLX5_DR_ACTION_ASO_CT + claim_zero(mlx5_glue->destroy_flow_action + (ct->dr_action_orig)); +#endif + ct->dr_action_orig = NULL; + } + if (ct->dr_action_rply) { +#ifdef HAVE_MLX5_DR_ACTION_ASO_CT + claim_zero(mlx5_glue->destroy_flow_action + (ct->dr_action_rply)); +#endif + ct->dr_action_rply = NULL; + } + /* Clear the state to free, no need in 1st allocation. */ + MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE); + rte_spinlock_lock(&mng->ct_sl); + LIST_INSERT_HEAD(&mng->free_cts, ct, next); + rte_spinlock_unlock(&mng->ct_sl); + } + return (int)ret; +} + +static inline int +flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx) +{ + uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx); + uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx); + struct rte_eth_dev *owndev = &rte_eth_devices[owner]; + RTE_SET_USED(dev); + + MLX5_ASSERT(owner < RTE_MAX_ETHPORTS); + if (dev->data->dev_started != 1) + return -1; + return flow_dv_aso_ct_dev_release(owndev, idx); +} + +/* + * Resize the ASO CT pools array by 64 pools. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * 0 on success, otherwise negative errno value and rte_errno is set. + */ +static int +flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng; + void *old_pools = mng->pools; + /* Magic number now, need a macro. */ + uint32_t resize = mng->n + 64; + uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize; + void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); + + if (!pools) { + rte_errno = ENOMEM; + return -rte_errno; + } + rte_rwlock_write_lock(&mng->resize_rwl); + /* ASO SQ/QP was already initialized in the startup. */ + if (old_pools) { + /* Realloc could be an alternative choice. */ + rte_memcpy(pools, old_pools, + mng->n * sizeof(struct mlx5_aso_ct_pool *)); + mlx5_free(old_pools); + } + mng->n = resize; + mng->pools = pools; + rte_rwlock_write_unlock(&mng->resize_rwl); + return 0; +} + +/* + * Create and initialize a new ASO CT pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] ct_free + * Where to put the pointer of a new CT action. + * + * @return + * The CT actions pool pointer and @p ct_free is set on success, + * NULL otherwise and rte_errno is set. + */ +static struct mlx5_aso_ct_pool * +flow_dv_ct_pool_create(struct rte_eth_dev *dev, + struct mlx5_aso_ct_action **ct_free) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng; + struct mlx5_aso_ct_pool *pool = NULL; + struct mlx5_devx_obj *obj = NULL; + uint32_t i; + uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL); + + obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->ctx, + priv->sh->pdn, log_obj_size); + if (!obj) { + rte_errno = ENODATA; + DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX."); + return NULL; + } + pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); + if (!pool) { + rte_errno = ENOMEM; + claim_zero(mlx5_devx_cmd_destroy(obj)); + return NULL; + } + pool->devx_obj = obj; + pool->index = mng->next; + /* Resize pools array if there is no room for the new pool in it. */ + if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) { + claim_zero(mlx5_devx_cmd_destroy(obj)); + mlx5_free(pool); + return NULL; + } + mng->pools[pool->index] = pool; + mng->next++; + /* Assign the first action in the new pool, the rest go to free list. */ + *ct_free = &pool->actions[0]; + /* Lock outside, the list operation is safe here. */ + for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { + /* refcnt is 0 when allocating the memory. */ + pool->actions[i].offset = i; + LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next); + } + return pool; +} + +/* + * Allocate a ASO CT action from free list. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] error + * Pointer to the error structure. + * + * @return + * Index to ASO CT action on success, 0 otherwise and rte_errno is set. + */ +static uint32_t +flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng; + struct mlx5_aso_ct_action *ct = NULL; + struct mlx5_aso_ct_pool *pool; + uint8_t reg_c; + uint32_t ct_idx; + + MLX5_ASSERT(mng); + if (!priv->config.devx) { + rte_errno = ENOTSUP; + return 0; + } + /* Get a free CT action, if no, a new pool will be created. */ + rte_spinlock_lock(&mng->ct_sl); + ct = LIST_FIRST(&mng->free_cts); + if (ct) { + LIST_REMOVE(ct, next); + } else if (!flow_dv_ct_pool_create(dev, &ct)) { + rte_spinlock_unlock(&mng->ct_sl); + rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "failed to create ASO CT pool"); + return 0; + } + rte_spinlock_unlock(&mng->ct_sl); + pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]); + ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset); + /* 0: inactive, 1: created, 2+: used by flows. */ + __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED); + reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error); + if (!ct->dr_action_orig) { +#ifdef HAVE_MLX5_DR_ACTION_ASO_CT + ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso + (priv->sh->rx_domain, pool->devx_obj->obj, + ct->offset, + MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR, + reg_c - REG_C_0); +#else + RTE_SET_USED(reg_c); +#endif + if (!ct->dr_action_orig) { + flow_dv_aso_ct_dev_release(dev, ct_idx); + rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "failed to create ASO CT action"); + return 0; + } + } + if (!ct->dr_action_rply) { +#ifdef HAVE_MLX5_DR_ACTION_ASO_CT + ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso + (priv->sh->rx_domain, pool->devx_obj->obj, + ct->offset, + MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER, + reg_c - REG_C_0); +#endif + if (!ct->dr_action_rply) { + flow_dv_aso_ct_dev_release(dev, ct_idx); + rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "failed to create ASO CT action"); + return 0; + } + } + return ct_idx; +} + +/* + * Create a conntrack object with context and actions by using ASO mechanism. + * + * @param[in] dev + * Pointer to rte_eth_dev structure. + * @param[in] pro + * Pointer to conntrack information profile. + * @param[out] error + * Pointer to the error structure. + * + * @return + * Index to conntrack object on success, 0 otherwise. + */ +static uint32_t +flow_dv_translate_create_conntrack(struct rte_eth_dev *dev, + const struct rte_flow_action_conntrack *pro, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + struct mlx5_aso_ct_action *ct; + uint32_t idx; + + if (!sh->ct_aso_en) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Connection is not supported"); + idx = flow_dv_aso_ct_alloc(dev, error); + if (!idx) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Failed to allocate CT object"); + ct = flow_aso_ct_get_by_dev_idx(dev, idx); + if (mlx5_aso_ct_update_by_wqe(sh, ct, pro)) + return rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Failed to update CT"); + ct->is_original = !!pro->is_original_dir; + ct->peer = pro->peer_port; + return idx; +} + +/** + * Fill the flow with DV spec, lock free + * (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to rte_eth_dev structure. + * @param[in, out] dev_flow * Pointer to the sub flow. * @param[in] attr * Pointer to the flow attributes. @@ -10787,8 +12541,7 @@ flow_dv_translate(struct rte_eth_dev *dev, uint64_t action_flags = 0; struct mlx5_flow_dv_matcher matcher = { .mask = { - .size = sizeof(matcher.mask.buf) - - MLX5_ST_SZ_BYTES(fte_match_set_misc4), + .size = sizeof(matcher.mask.buf), }, }; int actions_n = 0; @@ -10801,7 +12554,7 @@ flow_dv_translate(struct rte_eth_dev *dev, } mhdr_dummy; struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; const struct rte_flow_action_count *count = NULL; - const struct rte_flow_action_age *age = NULL; + const struct rte_flow_action_age *non_shared_age = NULL; union flow_dv_attr flow_attr = { .attr = 0 }; uint32_t tag_be; union mlx5_flow_tbl_key tbl_key; @@ -10816,18 +12569,21 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_sample *sample = NULL; struct mlx5_flow_sub_actions_list *sample_act; uint32_t sample_act_pos = UINT32_MAX; + uint32_t age_act_pos = UINT32_MAX; uint32_t num_of_dest = 0; int tmp_actions_n = 0; uint32_t table; int ret = 0; - const struct mlx5_flow_tunnel *tunnel; + const struct mlx5_flow_tunnel *tunnel = NULL; struct flow_grp_info grp_info = { .external = !!dev_flow->external, .transfer = !!attr->transfer, .fdb_def_rule = !!priv->fdb_def_rule, .skip_scale = dev_flow->skip_scale & (1 << MLX5_SCALE_FLOW_GROUP_BIT), + .std_tbl_fix = true, }; + const struct rte_flow_item *head_item = items; if (!wks) return rte_flow_error_set(error, ENOMEM, @@ -10841,15 +12597,21 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5DV_FLOW_TABLE_TYPE_NIC_RX; /* update normal path action resource into last index of array */ sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1]; - tunnel = is_flow_tunnel_match_rule(dev, attr, items, actions) ? - flow_items_to_tunnel(items) : - is_flow_tunnel_steer_rule(dev, attr, items, actions) ? - flow_actions_to_tunnel(actions) : - dev_flow->tunnel ? dev_flow->tunnel : NULL; + if (is_tunnel_offload_active(dev)) { + if (dev_flow->tunnel) { + RTE_VERIFY(dev_flow->tof_type == + MLX5_TUNNEL_OFFLOAD_MISS_RULE); + tunnel = dev_flow->tunnel; + } else { + tunnel = mlx5_get_tof(items, actions, + &dev_flow->tof_type); + dev_flow->tunnel = tunnel; + } + grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate + (dev, attr, tunnel, dev_flow->tof_type); + } mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : MLX5DV_FLOW_TABLE_TYPE_NIC_RX; - grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate - (dev, tunnel, attr, items, actions); ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table, &grp_info, error); if (ret) @@ -10859,7 +12621,7 @@ flow_dv_translate(struct rte_eth_dev *dev, mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; /* number of actions must be set to 0 in case of dirty stack. */ mhdr_res->actions_num = 0; - if (is_flow_tunnel_match_rule(dev, attr, items, actions)) { + if (is_flow_tunnel_match_rule(dev_flow->tof_type)) { /* * do not add decap action if match rule drops packet * HW rejects rules with decap & drop @@ -10893,15 +12655,16 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; + struct mlx5_flow_counter *cnt_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; + uint32_t owner_idx; + struct mlx5_aso_ct_action *ct; if (!mlx5_flow_os_action_supported(action_type)) return rte_flow_error_set(error, ENOTSUP, @@ -11039,34 +12802,23 @@ flow_dv_translate(struct rte_eth_dev *dev, age_act = flow_aso_age_get_by_idx(dev, flow->age); __atomic_fetch_add(&age_act->refcnt, 1, __ATOMIC_RELAXED); - dev_flow->dv.actions[actions_n++] = age_act->dr_action; + age_act_pos = actions_n++; action_flags |= MLX5_FLOW_ACTION_AGE; break; case RTE_FLOW_ACTION_TYPE_AGE: - if (priv->sh->flow_hit_aso_en && attr->group) { - /* - * Create one shared age action, to be used - * by all sub-flows. - */ - if (!flow->age) { - flow->age = - flow_dv_translate_create_aso_age - (dev, action->conf, - error); - if (!flow->age) - return rte_flow_error_set - (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "can't create ASO age action"); - } - dev_flow->dv.actions[actions_n++] = - (flow_aso_age_get_by_idx - (dev, flow->age))->dr_action; - action_flags |= MLX5_FLOW_ACTION_AGE; - break; - } - /* Fall-through */ + non_shared_age = action->conf; + age_act_pos = actions_n++; + action_flags |= MLX5_FLOW_ACTION_AGE; + break; + case MLX5_RTE_FLOW_ACTION_TYPE_COUNT: + flow->counter = (uint32_t)(uintptr_t)(action->conf); + cnt_act = flow_dv_counter_get_by_idx(dev, flow->counter, + NULL); + __atomic_fetch_add(&cnt_act->shared_info.refcnt, 1, + __ATOMIC_RELAXED); + /* Save information first, will apply later. */ + action_flags |= MLX5_FLOW_ACTION_COUNT; + break; case RTE_FLOW_ACTION_TYPE_COUNT: if (!dev_conf->devx) { return rte_flow_error_set @@ -11076,10 +12828,7 @@ flow_dv_translate(struct rte_eth_dev *dev, "count action not supported"); } /* Save information first, will apply later. */ - if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT) - count = action->conf; - else - age = action->conf; + count = action->conf; action_flags |= MLX5_FLOW_ACTION_COUNT; break; case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: @@ -11186,6 +12935,11 @@ flow_dv_translate(struct rte_eth_dev *dev, /* If decap is followed by encap, handle it at encap. */ action_flags |= MLX5_FLOW_ACTION_DECAP; break; + case MLX5_RTE_FLOW_ACTION_TYPE_JUMP: + dev_flow->dv.actions[actions_n++] = + (void *)(uintptr_t)action->conf; + action_flags |= MLX5_FLOW_ACTION_JUMP; + break; case RTE_FLOW_ACTION_TYPE_JUMP: jump_group = ((const struct rte_flow_action_jump *) action->conf)->group; @@ -11205,7 +12959,7 @@ flow_dv_translate(struct rte_eth_dev *dev, attr->transfer, !!dev_flow->external, tunnel, jump_group, 0, - error); + 0, error); if (!tbl) return rte_flow_error_set (error, errno, @@ -11325,33 +13079,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -11384,6 +13118,31 @@ flow_dv_translate(struct rte_eth_dev *dev, return -rte_errno; action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD; break; + case RTE_FLOW_ACTION_TYPE_CONNTRACK: + owner_idx = (uint32_t)(uintptr_t)action->conf; + ct = flow_aso_ct_get_by_idx(dev, owner_idx); + if (!ct) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "Failed to get CT object."); + if (mlx5_aso_ct_available(priv->sh, ct)) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "CT is unavailable."); + if (ct->is_original) + dev_flow->dv.actions[actions_n] = + ct->dr_action_orig; + else + dev_flow->dv.actions[actions_n] = + ct->dr_action_rply; + flow->indirect_type = MLX5_INDIRECT_ACTION_TYPE_CT; + flow->ct = owner_idx; + __atomic_fetch_add(&ct->refcnt, 1, __ATOMIC_RELAXED); + actions_n++; + action_flags |= MLX5_FLOW_ACTION_CT; + break; case RTE_FLOW_ACTION_TYPE_END: actions_end = true; if (mhdr_res->actions_num) { @@ -11394,27 +13153,57 @@ flow_dv_translate(struct rte_eth_dev *dev, dev_flow->dv.actions[modify_action_position] = handle->dvh.modify_hdr->action; } + /* + * Handle AGE and COUNT action by single HW counter + * when they are not shared. + */ + if (action_flags & MLX5_FLOW_ACTION_AGE) { + if ((non_shared_age && + count && !count->shared) || + !(priv->sh->flow_hit_aso_en && + (attr->group || attr->transfer))) { + /* Creates age by counters. */ + cnt_act = flow_dv_prepare_counter + (dev, dev_flow, + flow, count, + non_shared_age, + error); + if (!cnt_act) + return -rte_errno; + dev_flow->dv.actions[age_act_pos] = + cnt_act->action; + break; + } + if (!flow->age && non_shared_age) { + flow->age = flow_dv_aso_age_alloc + (dev, error); + if (!flow->age) + return -rte_errno; + flow_dv_aso_age_params_init + (dev, flow->age, + non_shared_age->context ? + non_shared_age->context : + (void *)(uintptr_t) + (dev_flow->flow_idx), + non_shared_age->timeout); + } + age_act = flow_aso_age_get_by_idx(dev, + flow->age); + dev_flow->dv.actions[age_act_pos] = + age_act->dr_action; + } if (action_flags & MLX5_FLOW_ACTION_COUNT) { /* * Create one count action, to be used * by all sub-flows. */ - if (!flow->counter) { - flow->counter = - flow_dv_translate_create_counter - (dev, dev_flow, count, - age); - if (!flow->counter) - return rte_flow_error_set - (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, "cannot create counter" - " object."); - } - dev_flow->dv.actions[actions_n] = - (flow_dv_counter_get_by_idx(dev, - flow->counter, NULL))->action; - actions_n++; + cnt_act = flow_dv_prepare_counter(dev, dev_flow, + flow, count, + NULL, error); + if (!cnt_act) + return -rte_errno; + dev_flow->dv.actions[actions_n++] = + cnt_act->action; } default: break; @@ -11559,7 +13348,8 @@ flow_dv_translate(struct rte_eth_dev *dev, last_item = MLX5_FLOW_LAYER_GRE; break; case RTE_FLOW_ITEM_TYPE_VXLAN: - flow_dv_translate_item_vxlan(match_mask, match_value, + flow_dv_translate_item_vxlan(dev, attr, + match_mask, match_value, items, tunnel); matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); last_item = MLX5_FLOW_LAYER_VXLAN; @@ -11657,15 +13447,20 @@ flow_dv_translate(struct rte_eth_dev *dev, NULL, "cannot create eCPRI parser"); } - /* Adjust the length matcher and device flow value. */ - matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param); - dev_flow->dv.value.size = - MLX5_ST_SZ_BYTES(fte_match_param); flow_dv_translate_item_ecpri(dev, match_mask, match_value, items); /* No other protocol should follow eCPRI layer. */ last_item = MLX5_FLOW_LAYER_ECPRI; break; + case RTE_FLOW_ITEM_TYPE_INTEGRITY: + flow_dv_translate_item_integrity(match_mask, + match_value, + head_item, items); + break; + case RTE_FLOW_ITEM_TYPE_CONNTRACK: + flow_dv_translate_item_aso_ct(dev, match_mask, + match_value, items); + break; default: break; } @@ -11761,15 +13556,27 @@ flow_dv_translate(struct rte_eth_dev *dev, } dev_flow->dv.actions_n = actions_n; dev_flow->act_flags = action_flags; + if (wks->skip_matcher_reg) + return 0; /* Register matcher. */ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, matcher.mask.size); matcher.priority = mlx5_get_matcher_priority(dev, attr, matcher.priority); + /** + * When creating meter drop flow in drop table, using original + * 5-tuple match, the matcher priority should be lower than + * mtr_id matcher. + */ + if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER && + dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP && + matcher.priority <= MLX5_REG_BITS) + matcher.priority += MLX5_REG_BITS; /* reserved field no needs to be set to 0 here. */ - tbl_key.domain = attr->transfer; - tbl_key.direction = attr->egress; - tbl_key.table_id = dev_flow->dv.group; + tbl_key.is_fdb = attr->transfer; + tbl_key.is_egress = attr->egress; + tbl_key.level = dev_flow->dv.group; + tbl_key.id = dev_flow->dv.table_id; if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, tunnel, attr->group, error)) return -rte_errno; @@ -11795,28 +13602,51 @@ flow_dv_translate(struct rte_eth_dev *dev, static int __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, const uint64_t hash_fields, - const int tunnel, uint32_t hrxq_idx) { - uint32_t *hrxqs = tunnel ? action->hrxq : action->hrxq_tunnel; + uint32_t *hrxqs = action->hrxq; switch (hash_fields & ~IBV_RX_HASH_INNER) { case MLX5_RSS_HASH_IPV4: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_SRC_ONLY: hrxqs[0] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV4_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY: hrxqs[1] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV4_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY: hrxqs[2] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV6: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_SRC_ONLY: hrxqs[3] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV6_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY: hrxqs[4] = hrxq_idx; return 0; case MLX5_RSS_HASH_IPV6_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY: hrxqs[5] = hrxq_idx; return 0; case MLX5_RSS_HASH_NONE: @@ -11845,33 +13675,56 @@ __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, */ static uint32_t __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx, - const uint64_t hash_fields, - const int tunnel) + const uint64_t hash_fields) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_shared_action_rss *shared_rss = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); - const uint32_t *hrxqs = tunnel ? shared_rss->hrxq : - shared_rss->hrxq_tunnel; + const uint32_t *hrxqs = shared_rss->hrxq; switch (hash_fields & ~IBV_RX_HASH_INNER) { case MLX5_RSS_HASH_IPV4: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_SRC_ONLY: return hrxqs[0]; case MLX5_RSS_HASH_IPV4_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY: return hrxqs[1]; case MLX5_RSS_HASH_IPV4_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY: return hrxqs[2]; case MLX5_RSS_HASH_IPV6: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_SRC_ONLY: return hrxqs[3]; case MLX5_RSS_HASH_IPV6_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY: return hrxqs[4]; case MLX5_RSS_HASH_IPV6_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY: return hrxqs[5]; case MLX5_RSS_HASH_NONE: return hrxqs[6]; default: return 0; } + } /** @@ -11903,6 +13756,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, int idx; struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc; + uint8_t misc_mask; MLX5_ASSERT(wks); for (idx = wks->flow_idx - 1; idx >= 0; idx--) { @@ -11913,11 +13767,19 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, n = dv->actions_n; if (dh->fate_action == MLX5_FLOW_FATE_DROP) { if (dv->transfer) { - dv->actions[n++] = priv->sh->esw_drop_action; + MLX5_ASSERT(priv->sh->dr_drop_action); + dv->actions[n++] = priv->sh->dr_drop_action; } else { +#ifdef HAVE_MLX5DV_DR + /* DR supports drop action placeholder. */ + MLX5_ASSERT(priv->sh->dr_drop_action); + dv->actions[n++] = priv->sh->dr_drop_action; +#else + /* For DV we use the explicit drop queue. */ MLX5_ASSERT(priv->drop_queue.hrxq); dv->actions[n++] = priv->drop_queue.hrxq->action; +#endif } } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE && !dv_h->rix_sample && !dv_h->rix_dest_array)) { @@ -11941,9 +13803,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev, rss_desc->shared_rss, - dev_flow->hash_fields, - !!(dh->layers & - MLX5_FLOW_LAYER_TUNNEL)); + dev_flow->hash_fields); if (hrxq_idx) hrxq = mlx5_ipool_get (priv->sh->ipool[MLX5_IPOOL_HRXQ], @@ -11967,14 +13827,20 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, } dv->actions[n++] = priv->sh->default_miss_action; } + misc_mask = flow_dv_matcher_enable(dv->value.buf); + __flow_dv_adjust_buf_size(&dv->value.size, misc_mask); err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object, (void *)&dv->value, n, dv->actions, &dh->drv_flow); if (err) { - rte_flow_error_set(error, errno, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "hardware refuses to create flow"); + rte_flow_error_set + (error, errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + (!priv->config.allow_duplicate_pattern && + errno == EEXIST) ? + "duplicating pattern is not allowed" : + "hardware refuses to create flow"); goto error; } if (priv->vmwa_context && @@ -12008,14 +13874,15 @@ error: } void -flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry) +flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry) { - struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache), - entry); + struct mlx5_flow_dv_matcher *resource = container_of(entry, + typeof(*resource), + entry); - claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object)); - mlx5_free(cache); + claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object)); + mlx5_free(resource); } /** @@ -12039,26 +13906,17 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, int ret; MLX5_ASSERT(matcher->matcher_object); - ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry); + ret = mlx5_list_unregister(tbl->matchers, &matcher->entry); flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl); return ret; } -/** - * Release encap_decap resource. - * - * @param list - * Pointer to the hash list. - * @param entry - * Pointer to exist resource entry object. - */ void -flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list, - struct mlx5_hlist_entry *entry) +flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) { - struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_dev_ctx_shared *sh = tool_ctx; struct mlx5_flow_dv_encap_decap_resource *res = - container_of(entry, typeof(*res), entry); + container_of(entry, typeof(*res), entry); claim_zero(mlx5_flow_os_destroy_flow_action(res->action)); mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx); @@ -12080,15 +13938,14 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, uint32_t encap_decap_idx) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_encap_decap_resource *cache_resource; + struct mlx5_flow_dv_encap_decap_resource *resource; - cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], - encap_decap_idx); - if (!cache_resource) + resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], + encap_decap_idx); + if (!resource) return 0; - MLX5_ASSERT(cache_resource->action); - return mlx5_hlist_unregister(priv->sh->encaps_decaps, - &cache_resource->entry); + MLX5_ASSERT(resource->action); + return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry); } /** @@ -12117,14 +13974,14 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, } void -flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused, - struct mlx5_hlist_entry *entry) +flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) { struct mlx5_flow_dv_modify_hdr_resource *res = container_of(entry, typeof(*res), entry); + struct mlx5_dev_ctx_shared *sh = tool_ctx; claim_zero(mlx5_flow_os_destroy_flow_action(res->action)); - mlx5_free(entry); + mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx); } /** @@ -12150,15 +14007,14 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev, } void -flow_dv_port_id_remove_cb(struct mlx5_cache_list *list, - struct mlx5_cache_entry *entry) +flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) { - struct mlx5_dev_ctx_shared *sh = list->ctx; - struct mlx5_flow_dv_port_id_action_resource *cache = - container_of(entry, typeof(*cache), entry); + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_dv_port_id_action_resource *resource = + container_of(entry, typeof(*resource), entry); - claim_zero(mlx5_flow_os_destroy_flow_action(cache->action)); - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx); + claim_zero(mlx5_flow_os_destroy_flow_action(resource->action)); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx); } /** @@ -12177,14 +14033,14 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, uint32_t port_id) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_port_id_action_resource *cache; + struct mlx5_flow_dv_port_id_action_resource *resource; - cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id); - if (!cache) + resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id); + if (!resource) return 0; - MLX5_ASSERT(cache->action); - return mlx5_cache_unregister(&priv->sh->port_id_action_list, - &cache->entry); + MLX5_ASSERT(resource->action); + return mlx5_list_unregister(priv->sh->port_id_action_list, + &resource->entry); } /** @@ -12207,15 +14063,14 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss) } void -flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list, - struct mlx5_cache_entry *entry) +flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) { - struct mlx5_dev_ctx_shared *sh = list->ctx; - struct mlx5_flow_dv_push_vlan_action_resource *cache = - container_of(entry, typeof(*cache), entry); + struct mlx5_dev_ctx_shared *sh = tool_ctx; + struct mlx5_flow_dv_push_vlan_action_resource *resource = + container_of(entry, typeof(*resource), entry); - claim_zero(mlx5_flow_os_destroy_flow_action(cache->action)); - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx); + claim_zero(mlx5_flow_os_destroy_flow_action(resource->action)); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx); } /** @@ -12234,15 +14089,15 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_push_vlan_action_resource *cache; + struct mlx5_flow_dv_push_vlan_action_resource *resource; uint32_t idx = handle->dvh.rix_push_vlan; - cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); - if (!cache) + resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); + if (!resource) return 0; - MLX5_ASSERT(cache->action); - return mlx5_cache_unregister(&priv->sh->push_vlan_action_list, - &cache->entry); + MLX5_ASSERT(resource->action); + return mlx5_list_unregister(priv->sh->push_vlan_action_list, + &resource->entry); } /** @@ -12261,7 +14116,8 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev, return; switch (handle->fate_action) { case MLX5_FLOW_FATE_QUEUE: - mlx5_hrxq_release(dev, handle->rix_hrxq); + if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array) + mlx5_hrxq_release(dev, handle->rix_hrxq); break; case MLX5_FLOW_FATE_JUMP: flow_dv_jump_tbl_resource_release(dev, handle->rix_jump); @@ -12278,26 +14134,24 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev, } void -flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry) +flow_dv_sample_remove_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry) { - struct mlx5_flow_dv_sample_resource *cache_resource = - container_of(entry, typeof(*cache_resource), entry); - struct rte_eth_dev *dev = cache_resource->dev; + struct mlx5_flow_dv_sample_resource *resource = container_of(entry, + typeof(*resource), + entry); + struct rte_eth_dev *dev = resource->dev; struct mlx5_priv *priv = dev->data->dev_private; - if (cache_resource->verbs_action) + if (resource->verbs_action) claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->verbs_action)); - if (cache_resource->normal_path_tbl) + (resource->verbs_action)); + if (resource->normal_path_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), - cache_resource->normal_path_tbl); - flow_dv_sample_sub_actions_release(dev, - &cache_resource->sample_idx); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], - cache_resource->idx); - DRV_LOG(DEBUG, "sample resource %p: removed", - (void *)cache_resource); + resource->normal_path_tbl); + flow_dv_sample_sub_actions_release(dev, &resource->sample_idx); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx); + DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource); } /** @@ -12316,38 +14170,36 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_sample_resource *cache_resource; + struct mlx5_flow_dv_sample_resource *resource; - cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE], - handle->dvh.rix_sample); - if (!cache_resource) + resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE], + handle->dvh.rix_sample); + if (!resource) return 0; - MLX5_ASSERT(cache_resource->verbs_action); - return mlx5_cache_unregister(&priv->sh->sample_action_list, - &cache_resource->entry); + MLX5_ASSERT(resource->verbs_action); + return mlx5_list_unregister(priv->sh->sample_action_list, + &resource->entry); } void -flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused, - struct mlx5_cache_entry *entry) +flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry) { - struct mlx5_flow_dv_dest_array_resource *cache_resource = - container_of(entry, typeof(*cache_resource), entry); - struct rte_eth_dev *dev = cache_resource->dev; + struct mlx5_flow_dv_dest_array_resource *resource = + container_of(entry, typeof(*resource), entry); + struct rte_eth_dev *dev = resource->dev; struct mlx5_priv *priv = dev->data->dev_private; uint32_t i = 0; - MLX5_ASSERT(cache_resource->action); - if (cache_resource->action) - claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->action)); - for (; i < cache_resource->num_of_dest; i++) + MLX5_ASSERT(resource->action); + if (resource->action) + claim_zero(mlx5_flow_os_destroy_flow_action(resource->action)); + for (; i < resource->num_of_dest; i++) flow_dv_sample_sub_actions_release(dev, - &cache_resource->sample_idx[i]); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], - cache_resource->idx); + &resource->sample_idx[i]); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx); DRV_LOG(DEBUG, "destination array resource %p: removed", - (void *)cache_resource); + (void *)resource); } /** @@ -12366,15 +14218,15 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_dest_array_resource *cache; + struct mlx5_flow_dv_dest_array_resource *resource; - cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], - handle->dvh.rix_dest_array); - if (!cache) + resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], + handle->dvh.rix_dest_array); + if (!resource) return 0; - MLX5_ASSERT(cache->action); - return mlx5_cache_unregister(&priv->sh->dest_array_list, - &cache->entry); + MLX5_ASSERT(resource->action); + return mlx5_list_unregister(priv->sh->dest_array_list, + &resource->entry); } static void @@ -12447,6 +14299,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter_info *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12457,15 +14310,15 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], - flow->meter); + fm = flow_dv_meter_find_by_idx(priv, flow->meter); if (fm) - mlx5_flow_meter_detach(fm); + mlx5_flow_meter_detach(priv, fm); flow->meter = 0; } - if (flow->age) + /* Keep the current age handling by default. */ + if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct) + flow_dv_aso_ct_release(dev, flow->ct); + else if (flow->age) flow_dv_aso_age_release(dev, flow->age); if (flow->geneve_tlv_option) { flow_dv_geneve_tlv_option_resource_release(dev); @@ -12500,6 +14353,15 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); + else if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) + mlx5_ipool_free(priv->sh->ipool + [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -12554,8 +14416,85 @@ static int __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, struct mlx5_shared_action_rss *shared_rss) { - return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq) + - __flow_dv_hrxqs_release(dev, &shared_rss->hrxq_tunnel); + return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq); +} + +/** + * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to + * user input. + * + * Only one hash value is available for one L3+L4 combination: + * for example: + * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and + * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share + * same slot in mlx5_rss_hash_fields. + * + * @param[in] rss + * Pointer to the shared action RSS conf. + * @param[in, out] hash_field + * hash_field variable needed to be adjusted. + * + * @return + * void + */ +static void +__flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss, + uint64_t *hash_field) +{ + uint64_t rss_types = rss->origin.types; + + switch (*hash_field & ~IBV_RX_HASH_INNER) { + case MLX5_RSS_HASH_IPV4: + if (rss_types & MLX5_IPV4_LAYER_TYPES) { + *hash_field &= ~MLX5_RSS_HASH_IPV4; + if (rss_types & ETH_RSS_L3_DST_ONLY) + *hash_field |= IBV_RX_HASH_DST_IPV4; + else if (rss_types & ETH_RSS_L3_SRC_ONLY) + *hash_field |= IBV_RX_HASH_SRC_IPV4; + else + *hash_field |= MLX5_RSS_HASH_IPV4; + } + return; + case MLX5_RSS_HASH_IPV6: + if (rss_types & MLX5_IPV6_LAYER_TYPES) { + *hash_field &= ~MLX5_RSS_HASH_IPV6; + if (rss_types & ETH_RSS_L3_DST_ONLY) + *hash_field |= IBV_RX_HASH_DST_IPV6; + else if (rss_types & ETH_RSS_L3_SRC_ONLY) + *hash_field |= IBV_RX_HASH_SRC_IPV6; + else + *hash_field |= MLX5_RSS_HASH_IPV6; + } + return; + case MLX5_RSS_HASH_IPV4_UDP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_UDP: + if (rss_types & ETH_RSS_UDP) { + *hash_field &= ~MLX5_UDP_IBV_RX_HASH; + if (rss_types & ETH_RSS_L4_DST_ONLY) + *hash_field |= IBV_RX_HASH_DST_PORT_UDP; + else if (rss_types & ETH_RSS_L4_SRC_ONLY) + *hash_field |= IBV_RX_HASH_SRC_PORT_UDP; + else + *hash_field |= MLX5_UDP_IBV_RX_HASH; + } + return; + case MLX5_RSS_HASH_IPV4_TCP: + /* fall-through. */ + case MLX5_RSS_HASH_IPV6_TCP: + if (rss_types & ETH_RSS_TCP) { + *hash_field &= ~MLX5_TCP_IBV_RX_HASH; + if (rss_types & ETH_RSS_L4_DST_ONLY) + *hash_field |= IBV_RX_HASH_DST_PORT_TCP; + else if (rss_types & ETH_RSS_L4_SRC_ONLY) + *hash_field |= IBV_RX_HASH_SRC_PORT_TCP; + else + *hash_field |= MLX5_TCP_IBV_RX_HASH; + } + return; + default: + return; + } } /** @@ -12601,23 +14540,26 @@ __flow_dv_action_rss_setup(struct rte_eth_dev *dev, for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) { uint32_t hrxq_idx; uint64_t hash_fields = mlx5_rss_hash_fields[i]; - int tunnel; + int tunnel = 0; - for (tunnel = 0; tunnel < 2; tunnel++) { - rss_desc.tunnel = tunnel; - rss_desc.hash_fields = hash_fields; - hrxq_idx = mlx5_hrxq_get(dev, &rss_desc); - if (!hrxq_idx) { - rte_flow_error_set - (error, rte_errno, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot get hash queue"); - goto error_hrxq_new; - } - err = __flow_dv_action_rss_hrxq_set - (shared_rss, hash_fields, tunnel, hrxq_idx); - MLX5_ASSERT(!err); + __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields); + if (shared_rss->origin.level > 1) { + hash_fields |= IBV_RX_HASH_INNER; + tunnel = 1; + } + rss_desc.tunnel = tunnel; + rss_desc.hash_fields = hash_fields; + hrxq_idx = mlx5_hrxq_get(dev, &rss_desc); + if (!hrxq_idx) { + rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot get hash queue"); + goto error_hrxq_new; } + err = __flow_dv_action_rss_hrxq_set + (shared_rss, hash_fields, hrxq_idx); + MLX5_ASSERT(!err); } return 0; error_hrxq_new: @@ -12648,7 +14590,7 @@ error_hrxq_new: */ static uint32_t __flow_dv_action_rss_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action_rss *rss, struct rte_flow_error *error) { @@ -12671,7 +14613,7 @@ __flow_dv_action_rss_create(struct rte_eth_dev *dev, "cannot allocate resource memory"); goto error_rss_init; } - if (idx > (1u << MLX5_SHARED_ACTION_TYPE_OFFSET)) { + if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) { rte_flow_error_set(error, E2BIG, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "rss action number out of range"); @@ -12784,7 +14726,7 @@ __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, } /** - * Create shared action, lock free, + * Create indirect action, lock free, * (mutex should be acquired by caller). * Dispatcher for action type specific call. * @@ -12793,7 +14735,7 @@ __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, * @param[in] conf * Shared action configuration. * @param[in] action - * Action specification used to create shared action. + * Action specification used to create indirect action. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -12802,52 +14744,69 @@ __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, * A valid shared action handle in case of success, NULL otherwise and * rte_errno is set. */ -static struct rte_flow_shared_action * +static struct rte_flow_action_handle * flow_dv_action_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action *action, struct rte_flow_error *err) { + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t age_idx = 0; uint32_t idx = 0; uint32_t ret = 0; switch (action->type) { case RTE_FLOW_ACTION_TYPE_RSS: ret = __flow_dv_action_rss_create(dev, conf, action->conf, err); - idx = (MLX5_SHARED_ACTION_TYPE_RSS << - MLX5_SHARED_ACTION_TYPE_OFFSET) | ret; + idx = (MLX5_INDIRECT_ACTION_TYPE_RSS << + MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret; break; case RTE_FLOW_ACTION_TYPE_AGE: - ret = flow_dv_translate_create_aso_age(dev, action->conf, err); - idx = (MLX5_SHARED_ACTION_TYPE_AGE << - MLX5_SHARED_ACTION_TYPE_OFFSET) | ret; - if (ret) { - struct mlx5_aso_age_action *aso_age = - flow_aso_age_get_by_idx(dev, ret); - - if (!aso_age->age_params.context) - aso_age->age_params.context = - (void *)(uintptr_t)idx; + age_idx = flow_dv_aso_age_alloc(dev, err); + if (!age_idx) { + ret = -rte_errno; + break; } + idx = (MLX5_INDIRECT_ACTION_TYPE_AGE << + MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx; + flow_dv_aso_age_params_init(dev, age_idx, + ((const struct rte_flow_action_age *) + action->conf)->context ? + ((const struct rte_flow_action_age *) + action->conf)->context : + (void *)(uintptr_t)idx, + ((const struct rte_flow_action_age *) + action->conf)->timeout); + ret = age_idx; + break; + case RTE_FLOW_ACTION_TYPE_COUNT: + ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL); + idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT << + MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret; + break; + case RTE_FLOW_ACTION_TYPE_CONNTRACK: + ret = flow_dv_translate_create_conntrack(dev, action->conf, + err); + idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret); break; default: rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "action type not supported"); break; } - return ret ? (struct rte_flow_shared_action *)(uintptr_t)idx : NULL; + return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL; } /** - * Destroy the shared action. + * Destroy the indirect action. * Release action related resources on the NIC and the memory. * Lock free, (mutex should be acquired by caller). * Dispatcher for action type specific call. * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] action - * The shared action object to be removed. + * @param[in] handle + * The indirect action object handle to be removed. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -12857,27 +14816,49 @@ flow_dv_action_create(struct rte_eth_dev *dev, */ static int flow_dv_action_destroy(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, + struct rte_flow_action_handle *handle, struct rte_flow_error *error) { - uint32_t act_idx = (uint32_t)(uintptr_t)action; - uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; - uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + uint32_t act_idx = (uint32_t)(uintptr_t)handle; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); + struct mlx5_flow_counter *cnt; + uint32_t no_flow_refcnt = 1; int ret; switch (type) { - case MLX5_SHARED_ACTION_TYPE_RSS: + case MLX5_INDIRECT_ACTION_TYPE_RSS: return __flow_dv_action_rss_release(dev, idx, error); - case MLX5_SHARED_ACTION_TYPE_AGE: + case MLX5_INDIRECT_ACTION_TYPE_COUNT: + cnt = flow_dv_counter_get_by_idx(dev, idx, NULL); + if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt, + &no_flow_refcnt, 1, false, + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED)) + return rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "Indirect count action has references"); + flow_dv_counter_free(dev, idx); + return 0; + case MLX5_INDIRECT_ACTION_TYPE_AGE: ret = flow_dv_aso_age_release(dev, idx); if (ret) /* * In this case, the last flow has a reference will * actually release the age action. */ - DRV_LOG(DEBUG, "Shared age action %" PRIu32 " was" + DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was" " released with references %d.", idx, ret); return 0; + case MLX5_INDIRECT_ACTION_TYPE_CT: + ret = flow_dv_aso_ct_release(dev, idx); + if (ret < 0) + return ret; + if (ret > 0) + DRV_LOG(DEBUG, "Connection tracking object %u still " + "has references %d.", idx, ret); + return 0; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -12920,6 +14901,10 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "invalid shared action to update"); + if (priv->obj_ops.ind_table_modify == NULL) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "cannot modify indirection table"); queue = mlx5_malloc(MLX5_MEM_ZERO, RTE_ALIGN_CEIL(queue_size, sizeof(void *)), 0, SOCKET_ID_ANY); @@ -12948,18 +14933,85 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx, return ret; } +/* + * Updates in place conntrack context or direction. + * Context update should be synchronized. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] idx + * The conntrack object ID to be updated. + * @param[in] update + * Pointer to the structure of information to update. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. + * + * @return + * 0 on success, otherwise negative errno value. + */ +static int +__flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx, + const struct rte_flow_modify_conntrack *update, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_ct_action *ct; + const struct rte_flow_action_conntrack *new_prf; + int ret = 0; + uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx); + uint32_t dev_idx; + + if (PORT_ID(priv) != owner) + return rte_flow_error_set(error, EACCES, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "CT object owned by another port"); + dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx); + ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx); + if (!ct->refcnt) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "CT object is inactive"); + new_prf = &update->new_ct; + if (update->direction) + ct->is_original = !!new_prf->is_original_dir; + if (update->state) { + /* Only validate the profile when it needs to be updated. */ + ret = mlx5_validate_action_ct(dev, new_prf, error); + if (ret) + return ret; + ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf); + if (ret) + return rte_flow_error_set(error, EIO, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Failed to send CT context update WQE"); + /* Block until ready or a failure. */ + ret = mlx5_aso_ct_available(priv->sh, ct); + if (ret) + rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Timeout to get the CT update"); + } + return ret; +} + /** * Updates in place shared action configuration, lock free, * (mutex should be acquired by caller). * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] action - * The shared action object to be updated. - * @param[in] action_conf - * Action specification used to modify *action*. - * *action_conf* should be of type correlating with type of the *action*, - * otherwise considered as invalid. + * @param[in] handle + * The indirect action object handle to be updated. + * @param[in] update + * Action specification used to modify the action pointed by *handle*. + * *update* could be of same type with the action pointed by the *handle* + * handle argument, or some other structures like a wrapper, depending on + * the indirect action type. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -12969,17 +15021,21 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx, */ static int flow_dv_action_update(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - const void *action_conf, + struct rte_flow_action_handle *handle, + const void *update, struct rte_flow_error *err) { - uint32_t act_idx = (uint32_t)(uintptr_t)action; - uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; - uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + uint32_t act_idx = (uint32_t)(uintptr_t)handle; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); + const void *action_conf; switch (type) { - case MLX5_SHARED_ACTION_TYPE_RSS: + case MLX5_INDIRECT_ACTION_TYPE_RSS: + action_conf = ((const struct rte_flow_action *)update)->conf; return __flow_dv_action_rss_update(dev, idx, action_conf, err); + case MLX5_INDIRECT_ACTION_TYPE_CT: + return __flow_dv_action_ct_update(dev, idx, update, err); default: return rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -12988,591 +15044,2048 @@ flow_dv_action_update(struct rte_eth_dev *dev, } } -static int -flow_dv_action_query(struct rte_eth_dev *dev, - const struct rte_flow_shared_action *action, void *data, - struct rte_flow_error *error) -{ - struct mlx5_age_param *age_param; - struct rte_flow_query_age *resp; - uint32_t act_idx = (uint32_t)(uintptr_t)action; - uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; - uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); - - switch (type) { - case MLX5_SHARED_ACTION_TYPE_AGE: - age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params; - resp = data; - resp->aged = __atomic_load_n(&age_param->state, - __ATOMIC_RELAXED) == AGE_TMOUT ? - 1 : 0; - resp->sec_since_last_hit_valid = !resp->aged; - if (resp->sec_since_last_hit_valid) - resp->sec_since_last_hit = __atomic_load_n - (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); - return 0; - default: - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "action type query not supported"); - } -} - /** - * Query a dv flow rule for its statistics via devx. + * Destroy the meter sub policy table rules. + * Lock free, (mutex should be acquired by caller). * * @param[in] dev * Pointer to Ethernet device. - * @param[in] flow - * Pointer to the sub flow. - * @param[out] data - * data retrieved by the query. - * @param[out] error - * Perform verbose error reporting if not NULL. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * @param[in] sub_policy + * Pointer to meter sub policy table. */ -static int -flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, - void *data, struct rte_flow_error *error) +static void +__flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_sub_policy *sub_policy) { struct mlx5_priv *priv = dev->data->dev_private; - struct rte_flow_query_count *qc = data; - - if (!priv->config.devx) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "counters are not supported"); - if (flow->counter) { - uint64_t pkts, bytes; - struct mlx5_flow_counter *cnt; - - cnt = flow_dv_counter_get_by_idx(dev, flow->counter, - NULL); - int err = _flow_dv_query_count(dev, flow->counter, &pkts, - &bytes); + struct mlx5_flow_tbl_data_entry *tbl; + struct mlx5_flow_meter_policy *policy = sub_policy->main_policy; + struct mlx5_flow_meter_info *next_fm; + struct mlx5_sub_policy_color_rule *color_rule; + void *tmp; + uint32_t i; - if (err) - return rte_flow_error_set(error, -err, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot read counters"); - qc->hits_set = 1; - qc->bytes_set = 1; - qc->hits = pkts - cnt->hits; - qc->bytes = bytes - cnt->bytes; - if (qc->reset) { - cnt->hits = pkts; - cnt->bytes = bytes; + for (i = 0; i < RTE_COLORS; i++) { + next_fm = NULL; + if (i == RTE_COLOR_GREEN && policy && + policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) + next_fm = mlx5_flow_meter_find(priv, + policy->act_cnt[i].next_mtr_id, NULL); + TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i], + next_port, tmp) { + claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule)); + tbl = container_of(color_rule->matcher->tbl, + typeof(*tbl), tbl); + mlx5_list_unregister(tbl->matchers, + &color_rule->matcher->entry); + TAILQ_REMOVE(&sub_policy->color_rules[i], + color_rule, next_port); + mlx5_free(color_rule); + if (next_fm) + mlx5_flow_meter_detach(priv, next_fm); } - return 0; } - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "counters are not available"); + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { + if (sub_policy->rix_hrxq[i]) { + if (policy && !policy->is_hierarchy) + mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]); + sub_policy->rix_hrxq[i] = 0; + } + if (sub_policy->jump_tbl[i]) { + flow_dv_tbl_resource_release(MLX5_SH(dev), + sub_policy->jump_tbl[i]); + sub_policy->jump_tbl[i] = NULL; + } + } + if (sub_policy->tbl_rsc) { + flow_dv_tbl_resource_release(MLX5_SH(dev), + sub_policy->tbl_rsc); + sub_policy->tbl_rsc = NULL; + } } /** - * Query a flow rule AGE action for aging information. + * Destroy policy rules, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. * * @param[in] dev - * Pointer to Ethernet device. - * @param[in] flow - * Pointer to the sub flow. - * @param[out] data - * data retrieved by the query. - * @param[out] error - * Perform verbose error reporting if not NULL. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. */ -static int -flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow, - void *data, struct rte_flow_error *error) +static void +flow_dv_destroy_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) { - struct rte_flow_query_age *resp = data; - struct mlx5_age_param *age_param; - - if (flow->age) { - struct mlx5_aso_age_action *act = - flow_aso_age_get_by_idx(dev, flow->age); - - age_param = &act->age_params; - } else if (flow->counter) { - age_param = flow_dv_counter_idx_get_age(dev, flow->counter); - - if (!age_param || !age_param->timeout) - return rte_flow_error_set - (error, EINVAL, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot read age data"); - } else { - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "age data not available"); + uint32_t i, j; + struct mlx5_flow_meter_sub_policy *sub_policy; + uint16_t sub_policy_num; + + for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { + sub_policy_num = (mtr_policy->sub_policy_num >> + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & + MLX5_MTR_SUB_POLICY_NUM_MASK; + for (j = 0; j < sub_policy_num; j++) { + sub_policy = mtr_policy->sub_policys[i][j]; + if (sub_policy) + __flow_dv_destroy_sub_policy_rules(dev, + sub_policy); + } } - resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) == - AGE_TMOUT ? 1 : 0; - resp->sec_since_last_hit_valid = !resp->aged; - if (resp->sec_since_last_hit_valid) - resp->sec_since_last_hit = __atomic_load_n - (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); - return 0; } /** - * Query a flow. + * Destroy policy action, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. * - * @see rte_flow_query() - * @see rte_flow_ops + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. */ -static int -flow_dv_query(struct rte_eth_dev *dev, - struct rte_flow *flow __rte_unused, - const struct rte_flow_action *actions __rte_unused, - void *data __rte_unused, - struct rte_flow_error *error __rte_unused) +static void +flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) { - int ret = -EINVAL; + struct rte_flow_action *rss_action; + struct mlx5_flow_handle dev_handle; + uint32_t i, j; - for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { - switch (actions->type) { - case RTE_FLOW_ACTION_TYPE_VOID: + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { + if (mtr_policy->act_cnt[i].rix_mark) { + flow_dv_tag_release(dev, + mtr_policy->act_cnt[i].rix_mark); + mtr_policy->act_cnt[i].rix_mark = 0; + } + if (mtr_policy->act_cnt[i].modify_hdr) { + dev_handle.dvh.modify_hdr = + mtr_policy->act_cnt[i].modify_hdr; + flow_dv_modify_hdr_resource_release(dev, &dev_handle); + } + switch (mtr_policy->act_cnt[i].fate_action) { + case MLX5_FLOW_FATE_SHARED_RSS: + rss_action = mtr_policy->act_cnt[i].rss; + mlx5_free(rss_action); break; - case RTE_FLOW_ACTION_TYPE_COUNT: - ret = flow_dv_query_count(dev, flow, data, error); + case MLX5_FLOW_FATE_PORT_ID: + if (mtr_policy->act_cnt[i].rix_port_id_action) { + flow_dv_port_id_action_resource_release(dev, + mtr_policy->act_cnt[i].rix_port_id_action); + mtr_policy->act_cnt[i].rix_port_id_action = 0; + } break; - case RTE_FLOW_ACTION_TYPE_AGE: - ret = flow_dv_query_age(dev, flow, data, error); + case MLX5_FLOW_FATE_DROP: + case MLX5_FLOW_FATE_JUMP: + for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++) + mtr_policy->act_cnt[i].dr_jump_action[j] = + NULL; break; default: - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, - actions, - "action not supported"); + /*Queue action do nothing*/ + break; + } + } + for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++) + mtr_policy->dr_drop_action[j] = NULL; +} + +/** + * Create policy action per domain, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. + * @param[in] action + * Action specification used to create meter actions. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. + * + * @return + * 0 on success, otherwise negative errno value. + */ +static int +__flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy, + const struct rte_flow_action *actions[RTE_COLORS], + enum mlx5_meter_domain domain, + struct rte_mtr_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct rte_flow_error flow_err; + const struct rte_flow_action *act; + uint64_t action_flags; + struct mlx5_flow_handle dh; + struct mlx5_flow dev_flow; + struct mlx5_flow_dv_port_id_action_resource port_id_action; + int i, ret; + uint8_t egress, transfer; + struct mlx5_meter_policy_action_container *act_cnt = NULL; + union { + struct mlx5_flow_dv_modify_hdr_resource res; + uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) + + sizeof(struct mlx5_modification_cmd) * + (MLX5_MAX_MODIFY_NUM + 1)]; + } mhdr_dummy; + struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; + + egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; + transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; + memset(&dh, 0, sizeof(struct mlx5_flow_handle)); + memset(&dev_flow, 0, sizeof(struct mlx5_flow)); + memset(&port_id_action, 0, + sizeof(struct mlx5_flow_dv_port_id_action_resource)); + memset(mhdr_res, 0, sizeof(*mhdr_res)); + mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : + (egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : + MLX5DV_FLOW_TABLE_TYPE_NIC_RX); + dev_flow.handle = &dh; + dev_flow.dv.port_id_action = &port_id_action; + dev_flow.external = true; + for (i = 0; i < RTE_COLORS; i++) { + if (i < MLX5_MTR_RTE_COLORS) + act_cnt = &mtr_policy->act_cnt[i]; + /* Skip the color policy actions creation. */ + if ((i == RTE_COLOR_YELLOW && mtr_policy->skip_y) || + (i == RTE_COLOR_GREEN && mtr_policy->skip_g)) + continue; + action_flags = 0; + for (act = actions[i]; + act && act->type != RTE_FLOW_ACTION_TYPE_END; act++) { + switch (act->type) { + case RTE_FLOW_ACTION_TYPE_MARK: + { + uint32_t tag_be = mlx5_flow_mark_set + (((const struct rte_flow_action_mark *) + (act->conf))->id); + + if (i >= MLX5_MTR_RTE_COLORS) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "cannot create policy " + "mark action for this color"); + dev_flow.handle->mark = 1; + if (flow_dv_tag_resource_register(dev, tag_be, + &dev_flow, &flow_err)) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "cannot setup policy mark action"); + MLX5_ASSERT(dev_flow.dv.tag_resource); + act_cnt->rix_mark = + dev_flow.handle->dvh.rix_tag; + action_flags |= MLX5_FLOW_ACTION_MARK; + break; + } + case RTE_FLOW_ACTION_TYPE_SET_TAG: + if (i >= MLX5_MTR_RTE_COLORS) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "cannot create policy " + "set tag action for this color"); + if (flow_dv_convert_action_set_tag + (dev, mhdr_res, + (const struct rte_flow_action_set_tag *) + act->conf, &flow_err)) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot convert policy " + "set tag action"); + if (!mhdr_res->actions_num) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot find policy " + "set tag action"); + action_flags |= MLX5_FLOW_ACTION_SET_TAG; + break; + case RTE_FLOW_ACTION_TYPE_DROP: + { + struct mlx5_flow_mtr_mng *mtrmng = + priv->sh->mtrmng; + struct mlx5_flow_tbl_data_entry *tbl_data; + + /* + * Create the drop table with + * METER DROP level. + */ + if (!mtrmng->drop_tbl[domain]) { + mtrmng->drop_tbl[domain] = + flow_dv_tbl_resource_get(dev, + MLX5_FLOW_TABLE_LEVEL_METER, + egress, transfer, false, NULL, 0, + 0, MLX5_MTR_TABLE_ID_DROP, &flow_err); + if (!mtrmng->drop_tbl[domain]) + return -rte_mtr_error_set + (error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "Failed to create meter drop table"); + } + tbl_data = container_of + (mtrmng->drop_tbl[domain], + struct mlx5_flow_tbl_data_entry, tbl); + if (i < MLX5_MTR_RTE_COLORS) { + act_cnt->dr_jump_action[domain] = + tbl_data->jump.action; + act_cnt->fate_action = + MLX5_FLOW_FATE_DROP; + } + if (i == RTE_COLOR_RED) + mtr_policy->dr_drop_action[domain] = + tbl_data->jump.action; + action_flags |= MLX5_FLOW_ACTION_DROP; + break; + } + case RTE_FLOW_ACTION_TYPE_QUEUE: + { + if (i >= MLX5_MTR_RTE_COLORS) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot create policy " + "fate queue for this color"); + act_cnt->queue = + ((const struct rte_flow_action_queue *) + (act->conf))->index; + act_cnt->fate_action = + MLX5_FLOW_FATE_QUEUE; + dev_flow.handle->fate_action = + MLX5_FLOW_FATE_QUEUE; + mtr_policy->is_queue = 1; + action_flags |= MLX5_FLOW_ACTION_QUEUE; + break; + } + case RTE_FLOW_ACTION_TYPE_RSS: + { + int rss_size; + + if (i >= MLX5_MTR_RTE_COLORS) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "cannot create policy " + "rss action for this color"); + /* + * Save RSS conf into policy struct + * for translate stage. + */ + rss_size = (int)rte_flow_conv + (RTE_FLOW_CONV_OP_ACTION, + NULL, 0, act, &flow_err); + if (rss_size <= 0) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "Get the wrong " + "rss action struct size"); + act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO, + rss_size, 0, SOCKET_ID_ANY); + if (!act_cnt->rss) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "Fail to malloc rss action memory"); + ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION, + act_cnt->rss, rss_size, + act, &flow_err); + if (ret < 0) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "Fail to save " + "rss action into policy struct"); + act_cnt->fate_action = + MLX5_FLOW_FATE_SHARED_RSS; + action_flags |= MLX5_FLOW_ACTION_RSS; + break; + } + case RTE_FLOW_ACTION_TYPE_PORT_ID: + { + struct mlx5_flow_dv_port_id_action_resource + port_id_resource; + uint32_t port_id = 0; + + if (i >= MLX5_MTR_RTE_COLORS) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot create policy " + "port action for this color"); + memset(&port_id_resource, 0, + sizeof(port_id_resource)); + if (flow_dv_translate_action_port_id(dev, act, + &port_id, &flow_err)) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot translate " + "policy port action"); + port_id_resource.port_id = port_id; + if (flow_dv_port_id_action_resource_register + (dev, &port_id_resource, + &dev_flow, &flow_err)) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot setup " + "policy port action"); + act_cnt->rix_port_id_action = + dev_flow.handle->rix_port_id_action; + act_cnt->fate_action = + MLX5_FLOW_FATE_PORT_ID; + action_flags |= MLX5_FLOW_ACTION_PORT_ID; + break; + } + case RTE_FLOW_ACTION_TYPE_JUMP: + { + uint32_t jump_group = 0; + uint32_t table = 0; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct flow_grp_info grp_info = { + .external = !!dev_flow.external, + .transfer = !!transfer, + .fdb_def_rule = !!priv->fdb_def_rule, + .std_tbl_fix = 0, + .skip_scale = dev_flow.skip_scale & + (1 << MLX5_SCALE_FLOW_GROUP_BIT), + }; + struct mlx5_flow_meter_sub_policy *sub_policy = + mtr_policy->sub_policys[domain][0]; + + if (i >= MLX5_MTR_RTE_COLORS) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "cannot create policy " + "jump action for this color"); + jump_group = + ((const struct rte_flow_action_jump *) + act->conf)->group; + if (mlx5_flow_group_to_table(dev, NULL, + jump_group, + &table, + &grp_info, &flow_err)) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot setup " + "policy jump action"); + sub_policy->jump_tbl[i] = + flow_dv_tbl_resource_get(dev, + table, egress, + transfer, + !!dev_flow.external, + NULL, jump_group, 0, + 0, &flow_err); + if + (!sub_policy->jump_tbl[i]) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot create jump action."); + tbl_data = container_of + (sub_policy->jump_tbl[i], + struct mlx5_flow_tbl_data_entry, tbl); + act_cnt->dr_jump_action[domain] = + tbl_data->jump.action; + act_cnt->fate_action = + MLX5_FLOW_FATE_JUMP; + action_flags |= MLX5_FLOW_ACTION_JUMP; + break; + } + /* + * No need to check meter hierarchy for Y or R colors + * here since it is done in the validation stage. + */ + case RTE_FLOW_ACTION_TYPE_METER: + { + const struct rte_flow_action_meter *mtr; + struct mlx5_flow_meter_info *next_fm; + struct mlx5_flow_meter_policy *next_policy; + struct rte_flow_action tag_action; + struct mlx5_rte_flow_action_set_tag set_tag; + uint32_t next_mtr_idx = 0; + + mtr = act->conf; + next_fm = mlx5_flow_meter_find(priv, + mtr->mtr_id, + &next_mtr_idx); + if (!next_fm) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_MTR_ID, NULL, + "Fail to find next meter."); + if (next_fm->def_policy) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_MTR_ID, NULL, + "Hierarchy only supports termination meter."); + next_policy = mlx5_flow_meter_policy_find(dev, + next_fm->policy_id, NULL); + MLX5_ASSERT(next_policy); + if (next_fm->drop_cnt) { + set_tag.id = + (enum modify_reg) + mlx5_flow_get_reg_id(dev, + MLX5_MTR_ID, + 0, + (struct rte_flow_error *)error); + set_tag.offset = (priv->mtr_reg_share ? + MLX5_MTR_COLOR_BITS : 0); + set_tag.length = (priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : + MLX5_REG_BITS); + set_tag.data = next_mtr_idx; + tag_action.type = + (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action.conf = &set_tag; + if (flow_dv_convert_action_set_reg + (mhdr_res, &tag_action, + (struct rte_flow_error *)error)) + return -rte_errno; + action_flags |= + MLX5_FLOW_ACTION_SET_TAG; + } + act_cnt->fate_action = MLX5_FLOW_FATE_MTR; + act_cnt->next_mtr_id = next_fm->meter_id; + act_cnt->next_sub_policy = NULL; + mtr_policy->is_hierarchy = 1; + mtr_policy->dev = next_policy->dev; + action_flags |= + MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY; + break; + } + default: + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "action type not supported"); + } + if (action_flags & MLX5_FLOW_ACTION_SET_TAG) { + /* create modify action if needed. */ + dev_flow.dv.group = 1; + if (flow_dv_modify_hdr_resource_register + (dev, mhdr_res, &dev_flow, &flow_err)) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "cannot register policy " + "set tag action"); + act_cnt->modify_hdr = + dev_flow.handle->dvh.modify_hdr; + } + } + } + return 0; +} + +/** + * Create policy action per domain, lock free, + * (mutex should be acquired by caller). + * Dispatcher for action type specific call. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_policy + * Meter policy struct. + * @param[in] action + * Action specification used to create meter actions. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. + * + * @return + * 0 on success, otherwise negative errno value. + */ +static int +flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy, + const struct rte_flow_action *actions[RTE_COLORS], + struct rte_mtr_error *error) +{ + int ret, i; + uint16_t sub_policy_num; + + for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { + sub_policy_num = (mtr_policy->sub_policy_num >> + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & + MLX5_MTR_SUB_POLICY_NUM_MASK; + if (sub_policy_num) { + ret = __flow_dv_create_domain_policy_acts(dev, + mtr_policy, actions, + (enum mlx5_meter_domain)i, error); + /* Cleaning resource is done in the caller level. */ + if (ret) + return ret; + } + } + return 0; +} + +/** + * Query a DV flow rule for its statistics via DevX. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] cnt_idx + * Index to the flow counter. + * @param[out] data + * Data retrieved by the query. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct rte_flow_query_count *qc = data; + + if (!priv->config.devx) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "counters are not supported"); + if (cnt_idx) { + uint64_t pkts, bytes; + struct mlx5_flow_counter *cnt; + int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes); + + if (err) + return rte_flow_error_set(error, -err, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot read counters"); + cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL); + qc->hits_set = 1; + qc->bytes_set = 1; + qc->hits = pkts - cnt->hits; + qc->bytes = bytes - cnt->bytes; + if (qc->reset) { + cnt->hits = pkts; + cnt->bytes = bytes; + } + return 0; + } + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "counters are not available"); +} + +static int +flow_dv_action_query(struct rte_eth_dev *dev, + const struct rte_flow_action_handle *handle, void *data, + struct rte_flow_error *error) +{ + struct mlx5_age_param *age_param; + struct rte_flow_query_age *resp; + uint32_t act_idx = (uint32_t)(uintptr_t)handle; + uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_ct_action *ct; + uint16_t owner; + uint32_t dev_idx; + + switch (type) { + case MLX5_INDIRECT_ACTION_TYPE_AGE: + age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params; + resp = data; + resp->aged = __atomic_load_n(&age_param->state, + __ATOMIC_RELAXED) == AGE_TMOUT ? + 1 : 0; + resp->sec_since_last_hit_valid = !resp->aged; + if (resp->sec_since_last_hit_valid) + resp->sec_since_last_hit = __atomic_load_n + (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); + return 0; + case MLX5_INDIRECT_ACTION_TYPE_COUNT: + return flow_dv_query_count(dev, idx, data, error); + case MLX5_INDIRECT_ACTION_TYPE_CT: + owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx); + if (owner != PORT_ID(priv)) + return rte_flow_error_set(error, EACCES, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "CT object owned by another port"); + dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx); + ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx); + MLX5_ASSERT(ct); + if (!ct->refcnt) + return rte_flow_error_set(error, EFAULT, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "CT object is inactive"); + ((struct rte_flow_action_conntrack *)data)->peer_port = + ct->peer; + ((struct rte_flow_action_conntrack *)data)->is_original_dir = + ct->is_original; + if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data)) + return rte_flow_error_set(error, EIO, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Failed to query CT context"); + return 0; + default: + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "action type query not supported"); + } +} + +/** + * Query a flow rule AGE action for aging information. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] flow + * Pointer to the sub flow. + * @param[out] data + * data retrieved by the query. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow, + void *data, struct rte_flow_error *error) +{ + struct rte_flow_query_age *resp = data; + struct mlx5_age_param *age_param; + + if (flow->age) { + struct mlx5_aso_age_action *act = + flow_aso_age_get_by_idx(dev, flow->age); + + age_param = &act->age_params; + } else if (flow->counter) { + age_param = flow_dv_counter_idx_get_age(dev, flow->counter); + + if (!age_param || !age_param->timeout) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot read age data"); + } else { + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "age data not available"); + } + resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) == + AGE_TMOUT ? 1 : 0; + resp->sec_since_last_hit_valid = !resp->aged; + if (resp->sec_since_last_hit_valid) + resp->sec_since_last_hit = __atomic_load_n + (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); + return 0; +} + +/** + * Query a flow. + * + * @see rte_flow_query() + * @see rte_flow_ops + */ +static int +flow_dv_query(struct rte_eth_dev *dev, + struct rte_flow *flow __rte_unused, + const struct rte_flow_action *actions __rte_unused, + void *data __rte_unused, + struct rte_flow_error *error __rte_unused) +{ + int ret = -EINVAL; + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_VOID: + break; + case RTE_FLOW_ACTION_TYPE_COUNT: + ret = flow_dv_query_count(dev, flow->counter, data, + error); + break; + case RTE_FLOW_ACTION_TYPE_AGE: + ret = flow_dv_query_age(dev, flow, data, error); + break; + default: + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "action not supported"); } } return ret; } /** - * Destroy the meter table set. - * Lock free, (mutex should be acquired by caller). + * Destroy the meter table set. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] fm + * Meter information table. + */ +static void +flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm) +{ + struct mlx5_priv *priv = dev->data->dev_private; + int i; + + if (!fm || !priv->config.dv_flow_en) + return; + for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { + if (fm->drop_rule[i]) { + claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i])); + fm->drop_rule[i] = NULL; + } + } +} + +static void +flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_flow_tbl_data_entry *tbl; + int i, j; + + for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { + if (mtrmng->def_rule[i]) { + claim_zero(mlx5_flow_os_destroy_flow + (mtrmng->def_rule[i])); + mtrmng->def_rule[i] = NULL; + } + if (mtrmng->def_matcher[i]) { + tbl = container_of(mtrmng->def_matcher[i]->tbl, + struct mlx5_flow_tbl_data_entry, tbl); + mlx5_list_unregister(tbl->matchers, + &mtrmng->def_matcher[i]->entry); + mtrmng->def_matcher[i] = NULL; + } + for (j = 0; j < MLX5_REG_BITS; j++) { + if (mtrmng->drop_matcher[i][j]) { + tbl = + container_of(mtrmng->drop_matcher[i][j]->tbl, + struct mlx5_flow_tbl_data_entry, + tbl); + mlx5_list_unregister(tbl->matchers, + &mtrmng->drop_matcher[i][j]->entry); + mtrmng->drop_matcher[i][j] = NULL; + } + } + if (mtrmng->drop_tbl[i]) { + flow_dv_tbl_resource_release(MLX5_SH(dev), + mtrmng->drop_tbl[i]); + mtrmng->drop_tbl[i] = NULL; + } + } +} + +/* Number of meter flow actions, count and jump or count and drop. */ +#define METER_ACTIONS 2 + +static void +__flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev, + enum mlx5_meter_domain domain) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter_def_policy *def_policy = + priv->sh->mtrmng->def_policy[domain]; + + __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy); + mlx5_free(def_policy); + priv->sh->mtrmng->def_policy[domain] = NULL; +} + +/** + * Destroy the default policy table set. * * @param[in] dev * Pointer to Ethernet device. - * @param[in] tbl - * Pointer to the meter table set. - * - * @return - * Always 0. */ -static int -flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, - struct mlx5_meter_domains_infos *tbl) +static void +flow_dv_destroy_def_policy(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_meter_domains_infos *mtd = - (struct mlx5_meter_domains_infos *)tbl; + int i; - if (!mtd || !priv->config.dv_flow_en) - return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); - if (mtd->egress.tbl) - flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); - if (mtd->egress.sfx_tbl) - flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); - if (mtd->ingress.tbl) - flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); - if (mtd->ingress.sfx_tbl) - flow_dv_tbl_resource_release(MLX5_SH(dev), - mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); - if (mtd->transfer.tbl) - flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); - if (mtd->transfer.sfx_tbl) - flow_dv_tbl_resource_release(MLX5_SH(dev), - mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); - mlx5_free(mtd); + for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) + if (priv->sh->mtrmng->def_policy[i]) + __flow_dv_destroy_domain_def_policy(dev, + (enum mlx5_meter_domain)i); + priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; +} + +static int +__flow_dv_create_policy_flow(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + enum rte_color color, void *matcher_object, + int actions_n, void *actions, + bool match_src_port, const struct rte_flow_item *item, + void **rule, const struct rte_flow_attr *attr) +{ + int ret; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf), + }; + struct mlx5_flow_dv_match_params matcher = { + .size = sizeof(matcher.buf), + }; + struct mlx5_priv *priv = dev->data->dev_private; + uint8_t misc_mask; + + if (match_src_port && (priv->representor || priv->master)) { + if (flow_dv_translate_item_port_id(dev, matcher.buf, + value.buf, item, attr)) { + DRV_LOG(ERR, "Failed to create meter policy%d flow's" + " value with port.", color); + return -1; + } + } + flow_dv_match_meta_reg(matcher.buf, value.buf, + (enum modify_reg)color_reg_c_idx, + rte_col_2_mlx5_col(color), UINT32_MAX); + misc_mask = flow_dv_matcher_enable(value.buf); + __flow_dv_adjust_buf_size(&value.size, misc_mask); + ret = mlx5_flow_os_create_flow(matcher_object, (void *)&value, + actions_n, actions, rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policy%d flow.", color); + return -1; + } return 0; } -/* Number of meter flow actions, count and jump or count and drop. */ -#define METER_ACTIONS 2 +static int +__flow_dv_create_policy_matcher(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint16_t priority, + struct mlx5_flow_meter_sub_policy *sub_policy, + const struct rte_flow_attr *attr, + bool match_src_port, + const struct rte_flow_item *item, + struct mlx5_flow_dv_matcher **policy_matcher, + struct rte_flow_error *error) +{ + struct mlx5_list_entry *entry; + struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf), + }, + .tbl = tbl_rsc, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_priv *priv = dev->data->dev_private; + const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; + + if (match_src_port && (priv->representor || priv->master)) { + if (flow_dv_translate_item_port_id(dev, matcher.mask.buf, + value.buf, item, attr)) { + DRV_LOG(ERR, "Failed to register meter policy%d matcher" + " with port.", priority); + return -1; + } + } + tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl); + if (priority < RTE_COLOR_RED) + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + (enum modify_reg)color_reg_c_idx, 0, color_mask); + matcher.priority = priority; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_list_register(tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + *policy_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + return 0; +} /** - * Create specify domain meter table and suffix table. + * Create the policy rules per domain. * * @param[in] dev * Pointer to Ethernet device. - * @param[in,out] mtb - * Pointer to DV meter table set. + * @param[in] sub_policy + * Pointer to sub policy table.. * @param[in] egress - * Table attribute. + * Direction of the table. * @param[in] transfer - * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. + * E-Switch or NIC flow. + * @param[in] acts + * Pointer to policy action list per color. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int -flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, - struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) +__flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_sub_policy *sub_policy, + uint8_t egress, uint8_t transfer, bool match_src_port, + struct mlx5_meter_policy_acts acts[RTE_COLORS]) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, + struct rte_flow_error flow_err; + uint32_t color_reg_c_idx; + struct rte_flow_attr attr = { + .group = MLX5_FLOW_TABLE_LEVEL_POLICY, .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, + .ingress = 0, + .egress = !!egress, + .transfer = !!transfer, + .reserved = 0, }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; - struct rte_flow_error error; - int i = 0; - int ret; + int i; + int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err); + struct mlx5_sub_policy_color_rule *color_rule; + bool svport_match; + struct mlx5_sub_policy_color_rule *tmp_rules[RTE_COLORS] = {NULL}; - if (transfer) - dtb = &mtb->transfer; - else if (egress) - dtb = &mtb->egress; - else - dtb = &mtb->ingress; - /* Create the meter table with METER level. */ - dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, - egress, transfer, false, NULL, 0, - 0, &error); - if (!dtb->tbl) { - DRV_LOG(ERR, "Failed to create meter policer table."); + if (ret < 0) return -1; - } - /* Create the meter suffix table with SUFFIX level. */ - dtb->sfx_tbl = flow_dv_tbl_resource_get(dev, - MLX5_FLOW_TABLE_LEVEL_SUFFIX, - egress, transfer, false, NULL, 0, - 0, &error); - if (!dtb->sfx_tbl) { - DRV_LOG(ERR, "Failed to create meter suffix table."); + /* Create policy table with POLICY level. */ + if (!sub_policy->tbl_rsc) + sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev, + MLX5_FLOW_TABLE_LEVEL_POLICY, + egress, transfer, false, NULL, 0, 0, + sub_policy->idx, &flow_err); + if (!sub_policy->tbl_rsc) { + DRV_LOG(ERR, + "Failed to create meter sub policy table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; + /* Prepare matchers. */ + color_reg_c_idx = ret; + for (i = 0; i < RTE_COLORS; i++) { + TAILQ_INIT(&sub_policy->color_rules[i]); + if (!acts[i].actions_n) + continue; + color_rule = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(struct mlx5_sub_policy_color_rule), + 0, SOCKET_ID_ANY); + if (!color_rule) { + DRV_LOG(ERR, "No memory to create color rule."); + goto err_exit; + } + tmp_rules[i] = color_rule; + TAILQ_INSERT_TAIL(&sub_policy->color_rules[i], + color_rule, next_port); + color_rule->src_port = priv->representor_id; + /* No use. */ + attr.priority = i; + /* Create matchers for colors. */ + svport_match = (i != RTE_COLOR_RED) ? match_src_port : false; + if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx, + MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy, + &attr, svport_match, NULL, + &color_rule->matcher, &flow_err)) { + DRV_LOG(ERR, "Failed to create color%u matcher.", i); + goto err_exit; + } + /* Create flow, matching color. */ + if (__flow_dv_create_policy_flow(dev, + color_reg_c_idx, (enum rte_color)i, + color_rule->matcher->matcher_object, + acts[i].actions_n, acts[i].dv_actions, + svport_match, NULL, &color_rule->rule, + &attr)) { + DRV_LOG(ERR, "Failed to create color%u rule.", i); + goto err_exit; + } } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; + return 0; +err_exit: + /* All the policy rules will be cleared. */ + do { + color_rule = tmp_rules[i]; + if (color_rule) { + if (color_rule->rule) + mlx5_flow_os_destroy_flow(color_rule->rule); + if (color_rule->matcher) { + struct mlx5_flow_tbl_data_entry *tbl = + container_of(color_rule->matcher->tbl, + typeof(*tbl), tbl); + mlx5_list_unregister(tbl->matchers, + &color_rule->matcher->entry); + } + TAILQ_REMOVE(&sub_policy->color_rules[i], + color_rule, next_port); + mlx5_free(color_rule); + } + } while (i--); + return -1; +} + +static int +__flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy, + struct mlx5_flow_meter_sub_policy *sub_policy, + uint32_t domain) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_meter_policy_acts acts[RTE_COLORS]; + struct mlx5_flow_dv_tag_resource *tag; + struct mlx5_flow_dv_port_id_action_resource *port_action; + struct mlx5_hrxq *hrxq; + struct mlx5_flow_meter_info *next_fm = NULL; + struct mlx5_flow_meter_policy *next_policy; + struct mlx5_flow_meter_sub_policy *next_sub_policy; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct rte_flow_error error; + uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; + uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; + bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX); + bool match_src_port = false; + int i; + + /* If RSS or Queue, no previous actions / rules is created. */ + for (i = 0; i < RTE_COLORS; i++) { + acts[i].actions_n = 0; + if (i == RTE_COLOR_RED) { + /* Only support drop on red. */ + acts[i].dv_actions[0] = + mtr_policy->dr_drop_action[domain]; + acts[i].actions_n = 1; + continue; + } + if (i == RTE_COLOR_GREEN && + mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) { + struct rte_flow_attr attr = { + .transfer = transfer + }; + + next_fm = mlx5_flow_meter_find(priv, + mtr_policy->act_cnt[i].next_mtr_id, + NULL); + if (!next_fm) { + DRV_LOG(ERR, + "Failed to get next hierarchy meter."); + goto err_exit; + } + if (mlx5_flow_meter_attach(priv, next_fm, + &attr, &error)) { + DRV_LOG(ERR, "%s", error.message); + next_fm = NULL; + goto err_exit; + } + /* Meter action must be the first for TX. */ + if (mtr_first) { + acts[i].dv_actions[acts[i].actions_n] = + next_fm->meter_action; + acts[i].actions_n++; + } + } + if (mtr_policy->act_cnt[i].rix_mark) { + tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], + mtr_policy->act_cnt[i].rix_mark); + if (!tag) { + DRV_LOG(ERR, "Failed to find " + "mark action for policy."); + goto err_exit; + } + acts[i].dv_actions[acts[i].actions_n] = tag->action; + acts[i].actions_n++; + } + if (mtr_policy->act_cnt[i].modify_hdr) { + acts[i].dv_actions[acts[i].actions_n] = + mtr_policy->act_cnt[i].modify_hdr->action; + acts[i].actions_n++; + } + if (mtr_policy->act_cnt[i].fate_action) { + switch (mtr_policy->act_cnt[i].fate_action) { + case MLX5_FLOW_FATE_PORT_ID: + port_action = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_PORT_ID], + mtr_policy->act_cnt[i].rix_port_id_action); + if (!port_action) { + DRV_LOG(ERR, "Failed to find " + "port action for policy."); + goto err_exit; + } + acts[i].dv_actions[acts[i].actions_n] = + port_action->action; + acts[i].actions_n++; + mtr_policy->dev = dev; + match_src_port = true; + break; + case MLX5_FLOW_FATE_DROP: + case MLX5_FLOW_FATE_JUMP: + acts[i].dv_actions[acts[i].actions_n] = + mtr_policy->act_cnt[i].dr_jump_action[domain]; + acts[i].actions_n++; + break; + case MLX5_FLOW_FATE_SHARED_RSS: + case MLX5_FLOW_FATE_QUEUE: + hrxq = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_HRXQ], + sub_policy->rix_hrxq[i]); + if (!hrxq) { + DRV_LOG(ERR, "Failed to find " + "queue action for policy."); + goto err_exit; + } + acts[i].dv_actions[acts[i].actions_n] = + hrxq->action; + acts[i].actions_n++; + break; + case MLX5_FLOW_FATE_MTR: + if (!next_fm) { + DRV_LOG(ERR, + "No next hierarchy meter."); + goto err_exit; + } + if (!mtr_first) { + acts[i].dv_actions[acts[i].actions_n] = + next_fm->meter_action; + acts[i].actions_n++; + } + if (mtr_policy->act_cnt[i].next_sub_policy) { + next_sub_policy = + mtr_policy->act_cnt[i].next_sub_policy; + } else { + next_policy = + mlx5_flow_meter_policy_find(dev, + next_fm->policy_id, NULL); + MLX5_ASSERT(next_policy); + next_sub_policy = + next_policy->sub_policys[domain][0]; + } + tbl_data = + container_of(next_sub_policy->tbl_rsc, + struct mlx5_flow_tbl_data_entry, tbl); + acts[i].dv_actions[acts[i].actions_n++] = + tbl_data->jump.action; + if (mtr_policy->act_cnt[i].modify_hdr) + match_src_port = !!transfer; + break; + default: + /*Queue action do nothing*/ + break; + } + } + } + if (__flow_dv_create_domain_policy_rules(dev, sub_policy, + egress, transfer, match_src_port, acts)) { + DRV_LOG(ERR, + "Failed to create policy rules per domain."); + goto err_exit; } return 0; -error_exit: +err_exit: + if (next_fm) + mlx5_flow_meter_detach(priv, next_fm); return -1; } /** - * Create the needed meter and suffix tables. - * Lock free, (mutex should be acquired by caller). + * Create the policy rules. * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. + * @param[in,out] mtr_policy + * Pointer to meter policy table. * * @return - * Pointer to table set on success, NULL otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ -static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +static int +flow_dv_create_policy_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) +{ + int i; + uint16_t sub_policy_num; + + for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { + sub_policy_num = (mtr_policy->sub_policy_num >> + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & + MLX5_MTR_SUB_POLICY_NUM_MASK; + if (!sub_policy_num) + continue; + /* Prepare actions list and create policy rules. */ + if (__flow_dv_create_policy_acts_rules(dev, mtr_policy, + mtr_policy->sub_policys[i][0], i)) { + DRV_LOG(ERR, "Failed to create policy action " + "list per domain."); + return -1; + } + } + return 0; +} + +static int +__flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_meter_domains_infos *mtb; + struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_flow_meter_def_policy *def_policy; + struct mlx5_flow_tbl_resource *jump_tbl; + struct mlx5_flow_tbl_data_entry *tbl_data; + uint8_t egress, transfer; + struct rte_flow_error error; + struct mlx5_meter_policy_acts acts[RTE_COLORS]; int ret; + + egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; + transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; + def_policy = mtrmng->def_policy[domain]; + if (!def_policy) { + def_policy = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(struct mlx5_flow_meter_def_policy), + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); + if (!def_policy) { + DRV_LOG(ERR, "Failed to alloc default policy table."); + goto def_policy_error; + } + mtrmng->def_policy[domain] = def_policy; + /* Create the meter suffix table with SUFFIX level. */ + jump_tbl = flow_dv_tbl_resource_get(dev, + MLX5_FLOW_TABLE_LEVEL_METER, + egress, transfer, false, NULL, 0, + 0, MLX5_MTR_TABLE_ID_SUFFIX, &error); + if (!jump_tbl) { + DRV_LOG(ERR, + "Failed to create meter suffix table."); + goto def_policy_error; + } + def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl; + tbl_data = container_of(jump_tbl, + struct mlx5_flow_tbl_data_entry, tbl); + def_policy->dr_jump_action[RTE_COLOR_GREEN] = + tbl_data->jump.action; + acts[RTE_COLOR_GREEN].dv_actions[0] = tbl_data->jump.action; + acts[RTE_COLOR_GREEN].actions_n = 1; + /* + * YELLOW has the same default policy as GREEN does. + * G & Y share the same table and action. The 2nd time of table + * resource getting is just to update the reference count for + * the releasing stage. + */ + jump_tbl = flow_dv_tbl_resource_get(dev, + MLX5_FLOW_TABLE_LEVEL_METER, + egress, transfer, false, NULL, 0, + 0, MLX5_MTR_TABLE_ID_SUFFIX, &error); + if (!jump_tbl) { + DRV_LOG(ERR, + "Failed to get meter suffix table."); + goto def_policy_error; + } + def_policy->sub_policy.jump_tbl[RTE_COLOR_YELLOW] = jump_tbl; + tbl_data = container_of(jump_tbl, + struct mlx5_flow_tbl_data_entry, tbl); + def_policy->dr_jump_action[RTE_COLOR_YELLOW] = + tbl_data->jump.action; + acts[RTE_COLOR_YELLOW].dv_actions[0] = tbl_data->jump.action; + acts[RTE_COLOR_YELLOW].actions_n = 1; + /* Create jump action to the drop table. */ + if (!mtrmng->drop_tbl[domain]) { + mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get + (dev, MLX5_FLOW_TABLE_LEVEL_METER, + egress, transfer, false, NULL, 0, + 0, MLX5_MTR_TABLE_ID_DROP, &error); + if (!mtrmng->drop_tbl[domain]) { + DRV_LOG(ERR, "Failed to create meter " + "drop table for default policy."); + goto def_policy_error; + } + } + /* all RED: unique Drop table for jump action. */ + tbl_data = container_of(mtrmng->drop_tbl[domain], + struct mlx5_flow_tbl_data_entry, tbl); + def_policy->dr_jump_action[RTE_COLOR_RED] = + tbl_data->jump.action; + acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action; + acts[RTE_COLOR_RED].actions_n = 1; + /* Create default policy rules. */ + ret = __flow_dv_create_domain_policy_rules(dev, + &def_policy->sub_policy, + egress, transfer, false, acts); + if (ret) { + DRV_LOG(ERR, "Failed to create default policy rules."); + goto def_policy_error; + } + } + return 0; +def_policy_error: + __flow_dv_destroy_domain_def_policy(dev, + (enum mlx5_meter_domain)domain); + return -1; +} + +/** + * Create the default policy table set. + * + * @param[in] dev + * Pointer to Ethernet device. + * @return + * 0 on success, -1 otherwise. + */ +static int +flow_dv_create_def_policy(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; int i; - if (!priv->mtr_en) { + /* Non-termination policy table. */ + for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { + if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER) + continue; + if (__flow_dv_create_domain_def_policy(dev, i)) { + DRV_LOG(ERR, "Failed to create default policy"); + /* Rollback the created default policies for others. */ + flow_dv_destroy_def_policy(dev); + return -1; + } + } + return 0; +} + +/** + * Create the needed meter tables. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] fm + * Meter information table. + * @param[in] mtr_idx + * Meter index. + * @param[in] domain_bitmap + * Domain bitmap. + * @return + * 0 on success, -1 otherwise. + */ +static int +flow_dv_create_mtr_tbls(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm, + uint32_t mtr_idx, + uint8_t domain_bitmap) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng; + struct rte_flow_error error; + struct mlx5_flow_tbl_data_entry *tbl_data; + uint8_t egress, transfer; + void *actions[METER_ACTIONS]; + int domain, ret, i; + struct mlx5_flow_counter *cnt; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf), + }; + struct mlx5_flow_dv_match_params matcher_para = { + .size = sizeof(matcher_para.buf), + }; + int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + struct mlx5_list_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf), + }, + }; + struct mlx5_flow_dv_matcher *drop_matcher; + struct mlx5_flow_cb_ctx ctx = { + .error = &error, + .data = &matcher, + }; + uint8_t misc_mask; + + if (!priv->mtr_en || mtr_id_reg_c < 0) { rte_errno = ENOTSUP; - return NULL; + return -1; + } + for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) { + if (!(domain_bitmap & (1 << domain)) || + (mtrmng->def_rule[domain] && !fm->drop_cnt)) + continue; + egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0; + transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0; + /* Create the drop table with METER DROP level. */ + if (!mtrmng->drop_tbl[domain]) { + mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev, + MLX5_FLOW_TABLE_LEVEL_METER, + egress, transfer, false, NULL, 0, + 0, MLX5_MTR_TABLE_ID_DROP, &error); + if (!mtrmng->drop_tbl[domain]) { + DRV_LOG(ERR, "Failed to create meter drop table."); + goto policy_error; + } + } + /* Create default matcher in drop table. */ + matcher.tbl = mtrmng->drop_tbl[domain], + tbl_data = container_of(mtrmng->drop_tbl[domain], + struct mlx5_flow_tbl_data_entry, tbl); + if (!mtrmng->def_matcher[domain]) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + (enum modify_reg)mtr_id_reg_c, + 0, 0); + matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY; + matcher.crc = rte_raw_cksum + ((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_list_register(tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter " + "drop default matcher."); + goto policy_error; + } + mtrmng->def_matcher[domain] = container_of(entry, + struct mlx5_flow_dv_matcher, entry); + } + /* Create default rule in drop table. */ + if (!mtrmng->def_rule[domain]) { + i = 0; + actions[i++] = priv->sh->dr_drop_action; + flow_dv_match_meta_reg(matcher_para.buf, value.buf, + (enum modify_reg)mtr_id_reg_c, 0, 0); + misc_mask = flow_dv_matcher_enable(value.buf); + __flow_dv_adjust_buf_size(&value.size, misc_mask); + ret = mlx5_flow_os_create_flow + (mtrmng->def_matcher[domain]->matcher_object, + (void *)&value, i, actions, + &mtrmng->def_rule[domain]); + if (ret) { + DRV_LOG(ERR, "Failed to create meter " + "default drop rule for drop table."); + goto policy_error; + } + } + if (!fm->drop_cnt) + continue; + MLX5_ASSERT(mtrmng->max_mtr_bits); + if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + (enum modify_reg)mtr_id_reg_c, 0, + (mtr_id_mask << mtr_id_offset)); + matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits; + matcher.crc = rte_raw_cksum + ((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_list_register(tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, + "Failed to register meter drop matcher."); + goto policy_error; + } + mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] = + container_of(entry, struct mlx5_flow_dv_matcher, + entry); + } + drop_matcher = + mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]; + /* Create drop rule, matching meter_id only. */ + flow_dv_match_meta_reg(matcher_para.buf, value.buf, + (enum modify_reg)mtr_id_reg_c, + (mtr_idx << mtr_id_offset), UINT32_MAX); + i = 0; + cnt = flow_dv_counter_get_by_idx(dev, + fm->drop_cnt, NULL); + actions[i++] = cnt->action; + actions[i++] = priv->sh->dr_drop_action; + misc_mask = flow_dv_matcher_enable(value.buf); + __flow_dv_adjust_buf_size(&value.size, misc_mask); + ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object, + (void *)&value, i, actions, + &fm->drop_rule[domain]); + if (ret) { + DRV_LOG(ERR, "Failed to create meter " + "drop rule for drop table."); + goto policy_error; + } + } + return 0; +policy_error: + for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { + if (fm->drop_rule[i]) { + claim_zero(mlx5_flow_os_destroy_flow + (fm->drop_rule[i])); + fm->drop_rule[i] = NULL; + } + } + return -1; +} + +static struct mlx5_flow_meter_sub_policy * +__flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy, + struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS], + struct mlx5_flow_meter_sub_policy *next_sub_policy, + bool *is_reuse) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter_sub_policy *sub_policy = NULL; + uint32_t sub_policy_idx = 0; + uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0}; + uint32_t i, j; + struct mlx5_hrxq *hrxq; + struct mlx5_flow_handle dh; + struct mlx5_meter_policy_action_container *act_cnt; + uint32_t domain = MLX5_MTR_DOMAIN_INGRESS; + uint16_t sub_policy_num; + + rte_spinlock_lock(&mtr_policy->sl); + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { + if (!rss_desc[i]) + continue; + hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]); + if (!hrxq_idx[i]) { + rte_spinlock_unlock(&mtr_policy->sl); + return NULL; + } + } + sub_policy_num = (mtr_policy->sub_policy_num >> + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & + MLX5_MTR_SUB_POLICY_NUM_MASK; + for (j = 0; j < sub_policy_num; j++) { + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { + if (rss_desc[i] && + hrxq_idx[i] != + mtr_policy->sub_policys[domain][j]->rix_hrxq[i]) + break; + } + if (i >= MLX5_MTR_RTE_COLORS) { + /* + * Found the sub policy table with + * the same queue per color. + */ + rte_spinlock_unlock(&mtr_policy->sl); + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) + mlx5_hrxq_release(dev, hrxq_idx[i]); + *is_reuse = true; + return mtr_policy->sub_policys[domain][j]; + } } - mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY); - if (!mtb) { - DRV_LOG(ERR, "Failed to allocate memory for meter."); - return NULL; + /* Create sub policy. */ + if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) { + /* Reuse the first pre-allocated sub_policy. */ + sub_policy = mtr_policy->sub_policys[domain][0]; + sub_policy_idx = sub_policy->idx; + } else { + sub_policy = mlx5_ipool_zmalloc + (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], + &sub_policy_idx); + if (!sub_policy || + sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) { + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) + mlx5_hrxq_release(dev, hrxq_idx[i]); + goto rss_sub_policy_error; + } + sub_policy->idx = sub_policy_idx; + sub_policy->main_policy = mtr_policy; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { + if (!rss_desc[i]) continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; + sub_policy->rix_hrxq[i] = hrxq_idx[i]; + if (mtr_policy->is_hierarchy) { + act_cnt = &mtr_policy->act_cnt[i]; + act_cnt->next_sub_policy = next_sub_policy; + mlx5_hrxq_release(dev, hrxq_idx[i]); + } else { + /* + * Overwrite the last action from + * RSS action to Queue action. + */ + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], + hrxq_idx[i]); + if (!hrxq) { + DRV_LOG(ERR, "Failed to get policy hrxq"); + goto rss_sub_policy_error; + } + act_cnt = &mtr_policy->act_cnt[i]; + if (act_cnt->rix_mark || act_cnt->modify_hdr) { + memset(&dh, 0, sizeof(struct mlx5_flow_handle)); + if (act_cnt->rix_mark) + dh.mark = 1; + dh.fate_action = MLX5_FLOW_FATE_QUEUE; + dh.rix_hrxq = hrxq_idx[i]; + flow_drv_rxq_flags_set(dev, &dh); + } + } } - /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); - if (ret) { - DRV_LOG(ERR, "Failed to prepare egress meter table."); - goto error_exit; + if (__flow_dv_create_policy_acts_rules(dev, mtr_policy, + sub_policy, domain)) { + DRV_LOG(ERR, "Failed to create policy " + "rules for ingress domain."); + goto rss_sub_policy_error; } - /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); - if (ret) { - DRV_LOG(ERR, "Failed to prepare ingress meter table."); - goto error_exit; + if (sub_policy != mtr_policy->sub_policys[domain][0]) { + i = (mtr_policy->sub_policy_num >> + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & + MLX5_MTR_SUB_POLICY_NUM_MASK; + if (i >= MLX5_MTR_RSS_MAX_SUB_POLICY) { + DRV_LOG(ERR, "No free sub-policy slot."); + goto rss_sub_policy_error; + } + mtr_policy->sub_policys[domain][i] = sub_policy; + i++; + mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)); + mtr_policy->sub_policy_num |= + (i & MLX5_MTR_SUB_POLICY_NUM_MASK) << + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain); } - /* FDB meter table. */ - if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); - if (ret) { - DRV_LOG(ERR, "Failed to prepare fdb meter table."); - goto error_exit; + rte_spinlock_unlock(&mtr_policy->sl); + *is_reuse = false; + return sub_policy; +rss_sub_policy_error: + if (sub_policy) { + __flow_dv_destroy_sub_policy_rules(dev, sub_policy); + if (sub_policy != mtr_policy->sub_policys[domain][0]) { + i = (mtr_policy->sub_policy_num >> + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & + MLX5_MTR_SUB_POLICY_NUM_MASK; + mtr_policy->sub_policys[domain][i] = NULL; + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], + sub_policy->idx); } } - return mtb; -error_exit: - flow_dv_destroy_mtr_tbl(dev, mtb); + rte_spinlock_unlock(&mtr_policy->sl); return NULL; } /** - * Destroy domain policer rule. + * Find the policy table for prefix table with RSS. * - * @param[in] dt - * Pointer to domain table. + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] mtr_policy + * Pointer to meter policy table. + * @param[in] rss_desc + * Pointer to rss_desc + * @return + * Pointer to table set on success, NULL otherwise and rte_errno is set. */ -static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +static struct mlx5_flow_meter_sub_policy * +flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy, + struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) { - int i; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter_sub_policy *sub_policy = NULL; + struct mlx5_flow_meter_info *next_fm; + struct mlx5_flow_meter_policy *next_policy; + struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL; + struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM]; + struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM]; + uint32_t domain = MLX5_MTR_DOMAIN_INGRESS; + bool reuse_sub_policy; + uint32_t i = 0; + uint32_t j = 0; - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; + while (true) { + /* Iterate hierarchy to get all policies in this hierarchy. */ + policies[i++] = mtr_policy; + if (!mtr_policy->is_hierarchy) + break; + if (i >= MLX5_MTR_CHAIN_MAX_NUM) { + DRV_LOG(ERR, "Exceed max meter number in hierarchy."); + return NULL; + } + next_fm = mlx5_flow_meter_find(priv, + mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL); + if (!next_fm) { + DRV_LOG(ERR, "Failed to get next meter in hierarchy."); + return NULL; + } + next_policy = + mlx5_flow_meter_policy_find(dev, next_fm->policy_id, + NULL); + MLX5_ASSERT(next_policy); + mtr_policy = next_policy; + } + while (i) { + /** + * From last policy to the first one in hierarchy, + * create / get the sub policy for each of them. + */ + sub_policy = __flow_dv_meter_get_rss_sub_policy(dev, + policies[--i], + rss_desc, + next_sub_policy, + &reuse_sub_policy); + if (!sub_policy) { + DRV_LOG(ERR, "Failed to get the sub policy."); + goto err_exit; } + if (!reuse_sub_policy) + sub_policies[j++] = sub_policy; + next_sub_policy = sub_policy; } - if (dt->jump_actn) { - claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); - dt->jump_actn = NULL; + return sub_policy; +err_exit: + while (j) { + uint16_t sub_policy_num; + + sub_policy = sub_policies[--j]; + mtr_policy = sub_policy->main_policy; + __flow_dv_destroy_sub_policy_rules(dev, sub_policy); + if (sub_policy != mtr_policy->sub_policys[domain][0]) { + sub_policy_num = (mtr_policy->sub_policy_num >> + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & + MLX5_MTR_SUB_POLICY_NUM_MASK; + mtr_policy->sub_policys[domain][sub_policy_num - 1] = + NULL; + sub_policy_num--; + mtr_policy->sub_policy_num &= + ~(MLX5_MTR_SUB_POLICY_NUM_MASK << + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); + mtr_policy->sub_policy_num |= + (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], + sub_policy->idx); + } } + return NULL; } /** - * Destroy policer rules. + * Create the sub policy tag rule for all meters in hierarchy. * * @param[in] dev * Pointer to Ethernet device. * @param[in] fm - * Pointer to flow meter structure. - * @param[in] attr - * Pointer to flow attributes. - * - * @return - * Always 0. - */ -static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, - const struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) -{ - struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; - - if (!mtb) - return 0; - if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); - if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); - if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); - return 0; -} - -/** - * Create specify domain meter policer rule. - * - * @param[in] fm - * Pointer to flow meter structure. - * @param[in] mtb - * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. - * + * Meter information table. + * @param[in] src_port + * The src port this extra rule should use. + * @param[in] item + * The src port match item. + * @param[out] error + * Perform verbose error reporting if not NULL. * @return - * 0 on success, -1 otherwise. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, - struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) +flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm, + int32_t src_port, + const struct rte_flow_item *item, + struct rte_flow_error *error) { - struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter_policy *mtr_policy; + struct mlx5_flow_meter_sub_policy *sub_policy; + struct mlx5_flow_meter_info *next_fm = NULL; + struct mlx5_flow_meter_policy *next_policy; + struct mlx5_flow_meter_sub_policy *next_sub_policy; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_sub_policy_color_rule *color_rule; + struct mlx5_meter_policy_acts acts; + uint32_t color_reg_c_idx; + bool mtr_first = (src_port != UINT16_MAX) ? true : false; + struct rte_flow_attr attr = { + .group = MLX5_FLOW_TABLE_LEVEL_POLICY, + .priority = 0, + .ingress = 0, + .egress = 0, + .transfer = 1, + .reserved = 0, }; - struct mlx5_meter_domains_infos *mtb = fm->mfts; - void *actions[METER_ACTIONS]; + uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER; int i; - int ret = 0; - /* Create jump action. */ - if (!dtb->jump_actn) - ret = mlx5_flow_os_create_flow_action_dest_flow_tbl - (dtb->sfx_tbl->obj, &dtb->jump_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create policer jump action."); - goto error; + mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); + MLX5_ASSERT(mtr_policy); + if (!mtr_policy->is_hierarchy) + return 0; + next_fm = mlx5_flow_meter_find(priv, + mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL); + if (!next_fm) { + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Failed to find next meter in hierarchy."); } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); - if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); - goto error; + if (!next_fm->drop_cnt) + goto exit; + color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error); + sub_policy = mtr_policy->sub_policys[domain][0]; + for (i = 0; i < RTE_COLORS; i++) { + bool rule_exist = false; + struct mlx5_meter_policy_action_container *act_cnt; + + if (i >= RTE_COLOR_YELLOW) + break; + TAILQ_FOREACH(color_rule, + &sub_policy->color_rules[i], next_port) + if (color_rule->src_port == src_port) { + rule_exist = true; + break; + } + if (rule_exist) + continue; + color_rule = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(struct mlx5_sub_policy_color_rule), + 0, SOCKET_ID_ANY); + if (!color_rule) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "No memory to create tag color rule."); + color_rule->src_port = src_port; + attr.priority = i; + next_policy = mlx5_flow_meter_policy_find(dev, + next_fm->policy_id, NULL); + MLX5_ASSERT(next_policy); + next_sub_policy = next_policy->sub_policys[domain][0]; + tbl_data = container_of(next_sub_policy->tbl_rsc, + struct mlx5_flow_tbl_data_entry, tbl); + act_cnt = &mtr_policy->act_cnt[i]; + if (mtr_first) { + acts.dv_actions[0] = next_fm->meter_action; + acts.dv_actions[1] = act_cnt->modify_hdr->action; + } else { + acts.dv_actions[0] = act_cnt->modify_hdr->action; + acts.dv_actions[1] = next_fm->meter_action; } + acts.dv_actions[2] = tbl_data->jump.action; + acts.actions_n = 3; + if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) { + next_fm = NULL; + goto err_exit; + } + if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx, + MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy, + &attr, true, item, + &color_rule->matcher, error)) { + rte_flow_error_set(error, errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to create hierarchy meter matcher."); + goto err_exit; + } + if (__flow_dv_create_policy_flow(dev, color_reg_c_idx, + (enum rte_color)i, + color_rule->matcher->matcher_object, + acts.actions_n, acts.dv_actions, + true, item, + &color_rule->rule, &attr)) { + rte_flow_error_set(error, errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to create hierarchy meter rule."); + goto err_exit; + } + TAILQ_INSERT_TAIL(&sub_policy->color_rules[i], + color_rule, next_port); } - return 0; -error: - rte_errno = errno; - return -1; +exit: + /** + * Recursive call to iterate all meters in hierarchy and + * create needed rules. + */ + return flow_dv_meter_hierarchy_rule_create(dev, next_fm, + src_port, item, error); +err_exit: + if (color_rule) { + if (color_rule->rule) + mlx5_flow_os_destroy_flow(color_rule->rule); + if (color_rule->matcher) { + struct mlx5_flow_tbl_data_entry *tbl = + container_of(color_rule->matcher->tbl, + typeof(*tbl), tbl); + mlx5_list_unregister(tbl->matchers, + &color_rule->matcher->entry); + } + mlx5_free(color_rule); + } + if (next_fm) + mlx5_flow_meter_detach(priv, next_fm); + return -rte_errno; } /** - * Create policer rules. + * Destroy the sub policy table with RX queue. * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to flow meter structure. - * @param[in] attr - * Pointer to flow attributes. - * - * @return - * 0 on success, -1 otherwise. + * @param[in] mtr_policy + * Pointer to meter policy table. */ -static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +static void +flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, + struct mlx5_flow_meter_policy *mtr_policy) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_meter_domains_infos *mtb = fm->mfts; - int ret; - - if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); - if (ret) { - DRV_LOG(ERR, "Failed to create egress policer."); - goto error; - } - } - if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); - if (ret) { - DRV_LOG(ERR, "Failed to create ingress policer."); - goto error; - } - } - if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); - if (ret) { - DRV_LOG(ERR, "Failed to create transfer policer."); - goto error; + struct mlx5_flow_meter_sub_policy *sub_policy = NULL; + uint32_t domain = MLX5_MTR_DOMAIN_INGRESS; + uint32_t i, j; + uint16_t sub_policy_num, new_policy_num; + + rte_spinlock_lock(&mtr_policy->sl); + for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { + switch (mtr_policy->act_cnt[i].fate_action) { + case MLX5_FLOW_FATE_SHARED_RSS: + sub_policy_num = (mtr_policy->sub_policy_num >> + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) & + MLX5_MTR_SUB_POLICY_NUM_MASK; + new_policy_num = sub_policy_num; + for (j = 0; j < sub_policy_num; j++) { + sub_policy = + mtr_policy->sub_policys[domain][j]; + if (sub_policy) { + __flow_dv_destroy_sub_policy_rules(dev, + sub_policy); + if (sub_policy != + mtr_policy->sub_policys[domain][0]) { + mtr_policy->sub_policys[domain][j] = + NULL; + mlx5_ipool_free + (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], + sub_policy->idx); + new_policy_num--; + } + } + } + if (new_policy_num != sub_policy_num) { + mtr_policy->sub_policy_num &= + ~(MLX5_MTR_SUB_POLICY_NUM_MASK << + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)); + mtr_policy->sub_policy_num |= + (new_policy_num & + MLX5_MTR_SUB_POLICY_NUM_MASK) << + (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain); + } + break; + case MLX5_FLOW_FATE_QUEUE: + sub_policy = mtr_policy->sub_policys[domain][0]; + __flow_dv_destroy_sub_policy_rules(dev, + sub_policy); + break; + default: + /*Other actions without queue and do nothing*/ + break; } } - return 0; -error: - flow_dv_destroy_policer_rules(dev, fm, attr); - return -1; + rte_spinlock_unlock(&mtr_policy->sl); } /** @@ -13599,7 +17112,7 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) .size = sizeof(value.buf), }; struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, + .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS, .priority = 0, .match_criteria_enable = 0, .match_mask = (void *)&mask, @@ -13611,7 +17124,8 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) void *flow = NULL; int ret = -1; - tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, 0, NULL); + tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL, + 0, 0, 0, NULL); if (!tbl) goto err; dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); @@ -13621,13 +17135,14 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) &actions[0]); if (ret) goto err; - actions[1] = priv->drop_queue.hrxq->action; dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf); + __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable); ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj, &matcher); if (ret) goto err; - ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 2, + __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable); + ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1, actions, &flow); err: /* @@ -13724,7 +17239,7 @@ flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear, * @note: only stub for now */ static int -flow_get_aged_flows(struct rte_eth_dev *dev, +flow_dv_get_aged_flows(struct rte_eth_dev *dev, void **context, uint32_t nb_contexts, struct rte_flow_error *error) @@ -13775,15 +17290,15 @@ flow_dv_counter_allocate(struct rte_eth_dev *dev) } /** - * Validate shared action. + * Validate indirect action. * Dispatcher for action type specific validation. * * @param[in] dev * Pointer to the Ethernet device structure. * @param[in] conf - * Shared action configuration. + * Indirect action configuration. * @param[in] action - * The shared action object to validate. + * The indirect action object to validate. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -13793,7 +17308,7 @@ flow_dv_counter_allocate(struct rte_eth_dev *dev) */ static int flow_dv_action_validate(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_indir_action_conf *conf, const struct rte_flow_action *action, struct rte_flow_error *err) { @@ -13802,14 +17317,48 @@ flow_dv_action_validate(struct rte_eth_dev *dev, RTE_SET_USED(conf); switch (action->type) { case RTE_FLOW_ACTION_TYPE_RSS: + /* + * priv->obj_ops is set according to driver capabilities. + * When DevX capabilities are + * sufficient, it is set to devx_obj_ops. + * Otherwise, it is set to ibv_obj_ops. + * ibv_obj_ops doesn't support ind_table_modify operation. + * In this case the indirect RSS action can't be used. + */ + if (priv->obj_ops.ind_table_modify == NULL) + return rte_flow_error_set + (err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "Indirect RSS action not supported"); return mlx5_validate_action_rss(dev, action, err); case RTE_FLOW_ACTION_TYPE_AGE: if (!priv->sh->aso_age_mng) return rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "shared age action not supported"); + "Indirect age action not supported"); return flow_dv_validate_action_age(0, action, dev, err); + case RTE_FLOW_ACTION_TYPE_COUNT: + /* + * There are two mechanisms to share the action count. + * The old mechanism uses the shared field to share, while the + * new mechanism uses the indirect action API. + * This validation comes to make sure that the two mechanisms + * are not combined. + */ + if (is_shared_action_count(action)) + return rte_flow_error_set(err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "Mix shared and indirect counter is not supported"); + return flow_dv_validate_action_count(dev, true, 0, err); + case RTE_FLOW_ACTION_TYPE_CONNTRACK: + if (!priv->sh->ct_aso_en) + return rte_flow_error_set(err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "ASO CT is not supported"); + return mlx5_validate_action_ct(dev, action->conf, err); default: return rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -13818,6 +17367,450 @@ flow_dv_action_validate(struct rte_eth_dev *dev, } } +/* + * Check if the RSS configurations for colors of a meter policy match + * each other, except the queues. + * + * @param[in] r1 + * Pointer to the first RSS flow action. + * @param[in] r2 + * Pointer to the second RSS flow action. + * + * @return + * 0 on match, 1 on conflict. + */ +static inline int +flow_dv_mtr_policy_rss_compare(const struct rte_flow_action_rss *r1, + const struct rte_flow_action_rss *r2) +{ + if (!r1 || !r2) + return 0; + if (r1->func != r2->func || r1->level != r2->level || + r1->types != r2->types || r1->key_len != r2->key_len || + memcmp(r1->key, r2->key, r1->key_len)) + return 1; + return 0; +} + +/** + * Validate the meter hierarchy chain for meter policy. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] meter_id + * Meter id. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[out] is_rss + * Is RSS or not. + * @param[out] hierarchy_domain + * The domain bitmap for hierarchy policy. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. + * + * @return + * 0 on success, otherwise negative errno value with error set. + */ +static int +flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev, + uint32_t meter_id, + uint64_t action_flags, + bool *is_rss, + uint8_t *hierarchy_domain, + struct rte_mtr_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter_info *fm; + struct mlx5_flow_meter_policy *policy; + uint8_t cnt = 1; + + if (action_flags & (MLX5_FLOW_FATE_ACTIONS | + MLX5_FLOW_FATE_ESWITCH_ACTIONS)) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN, + NULL, + "Multiple fate actions not supported."); + while (true) { + fm = mlx5_flow_meter_find(priv, meter_id, NULL); + if (!fm) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_MTR_ID, NULL, + "Meter not found in meter hierarchy."); + if (fm->def_policy) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_MTR_ID, NULL, + "Non termination meter not supported in hierarchy."); + policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); + MLX5_ASSERT(policy); + if (!policy->is_hierarchy) { + if (policy->transfer) + *hierarchy_domain |= + MLX5_MTR_DOMAIN_TRANSFER_BIT; + if (policy->ingress) + *hierarchy_domain |= + MLX5_MTR_DOMAIN_INGRESS_BIT; + if (policy->egress) + *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT; + *is_rss = policy->is_rss; + break; + } + meter_id = policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id; + if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, + "Exceed max hierarchy meter number."); + } + return 0; +} + +/** + * Validate meter policy actions. + * Dispatcher for action type specific validation. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] action + * The meter policy action object to validate. + * @param[in] attr + * Attributes of flow to determine steering domain. + * @param[out] error + * Perform verbose error reporting if not NULL. Initialized in case of + * error only. + * + * @return + * 0 on success, otherwise negative errno value. + */ +static int +flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev, + const struct rte_flow_action *actions[RTE_COLORS], + struct rte_flow_attr *attr, + bool *is_rss, + uint8_t *domain_bitmap, + uint8_t *policy_mode, + struct rte_mtr_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *dev_conf = &priv->config; + const struct rte_flow_action *act; + uint64_t action_flags[RTE_COLORS] = {0}; + int actions_n; + int i, ret; + struct rte_flow_error flow_err; + uint8_t domain_color[RTE_COLORS] = {0}; + uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT; + uint8_t hierarchy_domain = 0; + const struct rte_flow_action_meter *mtr; + bool def_green = false; + bool def_yellow = false; + const struct rte_flow_action_rss *rss_color[RTE_COLORS] = {NULL}; + + if (!priv->config.dv_esw_en) + def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; + *domain_bitmap = def_domain; + /* Red color could only support DROP action. */ + if (!actions[RTE_COLOR_RED] || + actions[RTE_COLOR_RED]->type != RTE_FLOW_ACTION_TYPE_DROP) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "Red color only supports drop action."); + /* + * Check default policy actions: + * Green / Yellow: no action, Red: drop action + * Either G or Y will trigger default policy actions to be created. + */ + if (!actions[RTE_COLOR_GREEN] || + actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END) + def_green = true; + if (!actions[RTE_COLOR_YELLOW] || + actions[RTE_COLOR_YELLOW]->type == RTE_FLOW_ACTION_TYPE_END) + def_yellow = true; + if (def_green && def_yellow) { + *policy_mode = MLX5_MTR_POLICY_MODE_DEF; + return 0; + } else if (!def_green && def_yellow) { + *policy_mode = MLX5_MTR_POLICY_MODE_OG; + } else if (def_green && !def_yellow) { + *policy_mode = MLX5_MTR_POLICY_MODE_OY; + } + /* Set to empty string in case of NULL pointer access by user. */ + flow_err.message = ""; + for (i = 0; i < RTE_COLORS; i++) { + act = actions[i]; + for (action_flags[i] = 0, actions_n = 0; + act && act->type != RTE_FLOW_ACTION_TYPE_END; + act++) { + if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "too many actions"); + switch (act->type) { + case RTE_FLOW_ACTION_TYPE_PORT_ID: + if (!priv->config.dv_esw_en) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "PORT action validate check" + " fail for ESW disable"); + ret = flow_dv_validate_action_port_id(dev, + action_flags[i], + act, attr, &flow_err); + if (ret) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, flow_err.message ? + flow_err.message : + "PORT action validate check fail"); + ++actions_n; + action_flags[i] |= MLX5_FLOW_ACTION_PORT_ID; + break; + case RTE_FLOW_ACTION_TYPE_MARK: + ret = flow_dv_validate_action_mark(dev, act, + action_flags[i], + attr, &flow_err); + if (ret < 0) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, flow_err.message ? + flow_err.message : + "Mark action validate check fail"); + if (dev_conf->dv_xmeta_en != + MLX5_XMETA_MODE_LEGACY) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "Extend MARK action is " + "not supported. Please try use " + "default policy for meter."); + action_flags[i] |= MLX5_FLOW_ACTION_MARK; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_SET_TAG: + ret = flow_dv_validate_action_set_tag(dev, + act, action_flags[i], + attr, &flow_err); + if (ret) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, flow_err.message ? + flow_err.message : + "Set tag action validate check fail"); + action_flags[i] |= MLX5_FLOW_ACTION_SET_TAG; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_DROP: + ret = mlx5_flow_validate_action_drop + (action_flags[i], attr, &flow_err); + if (ret < 0) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, flow_err.message ? + flow_err.message : + "Drop action validate check fail"); + action_flags[i] |= MLX5_FLOW_ACTION_DROP; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_QUEUE: + /* + * Check whether extensive + * metadata feature is engaged. + */ + if (dev_conf->dv_flow_en && + (dev_conf->dv_xmeta_en != + MLX5_XMETA_MODE_LEGACY) && + mlx5_flow_ext_mreg_supported(dev)) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "Queue action with meta " + "is not supported. Please try use " + "default policy for meter."); + ret = mlx5_flow_validate_action_queue(act, + action_flags[i], dev, + attr, &flow_err); + if (ret < 0) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, flow_err.message ? + flow_err.message : + "Queue action validate check fail"); + action_flags[i] |= MLX5_FLOW_ACTION_QUEUE; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_RSS: + if (dev_conf->dv_flow_en && + (dev_conf->dv_xmeta_en != + MLX5_XMETA_MODE_LEGACY) && + mlx5_flow_ext_mreg_supported(dev)) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "RSS action with meta " + "is not supported. Please try use " + "default policy for meter."); + ret = mlx5_validate_action_rss(dev, act, + &flow_err); + if (ret < 0) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, flow_err.message ? + flow_err.message : + "RSS action validate check fail"); + action_flags[i] |= MLX5_FLOW_ACTION_RSS; + ++actions_n; + /* Either G or Y will set the RSS. */ + rss_color[i] = act->conf; + break; + case RTE_FLOW_ACTION_TYPE_JUMP: + ret = flow_dv_validate_action_jump(dev, + NULL, act, action_flags[i], + attr, true, &flow_err); + if (ret) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, flow_err.message ? + flow_err.message : + "Jump action validate check fail"); + ++actions_n; + action_flags[i] |= MLX5_FLOW_ACTION_JUMP; + break; + /* + * Only the last meter in the hierarchy will support + * the YELLOW color steering. Then in the meter policy + * actions list, there should be no other meter inside. + */ + case RTE_FLOW_ACTION_TYPE_METER: + if (i != RTE_COLOR_GREEN) + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, flow_err.message ? + flow_err.message : + "Meter hierarchy only supports GREEN color."); + mtr = act->conf; + ret = flow_dv_validate_policy_mtr_hierarchy(dev, + mtr->mtr_id, + action_flags[i], + is_rss, + &hierarchy_domain, + error); + if (ret) + return ret; + ++actions_n; + action_flags[i] |= + MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY; + break; + default: + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "Doesn't support optional action"); + } + } + if (action_flags[i] & MLX5_FLOW_ACTION_PORT_ID) + domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT; + else if ((action_flags[i] & + (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) || + (action_flags[i] & MLX5_FLOW_ACTION_MARK)) + /* + * Only support MLX5_XMETA_MODE_LEGACY + * so MARK action is only in ingress domain. + */ + domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT; + else if (action_flags[i] & + MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) + domain_color[i] = hierarchy_domain; + else + domain_color[i] = def_domain; + /* + * Non-termination actions only support NIC Tx domain. + * The adjustion should be skipped when there is no + * action or only END is provided. The default domains + * bit-mask is set to find the MIN intersection. + * The action flags checking should also be skipped. + */ + if ((def_green && i == RTE_COLOR_GREEN) || + (def_yellow && i == RTE_COLOR_YELLOW)) + continue; + /* + * Validate the drop action mutual exclusion + * with other actions. Drop action is mutually-exclusive + * with any other action, except for Count action. + */ + if ((action_flags[i] & MLX5_FLOW_ACTION_DROP) && + (action_flags[i] & ~MLX5_FLOW_ACTION_DROP)) { + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "Drop action is mutually-exclusive " + "with any other action"); + } + /* Eswitch has few restrictions on using items and actions */ + if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) { + if (!mlx5_flow_ext_mreg_supported(dev) && + action_flags[i] & MLX5_FLOW_ACTION_MARK) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "unsupported action MARK"); + if (action_flags[i] & MLX5_FLOW_ACTION_QUEUE) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "unsupported action QUEUE"); + if (action_flags[i] & MLX5_FLOW_ACTION_RSS) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "unsupported action RSS"); + if (!(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "no fate action is found"); + } else { + if (!(action_flags[i] & MLX5_FLOW_FATE_ACTIONS) && + (domain_color[i] & MLX5_MTR_DOMAIN_INGRESS_BIT)) { + if ((domain_color[i] & + MLX5_MTR_DOMAIN_EGRESS_BIT)) + domain_color[i] = + MLX5_MTR_DOMAIN_EGRESS_BIT; + else + return -rte_mtr_error_set(error, + ENOTSUP, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, + "no fate action is found"); + } + } + } + /* If both colors have RSS, the attributes should be the same. */ + if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN], + rss_color[RTE_COLOR_YELLOW])) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "policy RSS attr conflict"); + if (rss_color[RTE_COLOR_GREEN] || rss_color[RTE_COLOR_YELLOW]) + *is_rss = true; + /* "domain_color[C]" is non-zero for each color, default is ALL. */ + if (!def_green && !def_yellow && + domain_color[RTE_COLOR_GREEN] != domain_color[RTE_COLOR_YELLOW] && + !(action_flags[RTE_COLOR_GREEN] & MLX5_FLOW_ACTION_DROP) && + !(action_flags[RTE_COLOR_YELLOW] & MLX5_FLOW_ACTION_DROP)) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_POLICY, + NULL, "policy domains conflict"); + /* + * At least one color policy is listed in the actions, the domains + * to be supported should be the intersection. + */ + *domain_bitmap = domain_color[RTE_COLOR_GREEN] & + domain_color[RTE_COLOR_YELLOW]; + return 0; +} + static int flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags) { @@ -13851,14 +17844,25 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .remove = flow_dv_remove, .destroy = flow_dv_destroy, .query = flow_dv_query, - .create_mtr_tbls = flow_dv_create_mtr_tbl, - .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, - .destroy_policer_rules = flow_dv_destroy_policer_rules, + .create_mtr_tbls = flow_dv_create_mtr_tbls, + .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls, + .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls, + .create_meter = flow_dv_mtr_alloc, + .free_meter = flow_dv_aso_mtr_release_to_pool, + .validate_mtr_acts = flow_dv_validate_mtr_policy_acts, + .create_mtr_acts = flow_dv_create_mtr_policy_acts, + .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts, + .create_policy_rules = flow_dv_create_policy_rules, + .destroy_policy_rules = flow_dv_destroy_policy_rules, + .create_def_policy = flow_dv_create_def_policy, + .destroy_def_policy = flow_dv_destroy_def_policy, + .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare, + .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create, + .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, .counter_query = flow_dv_counter_query, - .get_aged_flows = flow_get_aged_flows, + .get_aged_flows = flow_dv_get_aged_flows, .action_validate = flow_dv_action_validate, .action_create = flow_dv_action_create, .action_destroy = flow_dv_action_destroy,