X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=1a74d5ac2b5c61e1e27fd0db7fafa82c0a0b3854;hb=c8238116ece889fe356c54930bca791091ab1938;hp=39bed291ae1d0abf103d2a54a647504b651a7ae5;hpb=abec3d9c4cde2c7cfcf697792e07691ae8e5c9a0;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 39bed291ae..1a74d5ac2b 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include @@ -35,7 +35,7 @@ #include "mlx5_rxtx.h" #include "rte_pmd_mlx5.h" -#ifdef HAVE_IBV_FLOW_DV_SUPPORT +#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 @@ -76,11 +76,17 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh, static int flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, - uint32_t encap_decap_idx); + uint32_t encap_decap_idx); static int flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, uint32_t port_id); +static void +flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss); + +static int +flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, + uint32_t rix_jump); /** * Initialize flow attributes structure according to flow items' types. @@ -276,45 +282,6 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, } } -/** - * Acquire the synchronizing object to protect multithreaded access - * to shared dv context. Lock occurs only if context is actually - * shared, i.e. we have multiport IB device and representors are - * created. - * - * @param[in] dev - * Pointer to the rte_eth_dev structure. - */ -static void -flow_dv_shared_lock(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - - if (sh->refcnt > 1) { - int ret; - - ret = pthread_mutex_lock(&sh->dv_mutex); - MLX5_ASSERT(!ret); - (void)ret; - } -} - -static void -flow_dv_shared_unlock(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - - if (sh->refcnt > 1) { - int ret; - - ret = pthread_mutex_unlock(&sh->dv_mutex); - MLX5_ASSERT(!ret); - (void)ret; - } -} - /* Update VLAN's VID/PCP based on input rte_flow_action. * * @param[in] action @@ -464,6 +431,7 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, (int)dcopy->offset < 0 ? off_b : dcopy->offset; /* Convert entire record to big-endian format. */ actions[i].data1 = rte_cpu_to_be_32(actions[i].data1); + ++dcopy; } else { MLX5_ASSERT(item->spec); data = flow_dv_fetch_field((const uint8_t *)item->spec + @@ -992,7 +960,7 @@ flow_dv_convert_action_set_reg RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many items to modify"); MLX5_ASSERT(conf->id != REG_NON); - MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field)); + MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field)); actions[i] = (struct mlx5_modification_cmd) { .action_type = MLX5_MODIFICATION_TYPE_SET, .field = reg_to_field[conf->id], @@ -1254,6 +1222,7 @@ flow_dv_convert_action_set_meta if (reg < 0) return reg; + MLX5_ASSERT(reg != REG_NON); /* * In datapath code there is no endianness * coversions for perfromance reasons, all @@ -1356,6 +1325,511 @@ flow_dv_convert_action_modify_ipv6_dscp MLX5_MODIFICATION_TYPE_SET, error); } +static void +mlx5_flow_field_id_to_modify_info + (const struct rte_flow_action_modify_data *data, + struct field_modify_info *info, + uint32_t *mask, uint32_t *value, uint32_t width, + struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + uint32_t idx = 0; + switch (data->field) { + case RTE_FLOW_FIELD_START: + /* not supported yet */ + MLX5_ASSERT(false); + break; + case RTE_FLOW_FIELD_MAC_DST: + if (mask) { + if (data->offset < 32) { + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DMAC_47_16}; + mask[idx] = 0xffffffff; + if (width < 32) { + mask[idx] = mask[idx] << (32 - width); + width = 0; + } else { + width -= 32; + } + if (!width) + break; + ++idx; + } + info[idx] = (struct field_modify_info){2, 4 * idx, + MLX5_MODI_OUT_DMAC_15_0}; + mask[idx] = (width) ? 0x0000ffff : 0x0; + if (width < 16) + mask[idx] = (mask[idx] << (16 - width)) & + 0x0000ffff; + } else { + if (data->offset < 32) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DMAC_47_16}; + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_DMAC_15_0}; + } + break; + case RTE_FLOW_FIELD_MAC_SRC: + if (mask) { + if (data->offset < 32) { + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SMAC_47_16}; + mask[idx] = 0xffffffff; + if (width < 32) { + mask[idx] = mask[idx] << (32 - width); + width = 0; + } else { + width -= 32; + } + if (!width) + break; + ++idx; + } + info[idx] = (struct field_modify_info){2, 4 * idx, + MLX5_MODI_OUT_SMAC_15_0}; + mask[idx] = (width) ? 0x0000ffff : 0x0; + if (width < 16) + mask[idx] = (mask[idx] << (16 - width)) & + 0x0000ffff; + } else { + if (data->offset < 32) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SMAC_47_16}; + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_SMAC_15_0}; + } + break; + case RTE_FLOW_FIELD_VLAN_TYPE: + /* not supported yet */ + break; + case RTE_FLOW_FIELD_VLAN_ID: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_FIRST_VID}; + if (mask) { + mask[idx] = 0x00000fff; + if (width < 12) + mask[idx] = (mask[idx] << (12 - width)) & + 0x00000fff; + } + break; + case RTE_FLOW_FIELD_MAC_TYPE: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_ETHERTYPE}; + if (mask) { + mask[idx] = 0x0000ffff; + if (width < 16) + mask[idx] = (mask[idx] << (16 - width)) & + 0x0000ffff; + } + break; + case RTE_FLOW_FIELD_IPV4_DSCP: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_IP_DSCP}; + if (mask) { + mask[idx] = 0x0000003f; + if (width < 6) + mask[idx] = (mask[idx] << (6 - width)) & + 0x0000003f; + } + break; + case RTE_FLOW_FIELD_IPV4_TTL: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_IPV4_TTL}; + if (mask) { + mask[idx] = 0x000000ff; + if (width < 8) + mask[idx] = (mask[idx] << (8 - width)) & + 0x000000ff; + } + break; + case RTE_FLOW_FIELD_IPV4_SRC: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV4}; + if (mask) { + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = mask[idx] << (32 - width); + } + break; + case RTE_FLOW_FIELD_IPV4_DST: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV4}; + if (mask) { + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = mask[idx] << (32 - width); + } + break; + case RTE_FLOW_FIELD_IPV6_DSCP: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_IP_DSCP}; + if (mask) { + mask[idx] = 0x0000003f; + if (width < 6) + mask[idx] = (mask[idx] << (6 - width)) & + 0x0000003f; + } + break; + case RTE_FLOW_FIELD_IPV6_HOPLIMIT: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_IPV6_HOPLIMIT}; + if (mask) { + mask[idx] = 0x000000ff; + if (width < 8) + mask[idx] = (mask[idx] << (8 - width)) & + 0x000000ff; + } + break; + case RTE_FLOW_FIELD_IPV6_SRC: + if (mask) { + if (data->offset < 32) { + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_127_96}; + mask[idx] = 0xffffffff; + if (width < 32) { + mask[idx] = mask[idx] << (32 - width); + width = 0; + } else { + width -= 32; + } + if (!width) + break; + ++idx; + } + if (data->offset < 64) { + info[idx] = (struct field_modify_info){4, + 4 * idx, + MLX5_MODI_OUT_SIPV6_95_64}; + mask[idx] = 0xffffffff; + if (width < 32) { + mask[idx] = mask[idx] << (32 - width); + width = 0; + } else { + width -= 32; + } + if (!width) + break; + ++idx; + } + if (data->offset < 96) { + info[idx] = (struct field_modify_info){4, + 8 * idx, + MLX5_MODI_OUT_SIPV6_63_32}; + mask[idx] = 0xffffffff; + if (width < 32) { + mask[idx] = mask[idx] << (32 - width); + width = 0; + } else { + width -= 32; + } + if (!width) + break; + ++idx; + } + info[idx] = (struct field_modify_info){4, 12 * idx, + MLX5_MODI_OUT_SIPV6_31_0}; + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = mask[idx] << (32 - width); + } else { + if (data->offset < 32) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_127_96}; + if (data->offset < 64) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_95_64}; + if (data->offset < 96) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_63_32}; + if (data->offset < 128) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_31_0}; + } + break; + case RTE_FLOW_FIELD_IPV6_DST: + if (mask) { + if (data->offset < 32) { + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_127_96}; + mask[idx] = 0xffffffff; + if (width < 32) { + mask[idx] = mask[idx] << (32 - width); + width = 0; + } else { + width -= 32; + } + if (!width) + break; + ++idx; + } + if (data->offset < 64) { + info[idx] = (struct field_modify_info){4, + 4 * idx, + MLX5_MODI_OUT_DIPV6_95_64}; + mask[idx] = 0xffffffff; + if (width < 32) { + mask[idx] = mask[idx] << (32 - width); + width = 0; + } else { + width -= 32; + } + if (!width) + break; + ++idx; + } + if (data->offset < 96) { + info[idx] = (struct field_modify_info){4, + 8 * idx, + MLX5_MODI_OUT_DIPV6_63_32}; + mask[idx] = 0xffffffff; + if (width < 32) { + mask[idx] = mask[idx] << (32 - width); + width = 0; + } else { + width -= 32; + } + if (!width) + break; + ++idx; + } + info[idx] = (struct field_modify_info){4, 12 * idx, + MLX5_MODI_OUT_DIPV6_31_0}; + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = mask[idx] << (32 - width); + } else { + if (data->offset < 32) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_127_96}; + if (data->offset < 64) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_95_64}; + if (data->offset < 96) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_63_32}; + if (data->offset < 128) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_31_0}; + } + break; + case RTE_FLOW_FIELD_TCP_PORT_SRC: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_TCP_SPORT}; + if (mask) { + mask[idx] = 0x0000ffff; + if (width < 16) + mask[idx] = (mask[idx] << (16 - width)) & + 0x0000ffff; + } + break; + case RTE_FLOW_FIELD_TCP_PORT_DST: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_TCP_DPORT}; + if (mask) { + mask[idx] = 0x0000ffff; + if (width < 16) + mask[idx] = (mask[idx] << (16 - width)) & + 0x0000ffff; + } + break; + case RTE_FLOW_FIELD_TCP_SEQ_NUM: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_TCP_SEQ_NUM}; + if (mask) { + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = (mask[idx] << (32 - width)); + } + break; + case RTE_FLOW_FIELD_TCP_ACK_NUM: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_TCP_ACK_NUM}; + if (mask) { + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = (mask[idx] << (32 - width)); + } + break; + case RTE_FLOW_FIELD_TCP_FLAGS: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_TCP_FLAGS}; + if (mask) { + mask[idx] = 0x0000003f; + if (width < 6) + mask[idx] = (mask[idx] << (6 - width)) & + 0x0000003f; + } + break; + case RTE_FLOW_FIELD_UDP_PORT_SRC: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_UDP_SPORT}; + if (mask) { + mask[idx] = 0x0000ffff; + if (width < 16) + mask[idx] = (mask[idx] << (16 - width)) & + 0x0000ffff; + } + break; + case RTE_FLOW_FIELD_UDP_PORT_DST: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_UDP_DPORT}; + if (mask) { + mask[idx] = 0x0000ffff; + if (width < 16) + mask[idx] = (mask[idx] << (16 - width)) & + 0x0000ffff; + } + break; + case RTE_FLOW_FIELD_VXLAN_VNI: + /* not supported yet */ + break; + case RTE_FLOW_FIELD_GENEVE_VNI: + /* not supported yet*/ + break; + case RTE_FLOW_FIELD_GTP_TEID: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_GTP_TEID}; + if (mask) { + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = mask[idx] << (32 - width); + } + break; + case RTE_FLOW_FIELD_TAG: + { + int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, + data->level, error); + if (reg < 0) + return; + MLX5_ASSERT(reg != REG_NON); + MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); + info[idx] = (struct field_modify_info){4, 0, + reg_to_field[reg]}; + if (mask) { + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = mask[idx] << (32 - width); + } + } + break; + case RTE_FLOW_FIELD_MARK: + { + int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, + 0, error); + if (reg < 0) + return; + MLX5_ASSERT(reg != REG_NON); + MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); + info[idx] = (struct field_modify_info){4, 0, + reg_to_field[reg]}; + if (mask) { + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = mask[idx] << (32 - width); + } + } + break; + case RTE_FLOW_FIELD_META: + { + int reg = flow_dv_get_metadata_reg(dev, attr, error); + if (reg < 0) + return; + MLX5_ASSERT(reg != REG_NON); + MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); + info[idx] = (struct field_modify_info){4, 0, + reg_to_field[reg]}; + if (mask) { + mask[idx] = 0xffffffff; + if (width < 32) + mask[idx] = mask[idx] << (32 - width); + } + } + break; + case RTE_FLOW_FIELD_POINTER: + for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) { + if (mask[idx]) { + memcpy(&value[idx], + (void *)(uintptr_t)data->value, 32); + value[idx] = RTE_BE32(value[idx]); + break; + } + } + break; + case RTE_FLOW_FIELD_VALUE: + for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) { + if (mask[idx]) { + value[idx] = RTE_BE32((uint32_t)data->value); + break; + } + } + break; + default: + MLX5_ASSERT(false); + break; + } +} + +/** + * Convert modify_field action to DV specification. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_field + (struct rte_eth_dev *dev, + struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + const struct rte_flow_action_modify_field *conf = + (const struct rte_flow_action_modify_field *)(action->conf); + struct rte_flow_item item; + struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = { + {0, 0, 0} }; + struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = { + {0, 0, 0} }; + uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; + uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; + uint32_t type; + + if (conf->src.field == RTE_FLOW_FIELD_POINTER || + conf->src.field == RTE_FLOW_FIELD_VALUE) { + type = MLX5_MODIFICATION_TYPE_SET; + /** For SET fill the destination field (field) first. */ + mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask, + value, conf->width, dev, attr, error); + /** Then copy immediate value from source as per mask. */ + mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask, + value, conf->width, dev, attr, error); + item.spec = &value; + } else { + type = MLX5_MODIFICATION_TYPE_COPY; + /** For COPY fill the destination field (dcopy) without mask. */ + mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL, + value, conf->width, dev, attr, error); + /** Then construct the source field (field) with mask. */ + mlx5_flow_field_id_to_modify_info(&conf->src, field, mask, + value, conf->width, dev, attr, error); + } + item.mask = &mask; + return flow_dv_convert_modify_action(&item, + field, dcopy, resource, type, error); +} + /** * Validate MARK item. * @@ -1475,6 +1949,10 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, reg = flow_dv_get_metadata_reg(dev, attr, error); if (reg < 0) return reg; + if (reg == REG_NON) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "unavalable extended metadata register"); if (reg == REG_B) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, @@ -1482,13 +1960,20 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, "isn't supported"); if (reg != REG_A) nic_mask.data = priv->sh->dv_meta_mask; - } else if (attr->transfer) { - return rte_flow_error_set(error, ENOTSUP, + } else { + if (attr->transfer) + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, "extended metadata feature " "should be enabled when " "meta item is requested " "with e-switch mode "); + if (attr->ingress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "match on metadata for ingress " + "is not supported in legacy " + "metadata mode"); } if (!mask) mask = &rte_flow_item_meta_mask; @@ -1791,6 +2276,72 @@ flow_dv_validate_item_gtp(struct rte_eth_dev *dev, MLX5_ITEM_RANGE_NOT_ACCEPTED, error); } +/** + * Validate GTP PSC item. + * + * @param[in] item + * Item specification. + * @param[in] last_item + * Previous validated item in the pattern items. + * @param[in] gtp_item + * Previous GTP item specification. + * @param[in] attr + * Pointer to flow attributes. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item, + uint64_t last_item, + const struct rte_flow_item *gtp_item, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + const struct rte_flow_item_gtp *gtp_spec; + const struct rte_flow_item_gtp *gtp_mask; + const struct rte_flow_item_gtp_psc *spec; + const struct rte_flow_item_gtp_psc *mask; + const struct rte_flow_item_gtp_psc nic_mask = { + .pdu_type = 0xFF, + .qfi = 0xFF, + }; + + if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP)) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "GTP PSC item must be preceded with GTP item"); + gtp_spec = gtp_item->spec; + gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask; + /* GTP spec and E flag is requested to match zero. */ + if (gtp_spec && + (gtp_mask->v_pt_rsv_flags & + ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG)) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "GTP E flag must be 1 to match GTP PSC"); + /* Check the flow is not created in group zero. */ + if (!attr->transfer && !attr->group) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "GTP PSC is not supported for group 0"); + /* GTP spec is here and E flag is requested to match zero. */ + if (!item->spec) + return 0; + spec = item->spec; + mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask; + if (spec->pdu_type > MLX5_GTP_EXT_MAX_PDU_TYPE) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "PDU type should be smaller than 16"); + return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_gtp_psc), + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); +} + /** * Validate IPV4 item. * Use existing validation function mlx5_flow_validate_item_ipv4(), and @@ -2407,6 +2958,11 @@ flow_dv_validate_action_mark(struct rte_eth_dev *dev, const struct rte_flow_action_mark *mark = action->conf; int ret; + if (is_tunnel_offload_active(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "no mark action " + "if tunnel offload active"); /* Fall back if no extended metadata register support. */ if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) return mlx5_flow_validate_action_mark(action, action_flags, @@ -2483,6 +3039,10 @@ flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, reg = flow_dv_get_metadata_reg(dev, attr, error); if (reg < 0) return reg; + if (reg == REG_NON) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "unavalable extended metadata register"); if (reg != REG_A && reg != REG_B) { struct mlx5_priv *priv = dev->data->dev_private; @@ -2565,6 +3125,10 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to rte_eth_dev structure. + * @param[in] action + * Pointer to the action structure. + * @param[in] action_flags + * Holds the actions detected until now. * @param[out] error * Pointer to error structure. * @@ -2573,12 +3137,25 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, */ static int flow_dv_validate_action_count(struct rte_eth_dev *dev, + const struct rte_flow_action *action, + uint64_t action_flags, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_action_count *count; if (!priv->config.devx) goto notsup_err; + if (action_flags & MLX5_FLOW_ACTION_COUNT) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "duplicate count actions set"); + count = (const struct rte_flow_action_count *)action->conf; + if (count && count->shared && (action_flags & MLX5_FLOW_ACTION_AGE) && + !priv->sh->flow_hit_aso_en) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "old age and shared count combination is not supported"); #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS return 0; #endif @@ -2640,6 +3217,10 @@ flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, * Pointer to the rte_eth_dev structure. * @param[in] action_flags * Holds the actions detected until now. + * @param[in] action + * Pointer to the action structure. + * @param[in] item_flags + * Holds the items detected. * @param[in] attr * Pointer to flow attributes * @param[out] error @@ -2651,6 +3232,8 @@ flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, static int flow_dv_validate_action_decap(struct rte_eth_dev *dev, uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, const struct rte_flow_attr *attr, struct rte_flow_error *error) { @@ -2684,6 +3267,11 @@ flow_dv_validate_action_decap(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "decap action for VF representor " "not supported on NIC table"); + if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP && + !(item_flags & MLX5_FLOW_LAYER_VXLAN)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "VXLAN item should be present for VXLAN decap"); return 0; } @@ -2704,6 +3292,10 @@ const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,}; * Holds the actions detected until now. * @param[out] actions_n * pointer to the number of actions counter. + * @param[in] action + * Pointer to the action structure. + * @param[in] item_flags + * Holds the items detected. * @param[out] error * Pointer to error structure. * @@ -2716,7 +3308,8 @@ flow_dv_validate_action_raw_encap_decap const struct rte_flow_action_raw_decap *decap, const struct rte_flow_action_raw_encap *encap, const struct rte_flow_attr *attr, uint64_t *action_flags, - int *actions_n, struct rte_flow_error *error) + int *actions_n, const struct rte_flow_action *action, + uint64_t item_flags, struct rte_flow_error *error) { const struct mlx5_priv *priv = dev->data->dev_private; int ret; @@ -2751,8 +3344,8 @@ flow_dv_validate_action_raw_encap_decap "encap combination"); } if (decap) { - ret = flow_dv_validate_action_decap(dev, *action_flags, attr, - error); + ret = flow_dv_validate_action_decap(dev, *action_flags, action, + item_flags, attr, error); if (ret < 0) return ret; *action_flags |= MLX5_FLOW_ACTION_DECAP; @@ -2808,8 +3401,7 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused, cache_resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource, entry); - if (resource->entry.key == cache_resource->entry.key && - resource->reformat_type == cache_resource->reformat_type && + if (resource->reformat_type == cache_resource->reformat_type && resource->ft_type == cache_resource->ft_type && resource->flags == cache_resource->flags && resource->size == cache_resource->size && @@ -2902,26 +3494,41 @@ flow_dv_encap_decap_resource_register struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_hlist_entry *entry; - union mlx5_flow_encap_decap_key encap_decap_key = { + union { + struct { + uint32_t ft_type:8; + uint32_t refmt_type:8; + /* + * Header reformat actions can be shared between + * non-root tables. One bit to indicate non-root + * table or not. + */ + uint32_t is_root:1; + uint32_t reserve:15; + }; + uint32_t v32; + } encap_decap_key = { { .ft_type = resource->ft_type, .refmt_type = resource->reformat_type, - .buf_size = resource->size, - .table_level = !!dev_flow->dv.group, - .cksum = 0, + .is_root = !!dev_flow->dv.group, + .reserve = 0, } }; struct mlx5_flow_cb_ctx ctx = { .error = error, .data = resource, }; + uint64_t key64; resource->flags = dev_flow->dv.group ? 0 : 1; - encap_decap_key.cksum = __rte_raw_cksum(resource->buf, - resource->size, 0); - resource->entry.key = encap_decap_key.v64; - entry = mlx5_hlist_register(sh->encaps_decaps, resource->entry.key, - &ctx); + key64 = __rte_raw_cksum(&encap_decap_key.v32, + sizeof(encap_decap_key.v32), 0); + if (resource->reformat_type != + MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 && + resource->size) + key64 = __rte_raw_cksum(resource->buf, resource->size, key64); + entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx); if (!entry) return -rte_errno; resource = container_of(entry, typeof(*resource), entry); @@ -3594,8 +4201,6 @@ flow_dv_create_action_push_vlan(struct rte_eth_dev *dev, (dev, &res, dev_flow, error); } -static int fdb_mirror; - /** * Validate the modify-header actions. * @@ -3623,12 +4228,6 @@ flow_dv_validate_action_modify_hdr(const uint64_t action_flags, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can't have encap action before" " modify action"); - if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't support sample action before" - " modify action for E-Switch" - " mirroring"); return 0; } @@ -3916,38 +4515,197 @@ flow_dv_validate_action_modify_ttl(const uint64_t action_flags, return ret; } +static int +mlx5_flow_item_field_width(enum rte_flow_field_id field) +{ + switch (field) { + case RTE_FLOW_FIELD_START: + return 32; + case RTE_FLOW_FIELD_MAC_DST: + case RTE_FLOW_FIELD_MAC_SRC: + return 48; + case RTE_FLOW_FIELD_VLAN_TYPE: + return 16; + case RTE_FLOW_FIELD_VLAN_ID: + return 12; + case RTE_FLOW_FIELD_MAC_TYPE: + return 16; + case RTE_FLOW_FIELD_IPV4_DSCP: + return 6; + case RTE_FLOW_FIELD_IPV4_TTL: + return 8; + case RTE_FLOW_FIELD_IPV4_SRC: + case RTE_FLOW_FIELD_IPV4_DST: + return 32; + case RTE_FLOW_FIELD_IPV6_DSCP: + return 6; + case RTE_FLOW_FIELD_IPV6_HOPLIMIT: + return 8; + case RTE_FLOW_FIELD_IPV6_SRC: + case RTE_FLOW_FIELD_IPV6_DST: + return 128; + case RTE_FLOW_FIELD_TCP_PORT_SRC: + case RTE_FLOW_FIELD_TCP_PORT_DST: + return 16; + case RTE_FLOW_FIELD_TCP_SEQ_NUM: + case RTE_FLOW_FIELD_TCP_ACK_NUM: + return 32; + case RTE_FLOW_FIELD_TCP_FLAGS: + return 6; + case RTE_FLOW_FIELD_UDP_PORT_SRC: + case RTE_FLOW_FIELD_UDP_PORT_DST: + return 16; + case RTE_FLOW_FIELD_VXLAN_VNI: + case RTE_FLOW_FIELD_GENEVE_VNI: + return 24; + case RTE_FLOW_FIELD_GTP_TEID: + case RTE_FLOW_FIELD_TAG: + return 32; + case RTE_FLOW_FIELD_MARK: + return 24; + case RTE_FLOW_FIELD_META: + case RTE_FLOW_FIELD_POINTER: + case RTE_FLOW_FIELD_VALUE: + return 32; + default: + MLX5_ASSERT(false); + } + return 0; +} + /** - * Validate jump action. + * Validate the generic modify field actions. * - * @param[in] action - * Pointer to the jump action. * @param[in] action_flags * Holds the actions detected until now. - * @param[in] attributes - * Pointer to flow attributes - * @param[in] external - * Action belongs to flow rule created by request external to PMD. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. * @param[out] error * Pointer to error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * Number of header fields to modify (0 or more) on success, + * a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_jump(struct rte_eth_dev *dev, - const struct mlx5_flow_tunnel *tunnel, - const struct rte_flow_action *action, - uint64_t action_flags, - const struct rte_flow_attr *attributes, - bool external, struct rte_flow_error *error) +flow_dv_validate_action_modify_field(const uint64_t action_flags, + const struct rte_flow_action *action, + struct rte_flow_error *error) { - uint32_t target_group, table; int ret = 0; - struct flow_grp_info grp_info = { - .external = !!external, - .transfer = !!attributes->transfer, - .fdb_def_rule = 1, - .std_tbl_fix = 0 + const struct rte_flow_action_modify_field *action_modify_field = + action->conf; + uint32_t dst_width = + mlx5_flow_item_field_width(action_modify_field->dst.field); + uint32_t src_width = + mlx5_flow_item_field_width(action_modify_field->src.field); + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (ret) + return ret; + + if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE && + action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) { + if (action_modify_field->dst.offset >= dst_width || + (action_modify_field->dst.offset % 32)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "destination offset is too big" + " or not aligned to 4 bytes"); + if (action_modify_field->dst.level && + action_modify_field->dst.field != RTE_FLOW_FIELD_TAG) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot modify inner headers"); + } + if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE && + action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) { + if (action_modify_field->src.offset >= src_width || + (action_modify_field->src.offset % 32)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "source offset is too big" + " or not aligned to 4 bytes"); + if (action_modify_field->src.level && + action_modify_field->src.field != RTE_FLOW_FIELD_TAG) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot copy from inner headers"); + } + if (action_modify_field->width == 0) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "width is required for modify action"); + if (action_modify_field->dst.field == + action_modify_field->src.field) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "source and destination fields" + " cannot be the same"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE || + action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "immediate value or a pointer to it" + " cannot be used as a destination"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_START || + action_modify_field->src.field == RTE_FLOW_FIELD_START) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "modifications of an arbitrary" + " place in a packet is not supported"); + if (action_modify_field->operation != RTE_FLOW_MODIFY_SET) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "add and sub operations" + " are not supported"); + return (action_modify_field->width / 32) + + !!(action_modify_field->width % 32); +} + +/** + * Validate jump action. + * + * @param[in] action + * Pointer to the jump action. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] attributes + * Pointer to flow attributes + * @param[in] external + * Action belongs to flow rule created by request external to PMD. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_jump(struct rte_eth_dev *dev, + const struct mlx5_flow_tunnel *tunnel, + const struct rte_flow_action *action, + uint64_t action_flags, + const struct rte_flow_attr *attributes, + bool external, struct rte_flow_error *error) +{ + uint32_t target_group, table; + int ret = 0; + struct flow_grp_info grp_info = { + .external = !!external, + .transfer = !!attributes->transfer, + .fdb_def_rule = 1, + .std_tbl_fix = 0 }; if (action_flags & (MLX5_FLOW_FATE_ACTIONS | MLX5_FLOW_FATE_ESWITCH_ACTIONS)) @@ -3959,12 +4717,6 @@ flow_dv_validate_action_jump(struct rte_eth_dev *dev, return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "jump with meter not support"); - if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "E-Switch mirroring can't support" - " Sample action and jump action in" - " same flow now"); if (!action->conf) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, @@ -3972,7 +4724,7 @@ flow_dv_validate_action_jump(struct rte_eth_dev *dev, target_group = ((const struct rte_flow_action_jump *)action->conf)->group; ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table, - grp_info, error); + &grp_info, error); if (ret) return ret; if (attributes->group == target_group && @@ -4166,7 +4918,8 @@ flow_dv_validate_action_age(uint64_t action_flags, struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_age *age = action->conf; - if (!priv->config.devx || priv->sh->cmng.counter_fallback) + if (!priv->config.devx || (priv->sh->cmng.counter_fallback && + !priv->sh->aso_age_mng)) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -4331,7 +5084,7 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused, /** * Validate the sample action. * - * @param[in] action_flags + * @param[in, out] action_flags * Holds the actions detected until now. * @param[in] action * Pointer to the sample action. @@ -4339,6 +5092,16 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused, * Pointer to the Ethernet device structure. * @param[in] attr * Attributes of flow that includes this action. + * @param[in] item_flags + * Holds the items detected. + * @param[in] rss + * Pointer to the RSS action. + * @param[out] sample_rss + * Pointer to the RSS action in sample action list. + * @param[out] count + * Pointer to the COUNT action in sample action list. + * @param[out] fdb_mirror_limit + * Pointer to the FDB mirror limitation flag. * @param[out] error * Pointer to error structure. * @@ -4346,10 +5109,15 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_sample(uint64_t action_flags, +flow_dv_validate_action_sample(uint64_t *action_flags, const struct rte_flow_action *action, struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + uint64_t item_flags, + const struct rte_flow_action_rss *rss, + const struct rte_flow_action_rss **sample_rss, + const struct rte_flow_action_count **count, + int *fdb_mirror_limit, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; @@ -4360,7 +5128,6 @@ flow_dv_validate_action_sample(uint64_t action_flags, uint16_t queue_index = 0xFFFF; int actions_n = 0; int ret; - fdb_mirror = 0; if (!sample) return rte_flow_error_set(error, EINVAL, @@ -4375,17 +5142,17 @@ flow_dv_validate_action_sample(uint64_t action_flags, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "sample action not supported"); - if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + if (*action_flags & MLX5_FLOW_ACTION_SAMPLE) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Multiple sample actions not " "supported"); - if (action_flags & MLX5_FLOW_ACTION_METER) + if (*action_flags & MLX5_FLOW_ACTION_METER) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "wrong action order, meter should " "be after sample action"); - if (action_flags & MLX5_FLOW_ACTION_JUMP) + if (*action_flags & MLX5_FLOW_ACTION_JUMP) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "wrong action order, jump should " @@ -4409,6 +5176,28 @@ flow_dv_validate_action_sample(uint64_t action_flags, sub_action_flags |= MLX5_FLOW_ACTION_QUEUE; ++actions_n; break; + case RTE_FLOW_ACTION_TYPE_RSS: + *sample_rss = act->conf; + ret = mlx5_flow_validate_action_rss(act, + sub_action_flags, + dev, attr, + item_flags, + error); + if (ret < 0) + return ret; + if (rss && *sample_rss && + ((*sample_rss)->level != rss->level || + (*sample_rss)->types != rss->types)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "Can't use the different RSS types " + "or level in the same flow"); + if (*sample_rss != NULL && (*sample_rss)->queue_num) + queue_index = (*sample_rss)->queue[0]; + sub_action_flags |= MLX5_FLOW_ACTION_RSS; + ++actions_n; + break; case RTE_FLOW_ACTION_TYPE_MARK: ret = flow_dv_validate_action_mark(dev, act, sub_action_flags, @@ -4423,10 +5212,15 @@ flow_dv_validate_action_sample(uint64_t action_flags, ++actions_n; break; case RTE_FLOW_ACTION_TYPE_COUNT: - ret = flow_dv_validate_action_count(dev, error); + ret = flow_dv_validate_action_count + (dev, act, + *action_flags | sub_action_flags, + error); if (ret < 0) return ret; + *count = act->conf; sub_action_flags |= MLX5_FLOW_ACTION_COUNT; + *action_flags |= MLX5_FLOW_ACTION_COUNT; ++actions_n; break; case RTE_FLOW_ACTION_TYPE_PORT_ID: @@ -4443,7 +5237,7 @@ flow_dv_validate_action_sample(uint64_t action_flags, case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: ret = flow_dv_validate_action_raw_encap_decap (dev, NULL, act->conf, attr, &sub_action_flags, - &actions_n, error); + &actions_n, action, item_flags, error); if (ret < 0) return ret; ++actions_n; @@ -4457,7 +5251,8 @@ flow_dv_validate_action_sample(uint64_t action_flags, } } if (attr->ingress && !attr->transfer) { - if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE)) + if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE | + MLX5_FLOW_ACTION_RSS))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -4478,18 +5273,25 @@ flow_dv_validate_action_sample(uint64_t action_flags, "E-Switch doesn't support " "any optional action " "for sampling"); - fdb_mirror = 1; if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "unsupported action QUEUE"); + if (sub_action_flags & MLX5_FLOW_ACTION_RSS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported action QUEUE"); if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "E-Switch must has a dest " "port for mirroring"); + if (!priv->config.hca_attr.reg_c_preserve && + priv->representor_id != -1) + *fdb_mirror_limit = 1; } /* Continue validation for Xcap actions.*/ if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) && @@ -4544,6 +5346,7 @@ flow_dv_modify_hdr_resource_register .error = error, .data = resource, }; + uint64_t key64; resource->flags = dev_flow->dv.group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; @@ -4552,8 +5355,8 @@ flow_dv_modify_hdr_resource_register return rte_flow_error_set(error, EOVERFLOW, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many modify header items"); - resource->entry.key = __rte_raw_cksum(&resource->ft_type, key_len, 0); - entry = mlx5_hlist_register(sh->modify_cmds, resource->entry.key, &ctx); + key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0); + entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx); if (!entry) return -rte_errno; resource = container_of(entry, typeof(*resource), entry); @@ -5075,7 +5878,7 @@ flow_dv_counter_remove_from_age(struct rte_eth_dev *dev, * Index to the counter handler. */ static void -flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter) +flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_counter_pool *pool = NULL; @@ -5139,11 +5942,11 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, const struct mlx5_flow_tunnel *tunnel, const struct rte_flow_attr *attributes, - struct flow_grp_info grp_info, + const struct flow_grp_info *grp_info, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - uint32_t priority_max = priv->config.flow_prio - 1; + uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes); int ret = 0; #ifndef HAVE_MLX5DV_DR @@ -5164,8 +5967,8 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, if (!table) ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; #endif - if (attributes->priority != MLX5_FLOW_PRIO_RSVD && - attributes->priority >= priority_max) + if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && + attributes->priority > lowest_priority) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, NULL, @@ -5230,10 +6033,17 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, uint16_t ether_type = 0; int actions_n = 0; uint8_t item_ipv6_proto = 0; + int fdb_mirror_limit = 0; + int modify_after_mirror = 0; + const struct rte_flow_item *geneve_item = NULL; const struct rte_flow_item *gre_item = NULL; + const struct rte_flow_item *gtp_item = NULL; const struct rte_flow_action_raw_decap *decap; const struct rte_flow_action_raw_encap *encap; - const struct rte_flow_action_rss *rss; + const struct rte_flow_action_rss *rss = NULL; + const struct rte_flow_action_rss *sample_rss = NULL; + const struct rte_flow_action_count *count = NULL; + const struct rte_flow_action_count *sample_count = NULL; const struct rte_flow_item_tcp nic_tcp_mask = { .hdr = { .tcp_flags = 0xFF, @@ -5270,7 +6080,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, struct mlx5_dev_config *dev_conf = &priv->config; uint16_t queue_index = 0xFFFF; const struct rte_flow_item_vlan *vlan_m = NULL; - int16_t rw_act_num = 0; + uint32_t rw_act_num = 0; uint64_t is_root; const struct mlx5_flow_tunnel *tunnel; struct flow_grp_info grp_info = { @@ -5292,9 +6102,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, } else { tunnel = NULL; } + if (tunnel && priv->representor) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "decap not supported " + "for VF representor"); grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate (dev, tunnel, attr, items, actions); - ret = flow_dv_validate_attributes(dev, tunnel, attr, grp_info, error); + ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error); if (ret < 0) return ret; is_root = (uint64_t)ret; @@ -5506,8 +6321,19 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + geneve_item = items; last_item = MLX5_FLOW_LAYER_GENEVE; break; + case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: + ret = mlx5_flow_validate_item_geneve_opt(items, + last_item, + geneve_item, + dev, + error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_LAYER_GENEVE_OPT; + break; case RTE_FLOW_ITEM_TYPE_MPLS: ret = mlx5_flow_validate_item_mpls(dev, items, item_flags, @@ -5563,8 +6389,17 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + gtp_item = items; last_item = MLX5_FLOW_LAYER_GTP; break; + case RTE_FLOW_ITEM_TYPE_GTP_PSC: + ret = flow_dv_validate_item_gtp_psc(items, last_item, + gtp_item, attr, + error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_LAYER_GTP_PSC; + break; case RTE_FLOW_ITEM_TYPE_ECPRI: /* Capacity will be checked in the translate stage. */ ret = mlx5_flow_validate_item_ecpri(items, item_flags, @@ -5621,6 +6456,9 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; action_flags |= MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK_EXT; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; + } else { action_flags |= MLX5_FLOW_ACTION_FLAG; ++actions_n; @@ -5640,6 +6478,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; action_flags |= MLX5_FLOW_ACTION_MARK | MLX5_FLOW_ACTION_MARK_EXT; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; } else { action_flags |= MLX5_FLOW_ACTION_MARK; ++actions_n; @@ -5655,6 +6495,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_META; rw_act_num += MLX5_ACT_NUM_SET_META; break; @@ -5667,6 +6509,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_TAG; rw_act_num += MLX5_ACT_NUM_SET_TAG; break; @@ -5697,6 +6541,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + if (rss && sample_rss && + (sample_rss->level != rss->level || + sample_rss->types != rss->types)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "Can't use the different RSS types " + "or level in the same flow"); if (rss != NULL && rss->queue_num) queue_index = rss->queue[0]; action_flags |= MLX5_FLOW_ACTION_RSS; @@ -5712,9 +6564,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; break; case RTE_FLOW_ACTION_TYPE_COUNT: - ret = flow_dv_validate_action_count(dev, error); + ret = flow_dv_validate_action_count(dev, actions, + action_flags, + error); if (ret < 0) return ret; + count = actions->conf; action_flags |= MLX5_FLOW_ACTION_COUNT; ++actions_n; break; @@ -5771,6 +6626,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: ret = flow_dv_validate_action_decap(dev, action_flags, + actions, item_flags, attr, error); if (ret < 0) return ret; @@ -5780,7 +6636,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: ret = flow_dv_validate_action_raw_encap_decap (dev, NULL, actions->conf, attr, &action_flags, - &actions_n, error); + &actions_n, actions, item_flags, error); if (ret < 0) return ret; break; @@ -5798,7 +6654,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, (dev, decap ? decap : &empty_decap, encap, attr, &action_flags, &actions_n, - error); + actions, item_flags, error); if (ret < 0) return ret; break; @@ -5817,6 +6673,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? MLX5_FLOW_ACTION_SET_MAC_SRC : MLX5_FLOW_ACTION_SET_MAC_DST; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* * Even if the source and destination MAC addresses have * overlap in the header with 4B alignment, the convert @@ -5837,6 +6695,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? MLX5_FLOW_ACTION_SET_IPV4_SRC : @@ -5860,6 +6720,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? MLX5_FLOW_ACTION_SET_IPV6_SRC : @@ -5877,6 +6739,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? MLX5_FLOW_ACTION_SET_TP_SRC : @@ -5894,6 +6758,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_TTL ? MLX5_FLOW_ACTION_SET_TTL : @@ -5907,6 +6773,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret) return ret; + if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && + fdb_mirror_limit) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "sample and jump action combination is not supported"); ++actions_n; action_flags |= MLX5_FLOW_ACTION_JUMP; break; @@ -5922,6 +6794,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? MLX5_FLOW_ACTION_INC_TCP_SEQ : @@ -5940,6 +6814,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? MLX5_FLOW_ACTION_INC_TCP_ACK : @@ -5964,12 +6840,39 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Meter action will add one more TAG action. */ rw_act_num += MLX5_ACT_NUM_SET_TAG; break; + case MLX5_RTE_FLOW_ACTION_TYPE_AGE: + if (!attr->transfer && !attr->group) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Shared ASO age action is not supported for group 0"); + action_flags |= MLX5_FLOW_ACTION_AGE; + ++actions_n; + break; case RTE_FLOW_ACTION_TYPE_AGE: ret = flow_dv_validate_action_age(action_flags, actions, dev, error); if (ret < 0) return ret; + /* + * Validate the regular AGE action (using counter) + * mutual exclusion with share counter actions. + */ + if (!priv->sh->flow_hit_aso_en) { + if (count && count->shared) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "old age and shared count combination is not supported"); + if (sample_count) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "old age action and count must be in the same sub flow"); + } action_flags |= MLX5_FLOW_ACTION_AGE; ++actions_n; break; @@ -5984,6 +6887,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; rw_act_num += MLX5_ACT_NUM_SET_DSCP; break; @@ -5998,13 +6903,19 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; rw_act_num += MLX5_ACT_NUM_SET_DSCP; break; case RTE_FLOW_ACTION_TYPE_SAMPLE: - ret = flow_dv_validate_action_sample(action_flags, + ret = flow_dv_validate_action_sample(&action_flags, actions, dev, - attr, error); + attr, item_flags, + rss, &sample_rss, + &sample_count, + &fdb_mirror_limit, + error); if (ret < 0) return ret; action_flags |= MLX5_FLOW_ACTION_SAMPLE; @@ -6021,6 +6932,23 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET; break; + case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: + if (!attr->transfer && !attr->group) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "modify field action " + "is not supported for group 0"); + ret = flow_dv_validate_action_modify_field(action_flags, + actions, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_ACTION_MODIFY_FIELD)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD; + rw_act_num += ret; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -6080,8 +7008,11 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, * Validate the drop action mutual exclusion with other actions. * Drop action is mutually-exclusive with any other action, except for * Count action. + * Drop action compatibility with tunnel offload was already validated. */ - if ((action_flags & MLX5_FLOW_ACTION_DROP) && + if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH | + MLX5_FLOW_ACTION_TUNNEL_MATCH)); + else if ((action_flags & MLX5_FLOW_ACTION_DROP) && (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -6175,13 +7106,18 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && mlx5_flow_ext_mreg_supported(dev)) rw_act_num += MLX5_ACT_NUM_SET_TAG; - if ((uint32_t)rw_act_num > + if (rw_act_num > flow_dv_modify_hdr_action_max(dev, is_root)) { return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many header modify" " actions to support"); } + /* Eswitch egress mirror and modify flow has limitation on CX5 */ + if (fdb_mirror_limit && modify_after_mirror) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "sample before modify action is not supported"); return 0; } @@ -6233,8 +7169,9 @@ flow_dv_prepare(struct rte_eth_dev *dev, "not enough memory to create flow handle"); return NULL; } - MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows)); + MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows)); dev_flow = &wks->flows[wks->flow_idx++]; + memset(dev_flow, 0, sizeof(*dev_flow)); dev_flow->handle = dev_handle; dev_flow->handle_idx = handle_idx; /* @@ -6246,12 +7183,6 @@ flow_dv_prepare(struct rte_eth_dev *dev, */ dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) - MLX5_ST_SZ_BYTES(fte_match_set_misc4); - /* - * The matching value needs to be cleared to 0 before using. In the - * past, it will be automatically cleared when using rte_*alloc - * API. The time consumption will be almost the same as before. - */ - memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param)); dev_flow->ingress = attr->ingress; dev_flow->dv.transfer = attr->transfer; return dev_flow; @@ -7226,6 +8157,164 @@ flow_dv_translate_item_geneve(void *matcher, void *key, MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); } +/** + * Create Geneve TLV option resource. + * + * @param dev[in, out] + * Pointer to rte_eth_dev structure. + * @param[in, out] tag_be24 + * Tag value in big endian then R-shift 8. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ + +int +flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + struct mlx5_geneve_tlv_option_resource *geneve_opt_resource = + sh->geneve_tlv_option_resource; + struct mlx5_devx_obj *obj; + const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec; + int ret = 0; + + if (!geneve_opt_v) + return -1; + rte_spinlock_lock(&sh->geneve_tlv_opt_sl); + if (geneve_opt_resource != NULL) { + if (geneve_opt_resource->option_class == + geneve_opt_v->option_class && + geneve_opt_resource->option_type == + geneve_opt_v->option_type && + geneve_opt_resource->length == + geneve_opt_v->option_len) { + /* We already have GENVE TLV option obj allocated. */ + __atomic_fetch_add(&geneve_opt_resource->refcnt, 1, + __ATOMIC_RELAXED); + } else { + ret = rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Only one GENEVE TLV option supported"); + goto exit; + } + } else { + /* Create a GENEVE TLV object and resource. */ + obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->ctx, + geneve_opt_v->option_class, + geneve_opt_v->option_type, + geneve_opt_v->option_len); + if (!obj) { + ret = rte_flow_error_set(error, ENODATA, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to create GENEVE TLV Devx object"); + goto exit; + } + sh->geneve_tlv_option_resource = + mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*geneve_opt_resource), + 0, SOCKET_ID_ANY); + if (!sh->geneve_tlv_option_resource) { + claim_zero(mlx5_devx_cmd_destroy(obj)); + ret = rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "GENEVE TLV object memory allocation failed"); + goto exit; + } + geneve_opt_resource = sh->geneve_tlv_option_resource; + geneve_opt_resource->obj = obj; + geneve_opt_resource->option_class = geneve_opt_v->option_class; + geneve_opt_resource->option_type = geneve_opt_v->option_type; + geneve_opt_resource->length = geneve_opt_v->option_len; + __atomic_store_n(&geneve_opt_resource->refcnt, 1, + __ATOMIC_RELAXED); + } +exit: + rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); + return ret; +} + +/** + * Add Geneve TLV option item to matcher. + * + * @param[in, out] dev + * Pointer to rte_eth_dev structure. + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[out] error + * Pointer to error structure. + */ +static int +flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher, + void *key, const struct rte_flow_item *item, + struct rte_flow_error *error) +{ + const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask; + const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec; + void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); + void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); + void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, + misc_parameters_3); + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + rte_be32_t opt_data_key = 0, opt_data_mask = 0; + int ret = 0; + + if (!geneve_opt_v) + return -1; + if (!geneve_opt_m) + geneve_opt_m = &rte_flow_item_geneve_opt_mask; + ret = flow_dev_geneve_tlv_option_resource_register(dev, item, + error); + if (ret) { + DRV_LOG(ERR, "Failed to create geneve_tlv_obj"); + return ret; + } + /* + * Set the option length in GENEVE header if not requested. + * The GENEVE TLV option length is expressed by the option length field + * in the GENEVE header. + * If the option length was not requested but the GENEVE TLV option item + * is present we set the option length field implicitly. + */ + if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) { + MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len, + MLX5_GENEVE_OPTLEN_MASK); + MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, + geneve_opt_v->option_len + 1); + } + /* Set the data. */ + if (geneve_opt_v->data) { + memcpy(&opt_data_key, geneve_opt_v->data, + RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4), + sizeof(opt_data_key))); + MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <= + sizeof(opt_data_key)); + memcpy(&opt_data_mask, geneve_opt_m->data, + RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4), + sizeof(opt_data_mask))); + MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <= + sizeof(opt_data_mask)); + MLX5_SET(fte_match_set_misc3, misc3_m, + geneve_tlv_option_0_data, + rte_be_to_cpu_32(opt_data_mask)); + MLX5_SET(fte_match_set_misc3, misc3_v, + geneve_tlv_option_0_data, + rte_be_to_cpu_32(opt_data_key & opt_data_mask)); + } + return ret; +} + /** * Add MPLS item to matcher and to the value. * @@ -7481,6 +8570,7 @@ flow_dv_translate_item_meta(struct rte_eth_dev *dev, reg = flow_dv_get_metadata_reg(dev, attr, NULL); if (reg < 0) return; + MLX5_ASSERT(reg != REG_NON); /* * In datapath code there is no endianness * coversions for perfromance reasons, all @@ -7623,13 +8713,16 @@ flow_dv_translate_item_source_vport(void *matcher, void *key, * Flow matcher value. * @param[in] item * Flow pattern to translate. + * @param[in] + * Flow attributes. * * @return * 0 on success, a negative errno value otherwise. */ static int flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, - void *key, const struct rte_flow_item *item) + void *key, const struct rte_flow_item *item, + const struct rte_flow_attr *attr) { const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; @@ -7641,14 +8734,34 @@ flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, priv = mlx5_port_to_eswitch_info(id, item == NULL); if (!priv) return -rte_errno; - /* Translate to vport field or to metadata, depending on mode. */ - if (priv->vport_meta_mask) + /* + * Translate to vport field or to metadata, depending on mode. + * Kernel can use either misc.source_port or half of C0 metadata + * register. + */ + if (priv->vport_meta_mask) { + /* + * Provide the hint for SW steering library + * to insert the flow into ingress domain and + * save the extra vport match. + */ + if (mask == 0xffff && priv->vport_id == 0xffff && + priv->pf_bond < 0 && attr->transfer) + flow_dv_translate_item_source_vport + (matcher, key, priv->vport_id, mask); + /* + * We should always set the vport metadata register, + * otherwise the SW steering library can drop + * the rule if wire vport metadata value is not zero, + * it depends on kernel configuration. + */ flow_dv_translate_item_meta_vport(matcher, key, priv->vport_meta_tag, priv->vport_meta_mask); - else + } else { flow_dv_translate_item_source_vport(matcher, key, priv->vport_id, mask); + } return 0; } @@ -7816,6 +8929,82 @@ flow_dv_translate_item_gtp(void *matcher, void *key, rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid)); } +/** + * Add GTP PSC item to matcher. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + */ +static int +flow_dv_translate_item_gtp_psc(void *matcher, void *key, + const struct rte_flow_item *item) +{ + const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask; + const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec; + void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, + misc_parameters_3); + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + union { + uint32_t w32; + struct { + uint16_t seq_num; + uint8_t npdu_num; + uint8_t next_ext_header_type; + }; + } dw_2; + uint8_t gtp_flags; + + /* Always set E-flag match on one, regardless of GTP item settings. */ + gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags); + gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG; + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags); + gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags); + gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG; + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags); + /*Set next extension header type. */ + dw_2.seq_num = 0; + dw_2.npdu_num = 0; + dw_2.next_ext_header_type = 0xff; + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2, + rte_cpu_to_be_32(dw_2.w32)); + dw_2.seq_num = 0; + dw_2.npdu_num = 0; + dw_2.next_ext_header_type = 0x85; + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2, + rte_cpu_to_be_32(dw_2.w32)); + if (gtp_psc_v) { + union { + uint32_t w32; + struct { + uint8_t len; + uint8_t type_flags; + uint8_t qfi; + uint8_t reserved; + }; + } dw_0; + + /*Set extension header PDU type and Qos. */ + if (!gtp_psc_m) + gtp_psc_m = &rte_flow_item_gtp_psc_mask; + dw_0.w32 = 0; + dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->pdu_type); + dw_0.qfi = gtp_psc_m->qfi; + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0, + rte_cpu_to_be_32(dw_0.w32)); + dw_0.w32 = 0; + dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->pdu_type & + gtp_psc_m->pdu_type); + dw_0.qfi = gtp_psc_v->qfi & gtp_psc_m->qfi; + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0, + rte_cpu_to_be_32(dw_0.w32)); + } + return 0; +} + /** * Add eCPRI item to matcher and to the value. * @@ -7837,6 +9026,7 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_item_ecpri *ecpri_m = item->mask; const struct rte_flow_item_ecpri *ecpri_v = item->spec; + struct rte_ecpri_common_hdr common; void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_4); void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4); @@ -7865,7 +9055,7 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, prog_sample_field_value_0); /* Already big endian (network order) in the header. */ *(uint32_t *)dw_m = ecpri_m->hdr.common.u32; - *(uint32_t *)dw_v = ecpri_v->hdr.common.u32; + *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32; /* Sample#0, used for matching type, offset 0. */ MLX5_SET(fte_match_set_misc4, misc4_m, prog_sample_field_id_0, samples[0]); @@ -7877,7 +9067,8 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, * Some wildcard rules only matching type field should be supported. */ if (ecpri_m->hdr.dummy[0]) { - switch (ecpri_v->hdr.common.type) { + common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32); + switch (common.type) { case RTE_ECPRI_MSG_TYPE_IQ_DATA: case RTE_ECPRI_MSG_TYPE_RTC_CTRL: case RTE_ECPRI_MSG_TYPE_DLY_MSR: @@ -7886,7 +9077,8 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v, prog_sample_field_value_1); *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0]; - *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0]; + *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] & + ecpri_m->hdr.dummy[0]; /* Sample#1, to match message body, offset 4. */ MLX5_SET(fte_match_set_misc4, misc4_m, prog_sample_field_id_1, samples[1]); @@ -7967,9 +9159,12 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) tbl_data->idx = idx; tbl_data->tunnel = tt_prm->tunnel; tbl_data->group_id = tt_prm->group_id; - tbl_data->external = tt_prm->external; + tbl_data->external = !!tt_prm->external; tbl_data->tunnel_offload = is_tunnel_offload_active(dev); tbl_data->is_egress = !!key.direction; + tbl_data->is_transfer = !!key.domain; + tbl_data->dummy = !!key.dummy; + tbl_data->table_id = key.table_id; tbl = &tbl_data->tbl; if (key.dummy) return &tbl_data->entry; @@ -8010,6 +9205,21 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) return &tbl_data->entry; } +int +flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused, + struct mlx5_hlist_entry *entry, uint64_t key64, + void *cb_ctx __rte_unused) +{ + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + union mlx5_flow_tbl_key key = { .v64 = key64 }; + + return tbl_data->table_id != key.table_id || + tbl_data->dummy != key.dummy || + tbl_data->is_transfer != key.domain || + tbl_data->is_egress != key.direction; +} + /** * Get a flow table. * @@ -8067,6 +9277,8 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, "cannot get table"); return NULL; } + DRV_LOG(DEBUG, "Table_id %u tunnel %u group %u registered.", + table_id, tunnel ? tunnel->tunnel_id : 0, group_id); tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry); return &tbl_data->tbl; } @@ -8093,26 +9305,16 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list, tbl_data->tunnel->tunnel_id : 0, .group = tbl_data->group_id }; - union mlx5_flow_tbl_key table_key = { - .v64 = entry->key - }; - uint32_t table_id = table_key.table_id; + uint32_t table_id = tbl_data->table_id; tunnel_grp_hash = tbl_data->tunnel ? tbl_data->tunnel->groups : thub->groups; he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL); - if (he) { - struct tunnel_tbl_entry *tte; - tte = container_of(he, typeof(*tte), hash); - MLX5_ASSERT(tte->flow_table == table_id); - mlx5_hlist_remove(tunnel_grp_hash, he); - mlx5_free(tte); - } - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], - tunnel_flow_tbl_to_id(table_id)); + if (he) + mlx5_hlist_unregister(tunnel_grp_hash, he); DRV_LOG(DEBUG, - "Table_id %#x tunnel %u group %u released.", + "Table_id %u tunnel %u group %u released.", table_id, tbl_data->tunnel ? tbl_data->tunnel->tunnel_id : 0, @@ -8224,6 +9426,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, struct mlx5_flow_dv_matcher *ref, union mlx5_flow_tbl_key *key, struct mlx5_flow *dev_flow, + const struct mlx5_flow_tunnel *tunnel, + uint32_t group_id, struct rte_flow_error *error) { struct mlx5_cache_entry *entry; @@ -8235,8 +9439,14 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, .data = ref, }; - tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, - key->domain, false, NULL, 0, 0, error); + /** + * tunnel offload API requires this registration for cases when + * tunnel match rule was inserted before tunnel set rule. + */ + tbl = flow_dv_tbl_resource_get(dev, key->table_id, + key->direction, key->domain, + dev_flow->external, tunnel, + group_id, 0, error); if (!tbl) return -rte_errno; /* No need to refill the error info */ tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); @@ -8270,6 +9480,7 @@ flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx) return NULL; } entry->idx = idx; + entry->tag_id = key; ret = mlx5_flow_os_create_flow_action_tag(key, &entry->action); if (ret) { @@ -8282,8 +9493,19 @@ flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx) return &entry->entry; } -/** - * Find existing tag resource or create and register a new one. +int +flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused, + struct mlx5_hlist_entry *entry, uint64_t key, + void *cb_ctx __rte_unused) +{ + struct mlx5_flow_dv_tag_resource *tag = + container_of(entry, struct mlx5_flow_dv_tag_resource, entry); + + return key != tag->tag_id; +} + +/** + * Find existing tag resource or create and register a new one. * * @param dev[in, out] * Pointer to rte_eth_dev structure. @@ -8446,6 +9668,7 @@ flow_dv_translate_create_counter(struct rte_eth_dev *dev, __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED); return counter; } + /** * Add Tx queue matcher * @@ -8591,7 +9814,7 @@ flow_dv_hrxq_prepare(struct rte_eth_dev *dev, rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; rss_desc->hash_fields = dev_flow->hash_fields; rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL); - rss_desc->standalone = false; + rss_desc->shared_rss = 0; *hrxq_idx = mlx5_hrxq_get(dev, rss_desc); if (!*hrxq_idx) return NULL; @@ -8630,36 +9853,49 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev, flow_dv_tag_release(dev, act_res->rix_tag); act_res->rix_tag = 0; } - if (act_res->cnt) { - flow_dv_counter_release(dev, act_res->cnt); - act_res->cnt = 0; + if (act_res->rix_jump) { + flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump); + act_res->rix_jump = 0; } } -/** - * Find existing sample resource or create and register a new one. - * - * @param[in, out] dev - * Pointer to rte_eth_dev structure. - * @param[in] resource - * Pointer to sample resource. - * @parm[in, out] dev_flow - * Pointer to the dev_flow. - * @param[in, out] sample_dv_actions - * Pointer to sample actions list. - * @param[out] error - * pointer to error structure. - * - * @return - * 0 on success otherwise -errno and errno is set. - */ -static int -flow_dv_sample_resource_register(struct rte_eth_dev *dev, - struct mlx5_flow_dv_sample_resource *resource, - struct mlx5_flow *dev_flow, - void **sample_dv_actions, - struct rte_flow_error *error) +int +flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; + struct mlx5_flow_dv_sample_resource *resource = ctx->data; + struct mlx5_flow_dv_sample_resource *cache_resource = + container_of(entry, typeof(*cache_resource), entry); + + if (resource->ratio == cache_resource->ratio && + resource->ft_type == cache_resource->ft_type && + resource->ft_id == cache_resource->ft_id && + resource->set_action == cache_resource->set_action && + !memcmp((void *)&resource->sample_act, + (void *)&cache_resource->sample_act, + sizeof(struct mlx5_flow_sub_actions_list))) { + /* + * Existing sample action should release the prepared + * sub-actions reference counter. + */ + flow_dv_sample_sub_actions_release(dev, + &resource->sample_idx); + return 0; + } + return 1; +} + +struct mlx5_cache_entry * +flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) { + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; + struct mlx5_flow_dv_sample_resource *resource = ctx->data; + void **sample_dv_actions = resource->sub_actions; struct mlx5_flow_dv_sample_resource *cache_resource; struct mlx5dv_dr_flow_sampler_attr sampler_attr; struct mlx5_priv *priv = dev->data->dev_private; @@ -8670,42 +9906,17 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev, uint32_t next_ft_id = resource->ft_id + next_ft_step; uint8_t is_egress = 0; uint8_t is_transfer = 0; + struct rte_flow_error *error = ctx->error; - /* Lookup a matching resource from cache. */ - ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list, - idx, cache_resource, next) { - if (resource->ratio == cache_resource->ratio && - resource->ft_type == cache_resource->ft_type && - resource->ft_id == cache_resource->ft_id && - resource->set_action == cache_resource->set_action && - !memcmp((void *)&resource->sample_act, - (void *)&cache_resource->sample_act, - sizeof(struct mlx5_flow_sub_actions_list))) { - DRV_LOG(DEBUG, "sample resource %p: refcnt %d++", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, - __ATOMIC_RELAXED)); - __atomic_fetch_add(&cache_resource->refcnt, 1, - __ATOMIC_RELAXED); - dev_flow->handle->dvh.rix_sample = idx; - dev_flow->dv.sample_res = cache_resource; - /* - * Existing sample action should release the prepared - * sub-actions reference counter. - */ - flow_dv_sample_sub_actions_release(dev, - &resource->sample_idx); - return 0; - } - } /* Register new sample resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], - &dev_flow->handle->dvh.rix_sample); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx); + if (!cache_resource) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); + return NULL; + } *cache_resource = *resource; /* Create normal path table level */ if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) @@ -8714,7 +9925,7 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev, is_egress = 1; tbl = flow_dv_tbl_resource_get(dev, next_ft_id, is_egress, is_transfer, - dev_flow->external, NULL, 0, 0, error); + true, NULL, 0, 0, error); if (!tbl) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -8725,18 +9936,16 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev, } cache_resource->normal_path_tbl = tbl; if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) { - cache_resource->default_miss = - mlx5_glue->dr_create_flow_action_default_miss(); - if (!cache_resource->default_miss) { + if (!sh->default_miss_action) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot create default miss " - "action"); + "default miss action was not " + "created"); goto error; } sample_dv_actions[resource->sample_act.actions_num++] = - cache_resource->default_miss; + sh->default_miss_action; } /* Create a DR sample action */ sampler_attr.sample_ratio = cache_resource->ratio; @@ -8745,47 +9954,35 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev, sampler_attr.sample_actions = (struct mlx5dv_dr_action **) &sample_dv_actions[0]; sampler_attr.action = cache_resource->set_action; - cache_resource->verbs_action = - mlx5_glue->dr_create_flow_action_sampler(&sampler_attr); - if (!cache_resource->verbs_action) { + if (mlx5_os_flow_dr_create_flow_action_sampler + (&sampler_attr, &cache_resource->verbs_action)) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create sample action"); goto error; } - __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED); - ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list, - dev_flow->handle->dvh.rix_sample, cache_resource, - next); - dev_flow->dv.sample_res = cache_resource; - DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)); - return 0; + cache_resource->idx = idx; + cache_resource->dev = dev; + return &cache_resource->entry; error: - if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB && - cache_resource->default_miss) - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->default_miss)); - else + if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB) flow_dv_sample_sub_actions_release(dev, &cache_resource->sample_idx); if (cache_resource->normal_path_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), cache_resource->normal_path_tbl); - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], - dev_flow->handle->dvh.rix_sample); - dev_flow->handle->dvh.rix_sample = 0; - return -rte_errno; + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx); + return NULL; + } /** - * Find existing destination array resource or create and register a new one. + * Find existing sample resource or create and register a new one. * * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in] resource - * Pointer to destination array resource. + * Pointer to sample resource. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -8795,56 +9992,88 @@ error: * 0 on success otherwise -errno and errno is set. */ static int -flow_dv_dest_array_resource_register(struct rte_eth_dev *dev, - struct mlx5_flow_dv_dest_array_resource *resource, +flow_dv_sample_resource_register(struct rte_eth_dev *dev, + struct mlx5_flow_dv_sample_resource *resource, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { + struct mlx5_flow_dv_sample_resource *cache_resource; + struct mlx5_cache_entry *entry; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_cb_ctx ctx = { + .dev = dev, + .error = error, + .data = resource, + }; + + entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx); + if (!entry) + return -rte_errno; + cache_resource = container_of(entry, typeof(*cache_resource), entry); + dev_flow->handle->dvh.rix_sample = cache_resource->idx; + dev_flow->dv.sample_res = cache_resource; + return 0; +} + +int +flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_dest_array_resource *resource = ctx->data; + struct rte_eth_dev *dev = ctx->dev; + struct mlx5_flow_dv_dest_array_resource *cache_resource = + container_of(entry, typeof(*cache_resource), entry); + uint32_t idx = 0; + + if (resource->num_of_dest == cache_resource->num_of_dest && + resource->ft_type == cache_resource->ft_type && + !memcmp((void *)cache_resource->sample_act, + (void *)resource->sample_act, + (resource->num_of_dest * + sizeof(struct mlx5_flow_sub_actions_list)))) { + /* + * Existing sample action should release the prepared + * sub-actions reference counter. + */ + for (idx = 0; idx < resource->num_of_dest; idx++) + flow_dv_sample_sub_actions_release(dev, + &resource->sample_idx[idx]); + return 0; + } + return 1; +} + +struct mlx5_cache_entry * +flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; struct mlx5_flow_dv_dest_array_resource *cache_resource; + struct mlx5_flow_dv_dest_array_resource *resource = ctx->data; struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 }; struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM]; struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_flow_sub_actions_list *sample_act; struct mlx5dv_dr_domain *domain; - uint32_t idx = 0; + uint32_t idx = 0, res_idx = 0; + struct rte_flow_error *error = ctx->error; + uint64_t action_flags; + int ret; - /* Lookup a matching resource from cache. */ - ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY], - sh->dest_array_list, - idx, cache_resource, next) { - if (resource->num_of_dest == cache_resource->num_of_dest && - resource->ft_type == cache_resource->ft_type && - !memcmp((void *)cache_resource->sample_act, - (void *)resource->sample_act, - (resource->num_of_dest * - sizeof(struct mlx5_flow_sub_actions_list)))) { - DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, - __ATOMIC_RELAXED)); - __atomic_fetch_add(&cache_resource->refcnt, 1, - __ATOMIC_RELAXED); - dev_flow->handle->dvh.rix_dest_array = idx; - dev_flow->dv.dest_array_res = cache_resource; - /* - * Existing sample action should release the prepared - * sub-actions reference counter. - */ - for (idx = 0; idx < resource->num_of_dest; idx++) - flow_dv_sample_sub_actions_release(dev, - &resource->sample_idx[idx]); - return 0; - } - } /* Register new destination array resource. */ cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY], - &dev_flow->handle->dvh.rix_dest_array); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, + &res_idx); + if (!cache_resource) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); + return NULL; + } *cache_resource = *resource; if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) domain = sh->fdb_domain; @@ -8866,45 +10095,51 @@ flow_dv_dest_array_resource_register(struct rte_eth_dev *dev, } dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST; sample_act = &resource->sample_act[idx]; - if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) { + action_flags = sample_act->action_flags; + switch (action_flags) { + case MLX5_FLOW_ACTION_QUEUE: dest_attr[idx]->dest = sample_act->dr_queue_action; - } else if (sample_act->action_flags == - (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) { + break; + case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP): dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT; dest_attr[idx]->dest_reformat = &dest_reformat[idx]; dest_attr[idx]->dest_reformat->reformat = sample_act->dr_encap_action; dest_attr[idx]->dest_reformat->dest = sample_act->dr_port_id_action; - } else if (sample_act->action_flags == - MLX5_FLOW_ACTION_PORT_ID) { + break; + case MLX5_FLOW_ACTION_PORT_ID: dest_attr[idx]->dest = sample_act->dr_port_id_action; + break; + case MLX5_FLOW_ACTION_JUMP: + dest_attr[idx]->dest = sample_act->dr_jump_action; + break; + default: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported actions type"); + goto error; } } /* create a dest array actioin */ - cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array + ret = mlx5_os_flow_dr_create_flow_action_dest_array (domain, cache_resource->num_of_dest, - dest_attr); - if (!cache_resource->action) { + dest_attr, + &cache_resource->action); + if (ret) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create destination array action"); goto error; } - __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED); - ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY], - &sh->dest_array_list, - dev_flow->handle->dvh.rix_dest_array, cache_resource, - next); - dev_flow->dv.dest_array_res = cache_resource; - DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)); + cache_resource->idx = res_idx; + cache_resource->dev = dev; for (idx = 0; idx < resource->num_of_dest; idx++) mlx5_free(dest_attr[idx]); - return 0; + return &cache_resource->entry; error: for (idx = 0; idx < resource->num_of_dest; idx++) { struct mlx5_flow_sub_actions_idx *act_res = @@ -8921,14 +10156,55 @@ error: !flow_dv_port_id_action_resource_release(dev, act_res->rix_port_id_action)) act_res->rix_port_id_action = 0; + if (act_res->rix_jump && + !flow_dv_jump_tbl_resource_release(dev, + act_res->rix_jump)) + act_res->rix_jump = 0; if (dest_attr[idx]) mlx5_free(dest_attr[idx]); } - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], - dev_flow->handle->dvh.rix_dest_array); - dev_flow->handle->dvh.rix_dest_array = 0; - return -rte_errno; + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx); + return NULL; +} + +/** + * Find existing destination array resource or create and register a new one. + * + * @param[in, out] dev + * Pointer to rte_eth_dev structure. + * @param[in] resource + * Pointer to destination array resource. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ +static int +flow_dv_dest_array_resource_register(struct rte_eth_dev *dev, + struct mlx5_flow_dv_dest_array_resource *resource, + struct mlx5_flow *dev_flow, + struct rte_flow_error *error) +{ + struct mlx5_flow_dv_dest_array_resource *cache_resource; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_cache_entry *entry; + struct mlx5_flow_cb_ctx ctx = { + .dev = dev, + .error = error, + .data = resource, + }; + + entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx); + if (!entry) + return -rte_errno; + cache_resource = container_of(entry, typeof(*cache_resource), entry); + dev_flow->handle->dvh.rix_dest_array = cache_resource->idx; + dev_flow->dv.dest_array_res = cache_resource; + return 0; } /** @@ -8937,7 +10213,7 @@ error: * @param[in] dev * Pointer to rte_eth_dev structure. * @param[in] action - * Pointer to action structure. + * Pointer to sample action structure. * @param[in, out] dev_flow * Pointer to the mlx5_flow. * @param[in] attr @@ -8956,7 +10232,7 @@ error: */ static int flow_dv_translate_action_sample(struct rte_eth_dev *dev, - const struct rte_flow_action *action, + const struct rte_flow_action_sample *action, struct mlx5_flow *dev_flow, const struct rte_flow_attr *attr, uint32_t *num_of_dest, @@ -8965,22 +10241,20 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - const struct rte_flow_action_sample *sample_action; const struct rte_flow_action *sub_actions; - const struct rte_flow_action_queue *queue; struct mlx5_flow_sub_actions_list *sample_act; struct mlx5_flow_sub_actions_idx *sample_idx; struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + struct rte_flow *flow = dev_flow->flow; struct mlx5_flow_rss_desc *rss_desc; uint64_t action_flags = 0; MLX5_ASSERT(wks); - rss_desc = &wks->rss_desc[!!wks->flow_nested_idx]; + rss_desc = &wks->rss_desc; sample_act = &res->sample_act; sample_idx = &res->sample_idx; - sample_action = (const struct rte_flow_action_sample *)action->conf; - res->ratio = sample_action->ratio; - sub_actions = sample_action->actions; + res->ratio = action->ratio; + sub_actions = action->actions; for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) { int type = sub_actions->type; uint32_t pre_rix = 0; @@ -8988,6 +10262,7 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, switch (type) { case RTE_FLOW_ACTION_TYPE_QUEUE: { + const struct rte_flow_action_queue *queue; struct mlx5_hrxq *hrxq; uint32_t hrxq_idx; @@ -9014,6 +10289,45 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, MLX5_FLOW_FATE_QUEUE; break; } + case RTE_FLOW_ACTION_TYPE_RSS: + { + struct mlx5_hrxq *hrxq; + uint32_t hrxq_idx; + const struct rte_flow_action_rss *rss; + const uint8_t *rss_key; + + rss = sub_actions->conf; + memcpy(rss_desc->queue, rss->queue, + rss->queue_num * sizeof(uint16_t)); + rss_desc->queue_num = rss->queue_num; + /* NULL RSS key indicates default RSS key. */ + rss_key = !rss->key ? rss_hash_default_key : rss->key; + memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); + /* + * rss->level and rss.types should be set in advance + * when expanding items for RSS. + */ + flow_dv_hashfields_set(dev_flow, rss_desc); + hrxq = flow_dv_hrxq_prepare(dev, dev_flow, + rss_desc, &hrxq_idx); + if (!hrxq) + return rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot create fate queue"); + sample_act->dr_queue_action = hrxq->action; + sample_idx->rix_hrxq = hrxq_idx; + sample_actions[sample_act->actions_num++] = + hrxq->action; + (*num_of_dest)++; + action_flags |= MLX5_FLOW_ACTION_RSS; + if (action_flags & MLX5_FLOW_ACTION_MARK) + dev_flow->handle->rix_hrxq = hrxq_idx; + dev_flow->handle->fate_action = + MLX5_FLOW_FATE_QUEUE; + break; + } case RTE_FLOW_ACTION_TYPE_MARK: { uint32_t tag_be = mlx5_flow_mark_set @@ -9042,21 +10356,22 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, } case RTE_FLOW_ACTION_TYPE_COUNT: { - uint32_t counter; - - counter = flow_dv_translate_create_counter(dev, - dev_flow, sub_actions->conf, 0); - if (!counter) - return rte_flow_error_set + if (!flow->counter) { + flow->counter = + flow_dv_translate_create_counter(dev, + dev_flow, sub_actions->conf, + 0); + if (!flow->counter) + return rte_flow_error_set (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot create counter" - " object."); - sample_idx->cnt = counter; + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot create counter" + " object."); + } sample_act->dr_cnt_action = (flow_dv_counter_get_by_idx(dev, - counter, NULL))->action; + flow->counter, NULL))->action; sample_actions[sample_act->actions_num++] = sample_act->dr_cnt_action; action_flags |= MLX5_FLOW_ACTION_COUNT; @@ -9187,7 +10502,7 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, uint32_t hrxq_idx; MLX5_ASSERT(wks); - rss_desc = &wks->rss_desc[!!wks->flow_nested_idx]; + rss_desc = &wks->rss_desc; if (num_of_dest > 1) { if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) { /* Handle QP action for mirroring */ @@ -9213,36 +10528,311 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, sample_act->dr_encap_action = dev_flow->dv.encap_decap->action; } - if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) { - normal_idx++; - mdest_res->sample_idx[dest_index].rix_port_id_action = - dev_flow->handle->rix_port_id_action; - sample_act->dr_port_id_action = - dev_flow->dv.port_id_action->action; + if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) { + normal_idx++; + mdest_res->sample_idx[dest_index].rix_port_id_action = + dev_flow->handle->rix_port_id_action; + sample_act->dr_port_id_action = + dev_flow->dv.port_id_action->action; + } + if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) { + normal_idx++; + mdest_res->sample_idx[dest_index].rix_jump = + dev_flow->handle->rix_jump; + sample_act->dr_jump_action = + dev_flow->dv.jump->action; + dev_flow->handle->rix_jump = 0; + } + sample_act->actions_num = normal_idx; + /* update sample action resource into first index of array */ + mdest_res->ft_type = res->ft_type; + memcpy(&mdest_res->sample_idx[0], &res->sample_idx, + sizeof(struct mlx5_flow_sub_actions_idx)); + memcpy(&mdest_res->sample_act[0], &res->sample_act, + sizeof(struct mlx5_flow_sub_actions_list)); + mdest_res->num_of_dest = num_of_dest; + if (flow_dv_dest_array_resource_register(dev, mdest_res, + dev_flow, error)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "can't create sample " + "action"); + } else { + res->sub_actions = sample_actions; + if (flow_dv_sample_resource_register(dev, res, dev_flow, error)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "can't create sample action"); + } + return 0; +} + +/** + * Remove an ASO age action from age actions list. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] age + * Pointer to the aso age action handler. + */ +static void +flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev, + struct mlx5_aso_age_action *age) +{ + struct mlx5_age_info *age_info; + struct mlx5_age_param *age_param = &age->age_params; + struct mlx5_priv *priv = dev->data->dev_private; + uint16_t expected = AGE_CANDIDATE; + + age_info = GET_PORT_AGE_INFO(priv); + if (!__atomic_compare_exchange_n(&age_param->state, &expected, + AGE_FREE, false, __ATOMIC_RELAXED, + __ATOMIC_RELAXED)) { + /** + * We need the lock even it is age timeout, + * since age action may still in process. + */ + rte_spinlock_lock(&age_info->aged_sl); + LIST_REMOVE(age, next); + rte_spinlock_unlock(&age_info->aged_sl); + __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED); + } +} + +/** + * Release an ASO age action. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] age_idx + * Index of ASO age action to release. + * @param[in] flow + * True if the release operation is during flow destroy operation. + * False if the release operation is during action destroy operation. + * + * @return + * 0 when age action was removed, otherwise the number of references. + */ +static int +flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx); + uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED); + + if (!ret) { + flow_dv_aso_age_remove_from_age(dev, age); + rte_spinlock_lock(&mng->free_sl); + LIST_INSERT_HEAD(&mng->free, age, next); + rte_spinlock_unlock(&mng->free_sl); + } + return ret; +} + +/** + * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * 0 on success, otherwise negative errno value and rte_errno is set. + */ +static int +flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + void *old_pools = mng->pools; + uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE; + uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize; + void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); + + if (!pools) { + rte_errno = ENOMEM; + return -ENOMEM; + } + if (old_pools) { + memcpy(pools, old_pools, + mng->n * sizeof(struct mlx5_flow_counter_pool *)); + mlx5_free(old_pools); + } else { + /* First ASO flow hit allocation - starting ASO data-path. */ + int ret = mlx5_aso_queue_start(priv->sh); + + if (ret) { + mlx5_free(pools); + return ret; + } + } + mng->n = resize; + mng->pools = pools; + return 0; +} + +/** + * Create and initialize a new ASO aging pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] age_free + * Where to put the pointer of a new age action. + * + * @return + * The age actions pool pointer and @p age_free is set on success, + * NULL otherwise and rte_errno is set. + */ +static struct mlx5_aso_age_pool * +flow_dv_age_pool_create(struct rte_eth_dev *dev, + struct mlx5_aso_age_action **age_free) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + struct mlx5_aso_age_pool *pool = NULL; + struct mlx5_devx_obj *obj = NULL; + uint32_t i; + + obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->ctx, + priv->sh->pdn); + if (!obj) { + rte_errno = ENODATA; + DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX."); + return NULL; + } + pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); + if (!pool) { + claim_zero(mlx5_devx_cmd_destroy(obj)); + rte_errno = ENOMEM; + return NULL; + } + pool->flow_hit_aso_obj = obj; + pool->time_of_last_age_check = MLX5_CURR_TIME_SEC; + rte_spinlock_lock(&mng->resize_sl); + pool->index = mng->next; + /* Resize pools array if there is no room for the new pool in it. */ + if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) { + claim_zero(mlx5_devx_cmd_destroy(obj)); + mlx5_free(pool); + rte_spinlock_unlock(&mng->resize_sl); + return NULL; + } + mng->pools[pool->index] = pool; + mng->next++; + rte_spinlock_unlock(&mng->resize_sl); + /* Assign the first action in the new pool, the rest go to free list. */ + *age_free = &pool->actions[0]; + for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) { + pool->actions[i].offset = i; + LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next); + } + return pool; +} + +/** + * Allocate a ASO aging bit. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] error + * Pointer to the error structure. + * + * @return + * Index to ASO age action on success, 0 otherwise and rte_errno is set. + */ +static uint32_t +flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const struct mlx5_aso_age_pool *pool; + struct mlx5_aso_age_action *age_free = NULL; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + + MLX5_ASSERT(mng); + /* Try to get the next free age action bit. */ + rte_spinlock_lock(&mng->free_sl); + age_free = LIST_FIRST(&mng->free); + if (age_free) { + LIST_REMOVE(age_free, next); + } else if (!flow_dv_age_pool_create(dev, &age_free)) { + rte_spinlock_unlock(&mng->free_sl); + rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "failed to create ASO age pool"); + return 0; /* 0 is an error. */ + } + rte_spinlock_unlock(&mng->free_sl); + pool = container_of + ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL]) + (age_free - age_free->offset), const struct mlx5_aso_age_pool, + actions); + if (!age_free->dr_action) { + int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0, + error); + + if (reg_c < 0) { + rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "failed to get reg_c " + "for ASO flow hit"); + return 0; /* 0 is an error. */ + } +#ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO + age_free->dr_action = mlx5_glue->dv_create_flow_action_aso + (priv->sh->rx_domain, + pool->flow_hit_aso_obj->obj, age_free->offset, + MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET, + (reg_c - REG_C_0)); +#endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ + if (!age_free->dr_action) { + rte_errno = errno; + rte_spinlock_lock(&mng->free_sl); + LIST_INSERT_HEAD(&mng->free, age_free, next); + rte_spinlock_unlock(&mng->free_sl); + rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "failed to create ASO " + "flow hit action"); + return 0; /* 0 is an error. */ } - sample_act->actions_num = normal_idx; - /* update sample action resource into first index of array */ - mdest_res->ft_type = res->ft_type; - memcpy(&mdest_res->sample_idx[0], &res->sample_idx, - sizeof(struct mlx5_flow_sub_actions_idx)); - memcpy(&mdest_res->sample_act[0], &res->sample_act, - sizeof(struct mlx5_flow_sub_actions_list)); - mdest_res->num_of_dest = num_of_dest; - if (flow_dv_dest_array_resource_register(dev, mdest_res, - dev_flow, error)) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, "can't create sample " - "action"); - } else { - if (flow_dv_sample_resource_register(dev, res, dev_flow, - sample_actions, error)) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "can't create sample action"); } - return 0; + __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED); + return pool->index | ((age_free->offset + 1) << 16); +} + +/** + * Create a age action using ASO mechanism. + * + * @param[in] dev + * Pointer to rte_eth_dev structure. + * @param[in] age + * Pointer to the aging action configuration. + * @param[out] error + * Pointer to the error structure. + * + * @return + * Index to flow counter on success, 0 otherwise. + */ +static uint32_t +flow_dv_translate_create_aso_age(struct rte_eth_dev *dev, + const struct rte_flow_action_age *age, + struct rte_flow_error *error) +{ + uint32_t age_idx = 0; + struct mlx5_aso_age_action *aso_age; + + age_idx = flow_dv_aso_age_alloc(dev, error); + if (!age_idx) + return 0; + aso_age = flow_aso_age_get_by_idx(dev, age_idx); + aso_age->age_params.context = age->context; + aso_age->age_params.timeout = age->timeout; + aso_age->age_params.port_id = dev->data->port_id; + __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0, + __ATOMIC_RELAXED); + __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE, + __ATOMIC_RELAXED); + return age_idx; } /** @@ -9266,12 +10856,12 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -__flow_dv_translate(struct rte_eth_dev *dev, - struct mlx5_flow *dev_flow, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - struct rte_flow_error *error) +flow_dv_translate(struct rte_eth_dev *dev, + struct mlx5_flow *dev_flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_config *dev_conf = &priv->config; @@ -9282,7 +10872,6 @@ __flow_dv_translate(struct rte_eth_dev *dev, uint64_t item_flags = 0; uint64_t last_item = 0; uint64_t action_flags = 0; - uint64_t priority = attr->priority; struct mlx5_flow_dv_matcher matcher = { .mask = { .size = sizeof(matcher.mask.buf) - @@ -9311,6 +10900,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, struct mlx5_flow_dv_dest_array_resource mdest_res; struct mlx5_flow_dv_sample_resource sample_res; void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0}; + const struct rte_flow_action_sample *sample = NULL; struct mlx5_flow_sub_actions_list *sample_act; uint32_t sample_act_pos = UINT32_MAX; uint32_t num_of_dest = 0; @@ -9322,10 +10912,16 @@ __flow_dv_translate(struct rte_eth_dev *dev, .external = !!dev_flow->external, .transfer = !!attr->transfer, .fdb_def_rule = !!priv->fdb_def_rule, + .skip_scale = dev_flow->skip_scale & + (1 << MLX5_SCALE_FLOW_GROUP_BIT), }; - MLX5_ASSERT(wks); - rss_desc = &wks->rss_desc[!!wks->flow_nested_idx]; + if (!wks) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "failed to push flow workspace"); + rss_desc = &wks->rss_desc; memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource)); memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource)); mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : @@ -9342,24 +10938,26 @@ __flow_dv_translate(struct rte_eth_dev *dev, grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate (dev, tunnel, attr, items, actions); ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table, - grp_info, error); + &grp_info, error); if (ret) return ret; dev_flow->dv.group = table; if (attr->transfer) mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; - if (priority == MLX5_FLOW_PRIO_RSVD) - priority = dev_conf->flow_prio - 1; /* number of actions must be set to 0 in case of dirty stack. */ mhdr_res->actions_num = 0; if (is_flow_tunnel_match_rule(dev, attr, items, actions)) { /* * do not add decap action if match rule drops packet * HW rejects rules with decap & drop + * + * if tunnel match rule was inserted before matching tunnel set + * rule flow table used in the match rule must be registered. + * current implementation handles that in the + * flow_dv_match_register() at the function end. */ bool add_decap = true; const struct rte_flow_action *ptr = actions; - struct mlx5_flow_tbl_resource *tbl; for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) { if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) { @@ -9376,20 +10974,6 @@ __flow_dv_translate(struct rte_eth_dev *dev, dev_flow->dv.encap_decap->action; action_flags |= MLX5_FLOW_ACTION_DECAP; } - /* - * bind table_id with for tunnel match rule. - * Tunnel set rule establishes that bind in JUMP action handler. - * Required for scenario when application creates tunnel match - * rule before tunnel set rule. - */ - tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, - attr->transfer, - !!dev_flow->external, tunnel, - attr->group, 0, error); - if (!tbl) - return rte_flow_error_set - (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, - actions, "cannot register tunnel group"); } for (; !actions_end ; actions++) { const struct rte_flow_action_queue *queue; @@ -9398,6 +10982,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, const uint8_t *rss_key; const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; + struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; @@ -9532,9 +11117,43 @@ __flow_dv_translate(struct rte_eth_dev *dev, * when expanding items for RSS. */ action_flags |= MLX5_FLOW_ACTION_RSS; - dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; + dev_flow->handle->fate_action = rss_desc->shared_rss ? + MLX5_FLOW_FATE_SHARED_RSS : + MLX5_FLOW_FATE_QUEUE; + break; + case MLX5_RTE_FLOW_ACTION_TYPE_AGE: + flow->age = (uint32_t)(uintptr_t)(action->conf); + age_act = flow_aso_age_get_by_idx(dev, flow->age); + __atomic_fetch_add(&age_act->refcnt, 1, + __ATOMIC_RELAXED); + dev_flow->dv.actions[actions_n++] = age_act->dr_action; + action_flags |= MLX5_FLOW_ACTION_AGE; break; case RTE_FLOW_ACTION_TYPE_AGE: + if (priv->sh->flow_hit_aso_en && attr->group) { + /* + * Create one shared age action, to be used + * by all sub-flows. + */ + if (!flow->age) { + flow->age = + flow_dv_translate_create_aso_age + (dev, action->conf, + error); + if (!flow->age) + return rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "can't create ASO age action"); + } + dev_flow->dv.actions[actions_n++] = + (flow_aso_age_get_by_idx + (dev, flow->age))->dr_action; + action_flags |= MLX5_FLOW_ACTION_AGE; + break; + } + /* Fall-through */ case RTE_FLOW_ACTION_TYPE_COUNT: if (!dev_conf->devx) { return rte_flow_error_set @@ -9658,10 +11277,15 @@ __flow_dv_translate(struct rte_eth_dev *dev, jump_group = ((const struct rte_flow_action_jump *) action->conf)->group; grp_info.std_tbl_fix = 0; + if (dev_flow->skip_scale & + (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT)) + grp_info.skip_scale = 1; + else + grp_info.skip_scale = 0; ret = mlx5_flow_group_to_table(dev, tunnel, jump_group, &table, - grp_info, error); + &grp_info, error); if (ret) return ret; tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, @@ -9688,6 +11312,8 @@ __flow_dv_translate(struct rte_eth_dev *dev, dev_flow->dv.jump->action; action_flags |= MLX5_FLOW_ACTION_JUMP; dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP; + sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP; + num_of_dest++; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: @@ -9829,15 +11455,8 @@ __flow_dv_translate(struct rte_eth_dev *dev, break; case RTE_FLOW_ACTION_TYPE_SAMPLE: sample_act_pos = actions_n; - ret = flow_dv_translate_action_sample(dev, - actions, - dev_flow, attr, - &num_of_dest, - sample_actions, - &sample_res, - error); - if (ret < 0) - return ret; + sample = (const struct rte_flow_action_sample *) + action->conf; actions_n++; action_flags |= MLX5_FLOW_ACTION_SAMPLE; /* put encap action into group if work with port id */ @@ -9846,6 +11465,12 @@ __flow_dv_translate(struct rte_eth_dev *dev, sample_act->action_flags |= MLX5_FLOW_ACTION_ENCAP; break; + case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: + if (flow_dv_convert_action_modify_field + (dev, mhdr_res, actions, attr, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD; + break; case RTE_FLOW_ACTION_TYPE_END: actions_end = true; if (mhdr_res->actions_num) { @@ -9857,46 +11482,27 @@ __flow_dv_translate(struct rte_eth_dev *dev, handle->dvh.modify_hdr->action; } if (action_flags & MLX5_FLOW_ACTION_COUNT) { - flow->counter = - flow_dv_translate_create_counter(dev, - dev_flow, count, age); - - if (!flow->counter) - return rte_flow_error_set + /* + * Create one count action, to be used + * by all sub-flows. + */ + if (!flow->counter) { + flow->counter = + flow_dv_translate_create_counter + (dev, dev_flow, count, + age); + if (!flow->counter) + return rte_flow_error_set (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot create counter" - " object."); + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "cannot create counter" + " object."); + } dev_flow->dv.actions[actions_n] = (flow_dv_counter_get_by_idx(dev, flow->counter, NULL))->action; actions_n++; } - if (action_flags & MLX5_FLOW_ACTION_SAMPLE) { - ret = flow_dv_create_action_sample(dev, - dev_flow, - num_of_dest, - &sample_res, - &mdest_res, - sample_actions, - action_flags, - error); - if (ret < 0) - return rte_flow_error_set - (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot create sample action"); - if (num_of_dest > 1) { - dev_flow->dv.actions[sample_act_pos] = - dev_flow->dv.dest_array_res->action; - } else { - dev_flow->dv.actions[sample_act_pos] = - dev_flow->dv.sample_res->verbs_action; - } - } - break; default: break; } @@ -9904,33 +11510,6 @@ __flow_dv_translate(struct rte_eth_dev *dev, modify_action_position == UINT32_MAX) modify_action_position = actions_n++; } - /* - * For multiple destination (sample action with ratio=1), the encap - * action and port id action will be combined into group action. - * So need remove the original these actions in the flow and only - * use the sample action instead of. - */ - if (num_of_dest > 1 && sample_act->dr_port_id_action) { - int i; - void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0}; - - for (i = 0; i < actions_n; i++) { - if ((sample_act->dr_encap_action && - sample_act->dr_encap_action == - dev_flow->dv.actions[i]) || - (sample_act->dr_port_id_action && - sample_act->dr_port_id_action == - dev_flow->dv.actions[i])) - continue; - temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i]; - } - memcpy((void *)dev_flow->dv.actions, - (void *)temp_actions, - tmp_actions_n * sizeof(void *)); - actions_n = tmp_actions_n; - } - dev_flow->dv.actions_n = actions_n; - dev_flow->act_flags = action_flags; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); int item_type = items->type; @@ -9941,8 +11520,8 @@ __flow_dv_translate(struct rte_eth_dev *dev, NULL, "item not supported"); switch (item_type) { case RTE_FLOW_ITEM_TYPE_PORT_ID: - flow_dv_translate_item_port_id(dev, match_mask, - match_value, items); + flow_dv_translate_item_port_id + (dev, match_mask, match_value, items, attr); last_item = MLX5_FLOW_ITEM_PORT_ID; break; case RTE_FLOW_ITEM_TYPE_ETH: @@ -10085,6 +11664,17 @@ __flow_dv_translate(struct rte_eth_dev *dev, matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); last_item = MLX5_FLOW_LAYER_GENEVE; break; + case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: + ret = flow_dv_translate_item_geneve_opt(dev, match_mask, + match_value, + items, error); + if (ret) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "cannot create GENEVE TLV option"); + flow->geneve_tlv_option = 1; + last_item = MLX5_FLOW_LAYER_GENEVE_OPT; + break; case RTE_FLOW_ITEM_TYPE_MPLS: flow_dv_translate_item_mpls(match_mask, match_value, items, last_item, tunnel); @@ -10133,6 +11723,16 @@ __flow_dv_translate(struct rte_eth_dev *dev, matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); last_item = MLX5_FLOW_LAYER_GTP; break; + case RTE_FLOW_ITEM_TYPE_GTP_PSC: + ret = flow_dv_translate_item_gtp_psc(match_mask, + match_value, + items); + if (ret) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "cannot create GTP PSC item"); + last_item = MLX5_FLOW_LAYER_GTP_PSC; + break; case RTE_FLOW_ITEM_TYPE_ECPRI: if (!mlx5_flex_parser_ecpri_exist(dev)) { /* Create it only the first time to be used. */ @@ -10168,7 +11768,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && (priv->representor || priv->master)) { if (flow_dv_translate_item_port_id(dev, match_mask, - match_value, NULL)) + match_value, NULL, attr)) return -rte_errno; } #ifdef RTE_LIBRTE_MLX5_DEBUG @@ -10182,16 +11782,83 @@ __flow_dv_translate(struct rte_eth_dev *dev, handle->layers |= item_flags; if (action_flags & MLX5_FLOW_ACTION_RSS) flow_dv_hashfields_set(dev_flow, rss_desc); + /* If has RSS action in the sample action, the Sample/Mirror resource + * should be registered after the hash filed be update. + */ + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) { + ret = flow_dv_translate_action_sample(dev, + sample, + dev_flow, attr, + &num_of_dest, + sample_actions, + &sample_res, + error); + if (ret < 0) + return ret; + ret = flow_dv_create_action_sample(dev, + dev_flow, + num_of_dest, + &sample_res, + &mdest_res, + sample_actions, + action_flags, + error); + if (ret < 0) + return rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot create sample action"); + if (num_of_dest > 1) { + dev_flow->dv.actions[sample_act_pos] = + dev_flow->dv.dest_array_res->action; + } else { + dev_flow->dv.actions[sample_act_pos] = + dev_flow->dv.sample_res->verbs_action; + } + } + /* + * For multiple destination (sample action with ratio=1), the encap + * action and port id action will be combined into group action. + * So need remove the original these actions in the flow and only + * use the sample action instead of. + */ + if (num_of_dest > 1 && + (sample_act->dr_port_id_action || sample_act->dr_jump_action)) { + int i; + void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0}; + + for (i = 0; i < actions_n; i++) { + if ((sample_act->dr_encap_action && + sample_act->dr_encap_action == + dev_flow->dv.actions[i]) || + (sample_act->dr_port_id_action && + sample_act->dr_port_id_action == + dev_flow->dv.actions[i]) || + (sample_act->dr_jump_action && + sample_act->dr_jump_action == + dev_flow->dv.actions[i])) + continue; + temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i]; + } + memcpy((void *)dev_flow->dv.actions, + (void *)temp_actions, + tmp_actions_n * sizeof(void *)); + actions_n = tmp_actions_n; + } + dev_flow->dv.actions_n = actions_n; + dev_flow->act_flags = action_flags; /* Register matcher. */ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, matcher.mask.size); - matcher.priority = mlx5_flow_adjust_priority(dev, priority, - matcher.priority); + matcher.priority = mlx5_get_matcher_priority(dev, attr, + matcher.priority); /* reserved field no needs to be set to 0 here. */ tbl_key.domain = attr->transfer; tbl_key.direction = attr->egress; tbl_key.table_id = dev_flow->dv.group; - if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error)) + if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, + tunnel, attr->group, error)) return -rte_errno; return 0; } @@ -10251,8 +11918,10 @@ __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields) * and tunnel. * - * @param[in] action - * Shred RSS action holding hash RX queue objects. + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] idx + * Shared RSS action ID holding hash RX queue objects. * @param[in] hash_fields * Defines combination of packet fields to participate in RX hash. * @param[in] tunnel @@ -10262,11 +11931,15 @@ __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, * Valid hash RX queue index, otherwise 0. */ static uint32_t -__flow_dv_action_rss_hrxq_lookup(const struct mlx5_shared_action_rss *action, +__flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx, const uint64_t hash_fields, const int tunnel) { - const uint32_t *hrxqs = tunnel ? action->hrxq : action->hrxq_tunnel; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss = + mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); + const uint32_t *hrxqs = tunnel ? shared_rss->hrxq : + shared_rss->hrxq_tunnel; switch (hash_fields & ~IBV_RX_HASH_INNER) { case MLX5_RSS_HASH_IPV4: @@ -10288,51 +11961,6 @@ __flow_dv_action_rss_hrxq_lookup(const struct mlx5_shared_action_rss *action, } } -/** - * Retrieves hash RX queue suitable for the *flow*. - * If shared action configured for *flow* suitable hash RX queue will be - * retrieved from attached shared action. - * - * @param[in] flow - * Shred RSS action holding hash RX queue objects. - * @param[in] dev_flow - * Pointer to the sub flow. - * @param[out] hrxq - * Pointer to retrieved hash RX queue object. - * - * @return - * Valid hash RX queue index, otherwise 0 and rte_errno is set. - */ -static uint32_t -__flow_dv_rss_get_hrxq(struct rte_eth_dev *dev, struct rte_flow *flow, - struct mlx5_flow *dev_flow, - struct mlx5_hrxq **hrxq) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); - uint32_t hrxq_idx; - - if (flow->shared_rss) { - hrxq_idx = __flow_dv_action_rss_hrxq_lookup - (flow->shared_rss, dev_flow->hash_fields, - !!(dev_flow->handle->layers & - MLX5_FLOW_LAYER_TUNNEL)); - if (hrxq_idx) { - *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], - hrxq_idx); - __atomic_fetch_add(&(*hrxq)->refcnt, 1, - __ATOMIC_RELAXED); - } - } else { - struct mlx5_flow_rss_desc *rss_desc = - &wks->rss_desc[!!wks->flow_nested_idx]; - - *hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc, - &hrxq_idx); - } - return hrxq_idx; -} - /** * Apply the flow to the NIC, lock free, * (mutex should be acquired by caller). @@ -10348,8 +11976,8 @@ __flow_dv_rss_get_hrxq(struct rte_eth_dev *dev, struct rte_flow *flow, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -__flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, - struct rte_flow_error *error) +flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, + struct rte_flow_error *error) { struct mlx5_flow_dv_workspace *dv; struct mlx5_flow_handle *dh; @@ -10361,9 +11989,10 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, int err; int idx; struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc; MLX5_ASSERT(wks); - for (idx = wks->flow_idx - 1; idx >= wks->flow_nested_idx; idx--) { + for (idx = wks->flow_idx - 1; idx >= 0; idx--) { dev_flow = &wks->flows[idx]; dv = &dev_flow->dv; dh = dev_flow->handle; @@ -10377,11 +12006,13 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, dv->actions[n++] = priv->drop_queue.hrxq->action; } - } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && - !dv_h->rix_sample && !dv_h->rix_dest_array) { - struct mlx5_hrxq *hrxq = NULL; - uint32_t hrxq_idx = __flow_dv_rss_get_hrxq - (dev, flow, dev_flow, &hrxq); + } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE && + !dv_h->rix_sample && !dv_h->rix_dest_array)) { + struct mlx5_hrxq *hrxq; + uint32_t hrxq_idx; + + hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc, + &hrxq_idx); if (!hrxq) { rte_flow_error_set (error, rte_errno, @@ -10391,6 +12022,28 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, } dh->rix_hrxq = hrxq_idx; dv->actions[n++] = hrxq->action; + } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { + struct mlx5_hrxq *hrxq = NULL; + uint32_t hrxq_idx; + + hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev, + rss_desc->shared_rss, + dev_flow->hash_fields, + !!(dh->layers & + MLX5_FLOW_LAYER_TUNNEL)); + if (hrxq_idx) + hrxq = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_HRXQ], + hrxq_idx); + if (!hrxq) { + rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot get hash queue"); + goto error; + } + dh->rix_srss = rss_desc->shared_rss; + dv->actions[n++] = hrxq->action; } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) { if (!priv->sh->default_miss_action) { rte_flow_error_set @@ -10431,6 +12084,8 @@ error: if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) { mlx5_hrxq_release(dev, dh->rix_hrxq); dh->rix_hrxq = 0; + } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { + dh->rix_srss = 0; } if (dh->vf_vlan.tag && dh->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); @@ -10455,8 +12110,8 @@ flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused, * * @param dev * Pointer to Ethernet device. - * @param handle - * Pointer to mlx5_flow_handle. + * @param port_id + * Index to port ID action resource. * * @return * 1 while a reference on it exists, 0 when freed. @@ -10528,21 +12183,21 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, * * @param dev * Pointer to Ethernet device. - * @param handle - * Pointer to mlx5_flow_handle. + * @param rix_jump + * Index to the jump action resource. * * @return * 1 while a reference on it exists, 0 when freed. */ static int flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, - struct mlx5_flow_handle *handle) + uint32_t rix_jump) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_tbl_data_entry *tbl_data; tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP], - handle->rix_jump); + rix_jump); if (!tbl_data) return 0; return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl); @@ -10619,6 +12274,25 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, &cache->entry); } +/** + * Release shared RSS action resource. + * + * @param dev + * Pointer to Ethernet device. + * @param srss + * Shared RSS action index. + */ +static void +flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss; + + shared_rss = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss); + __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED); +} + void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list, struct mlx5_cache_entry *entry) @@ -10677,7 +12351,7 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev, mlx5_hrxq_release(dev, handle->rix_hrxq); break; case MLX5_FLOW_FATE_JUMP: - flow_dv_jump_tbl_resource_release(dev, handle); + flow_dv_jump_tbl_resource_release(dev, handle->rix_jump); break; case MLX5_FLOW_FATE_PORT_ID: flow_dv_port_id_action_resource_release(dev, @@ -10690,6 +12364,29 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev, handle->rix_fate = 0; } +void +flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry) +{ + struct mlx5_flow_dv_sample_resource *cache_resource = + container_of(entry, typeof(*cache_resource), entry); + struct rte_eth_dev *dev = cache_resource->dev; + struct mlx5_priv *priv = dev->data->dev_private; + + if (cache_resource->verbs_action) + claim_zero(mlx5_flow_os_destroy_flow_action + (cache_resource->verbs_action)); + if (cache_resource->normal_path_tbl) + flow_dv_tbl_resource_release(MLX5_SH(dev), + cache_resource->normal_path_tbl); + flow_dv_sample_sub_actions_release(dev, + &cache_resource->sample_idx); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], + cache_resource->idx); + DRV_LOG(DEBUG, "sample resource %p: removed", + (void *)cache_resource); +} + /** * Release an sample resource. * @@ -10706,41 +12403,38 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - uint32_t idx = handle->dvh.rix_sample; struct mlx5_flow_dv_sample_resource *cache_resource; cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE], - idx); + handle->dvh.rix_sample); if (!cache_resource) return 0; MLX5_ASSERT(cache_resource->verbs_action); - DRV_LOG(DEBUG, "sample resource %p: refcnt %d--", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)); - if (__atomic_sub_fetch(&cache_resource->refcnt, 1, - __ATOMIC_RELAXED) == 0) { - if (cache_resource->verbs_action) - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->verbs_action)); - if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) { - if (cache_resource->default_miss) - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->default_miss)); - } - if (cache_resource->normal_path_tbl) - flow_dv_tbl_resource_release(MLX5_SH(dev), - cache_resource->normal_path_tbl); + return mlx5_cache_unregister(&priv->sh->sample_action_list, + &cache_resource->entry); +} + +void +flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry) +{ + struct mlx5_flow_dv_dest_array_resource *cache_resource = + container_of(entry, typeof(*cache_resource), entry); + struct rte_eth_dev *dev = cache_resource->dev; + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t i = 0; + + MLX5_ASSERT(cache_resource->action); + if (cache_resource->action) + claim_zero(mlx5_flow_os_destroy_flow_action + (cache_resource->action)); + for (; i < cache_resource->num_of_dest; i++) flow_dv_sample_sub_actions_release(dev, - &cache_resource->sample_idx); - ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE], - &priv->sh->sample_action_list, idx, - cache_resource, next); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx); - DRV_LOG(DEBUG, "sample resource %p: removed", - (void *)cache_resource); - return 0; - } - return 1; + &cache_resource->sample_idx[i]); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], + cache_resource->idx); + DRV_LOG(DEBUG, "destination array resource %p: removed", + (void *)cache_resource); } /** @@ -10756,38 +12450,38 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev, */ static int flow_dv_dest_array_resource_release(struct rte_eth_dev *dev, - struct mlx5_flow_handle *handle) + struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_dest_array_resource *cache_resource; - uint32_t idx = handle->dvh.rix_dest_array; - uint32_t i = 0; + struct mlx5_flow_dv_dest_array_resource *cache; - cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], - idx); - if (!cache_resource) - return 0; - MLX5_ASSERT(cache_resource->action); - DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)); - if (__atomic_sub_fetch(&cache_resource->refcnt, 1, - __ATOMIC_RELAXED) == 0) { - if (cache_resource->action) - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->action)); - for (; i < cache_resource->num_of_dest; i++) - flow_dv_sample_sub_actions_release(dev, - &cache_resource->sample_idx[i]); - ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], - &priv->sh->dest_array_list, idx, - cache_resource, next); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx); - DRV_LOG(DEBUG, "destination array resource %p: removed", - (void *)cache_resource); + cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], + handle->dvh.rix_dest_array); + if (!cache) return 0; + MLX5_ASSERT(cache->action); + return mlx5_cache_unregister(&priv->sh->dest_array_list, + &cache->entry); +} + +static void +flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + struct mlx5_geneve_tlv_option_resource *geneve_opt_resource = + sh->geneve_tlv_option_resource; + rte_spinlock_lock(&sh->geneve_tlv_opt_sl); + if (geneve_opt_resource) { + if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1, + __ATOMIC_RELAXED))) { + claim_zero(mlx5_devx_cmd_destroy + (geneve_opt_resource->obj)); + mlx5_free(sh->geneve_tlv_option_resource); + sh->geneve_tlv_option_resource = NULL; + } } - return 1; + rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); } /** @@ -10800,7 +12494,7 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev, * Pointer to flow structure. */ static void -__flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) +flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dh; uint32_t handle_idx; @@ -10836,20 +12530,17 @@ __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) * Pointer to flow structure. */ static void -__flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) +flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct rte_flow_shared_action *shared; struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + uint32_t srss = 0; if (!flow) return; - __flow_dv_remove(dev, flow); - shared = mlx5_flow_get_shared_rss(flow); - if (shared) - __atomic_sub_fetch(&shared->refcnt, 1, __ATOMIC_RELAXED); + flow_dv_remove(dev, flow); if (flow->counter) { - flow_dv_counter_release(dev, flow->counter); + flow_dv_counter_free(dev, flow->counter); flow->counter = 0; } if (flow->meter) { @@ -10861,6 +12552,12 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) mlx5_flow_meter_detach(fm); flow->meter = 0; } + if (flow->age) + flow_dv_aso_age_release(dev, flow->age); + if (flow->geneve_tlv_option) { + flow_dv_geneve_tlv_option_resource_release(dev); + flow->geneve_tlv_option = 0; + } while (flow->dev_handles) { uint32_t tmp_idx = flow->dev_handles; @@ -10886,10 +12583,15 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) if (dev_handle->dvh.rix_tag) flow_dv_tag_release(dev, dev_handle->dvh.rix_tag); - flow_dv_fate_resource_release(dev, dev_handle); + if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS) + flow_dv_fate_resource_release(dev, dev_handle); + else if (!srss) + srss = dev_handle->rix_srss; mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } + if (srss) + flow_dv_shared_rss_action_release(dev, srss); } /** @@ -10937,10 +12639,10 @@ __flow_dv_hrxqs_release(struct rte_eth_dev *dev, */ static int __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, - struct mlx5_shared_action_rss *action) + struct mlx5_shared_action_rss *shared_rss) { - return __flow_dv_hrxqs_release(dev, &action->hrxq) + - __flow_dv_hrxqs_release(dev, &action->hrxq_tunnel); + return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq) + + __flow_dv_hrxqs_release(dev, &shared_rss->hrxq_tunnel); } /** @@ -10950,6 +12652,8 @@ __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to the Ethernet device structure. + * @param[in] action_idx + * Shared RSS action ipool index. * @param[in, out] action * Partially initialized shared RSS action. * @param[out] error @@ -10961,18 +12665,26 @@ __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, */ static int __flow_dv_action_rss_setup(struct rte_eth_dev *dev, - struct mlx5_shared_action_rss *action, - struct rte_flow_error *error) + uint32_t action_idx, + struct mlx5_shared_action_rss *shared_rss, + struct rte_flow_error *error) { struct mlx5_flow_rss_desc rss_desc = { 0 }; size_t i; int err; - memcpy(rss_desc.key, action->origin.key, MLX5_RSS_HASH_KEY_LEN); + if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) { + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot setup indirection table"); + } + memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN); rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN; - rss_desc.const_q = action->origin.queue; - rss_desc.queue_num = action->origin.queue_num; - rss_desc.standalone = true; + rss_desc.const_q = shared_rss->origin.queue; + rss_desc.queue_num = shared_rss->origin.queue_num; + /* Set non-zero value to indicate a shared RSS. */ + rss_desc.shared_rss = action_idx; + rss_desc.ind_tbl = shared_rss->ind_tbl; for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) { uint32_t hrxq_idx; uint64_t hash_fields = mlx5_rss_hash_fields[i]; @@ -10990,14 +12702,16 @@ __flow_dv_action_rss_setup(struct rte_eth_dev *dev, goto error_hrxq_new; } err = __flow_dv_action_rss_hrxq_set - (action, hash_fields, tunnel, hrxq_idx); + (shared_rss, hash_fields, tunnel, hrxq_idx); MLX5_ASSERT(!err); } } return 0; error_hrxq_new: err = rte_errno; - __flow_dv_action_rss_hrxqs_release(dev, action); + __flow_dv_action_rss_hrxqs_release(dev, shared_rss); + if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true)) + shared_rss->ind_tbl = NULL; rte_errno = err; return -rte_errno; } @@ -11016,35 +12730,52 @@ error_hrxq_new: * error only. * * @return - * A valid shared action handle in case of success, NULL otherwise and + * A valid shared action ID in case of success, 0 otherwise and * rte_errno is set. */ -static struct rte_flow_shared_action * +static uint32_t __flow_dv_action_rss_create(struct rte_eth_dev *dev, const struct rte_flow_shared_action_conf *conf, const struct rte_flow_action_rss *rss, struct rte_flow_error *error) { - struct rte_flow_shared_action *shared_action = NULL; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss = NULL; void *queue = NULL; - struct mlx5_shared_action_rss *shared_rss; struct rte_flow_action_rss *origin; const uint8_t *rss_key; uint32_t queue_size = rss->queue_num * sizeof(uint16_t); + uint32_t idx; RTE_SET_USED(conf); queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)), 0, SOCKET_ID_ANY); - shared_action = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*shared_action), 0, - SOCKET_ID_ANY); - if (!shared_action || !queue) { + shared_rss = mlx5_ipool_zmalloc + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx); + if (!shared_rss || !queue) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + goto error_rss_init; + } + if (idx > (1u << MLX5_SHARED_ACTION_TYPE_OFFSET)) { + rte_flow_error_set(error, E2BIG, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "rss action number out of range"); + goto error_rss_init; + } + shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*shared_rss->ind_tbl), + 0, SOCKET_ID_ANY); + if (!shared_rss->ind_tbl) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); goto error_rss_init; } - shared_rss = &shared_action->rss; - shared_rss->queue = queue; + memcpy(queue, rss->queue, queue_size); + shared_rss->ind_tbl->queues = queue; + shared_rss->ind_tbl->queues_n = rss->queue_num; origin = &shared_rss->origin; origin->func = rss->func; origin->level = rss->level; @@ -11055,17 +12786,27 @@ __flow_dv_action_rss_create(struct rte_eth_dev *dev, memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN); origin->key = &shared_rss->key[0]; origin->key_len = MLX5_RSS_HASH_KEY_LEN; - memcpy(shared_rss->queue, rss->queue, queue_size); - origin->queue = shared_rss->queue; + origin->queue = queue; origin->queue_num = rss->queue_num; - if (__flow_dv_action_rss_setup(dev, shared_rss, error)) + if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error)) goto error_rss_init; - shared_action->type = MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS; - return shared_action; + rte_spinlock_init(&shared_rss->action_rss_sl); + __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED); + rte_spinlock_lock(&priv->shared_act_sl); + ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + &priv->rss_shared_actions, idx, shared_rss, next); + rte_spinlock_unlock(&priv->shared_act_sl); + return idx; error_rss_init: - mlx5_free(shared_action); - mlx5_free(queue); - return NULL; + if (shared_rss) { + if (shared_rss->ind_tbl) + mlx5_free(shared_rss->ind_tbl); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + idx); + } + if (queue) + mlx5_free(queue); + return 0; } /** @@ -11074,8 +12815,8 @@ error_rss_init: * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] shared_rss - * The shared RSS action object to be removed. + * @param[in] idx + * The shared RSS action object ID to be removed. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -11084,31 +12825,48 @@ error_rss_init: * 0 on success, otherwise negative errno value. */ static int -__flow_dv_action_rss_release(struct rte_eth_dev *dev, - struct mlx5_shared_action_rss *shared_rss, - struct rte_flow_error *error) +__flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, + struct rte_flow_error *error) { - struct rte_flow_shared_action *shared_action = NULL; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss = + mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); uint32_t old_refcnt = 1; - int remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss); + int remaining; + uint16_t *queue = NULL; - if (remaining) { - return rte_flow_error_set(error, ETOOMANYREFS, + if (!shared_rss) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "invalid shared action"); + remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss); + if (remaining) + return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "shared rss hrxq has references"); - } - shared_action = container_of(shared_rss, - struct rte_flow_shared_action, rss); - if (!__atomic_compare_exchange_n(&shared_action->refcnt, &old_refcnt, - 0, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { - return rte_flow_error_set(error, ETOOMANYREFS, + if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt, + 0, 0, __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED)) + return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "shared rss has references"); - } - rte_free(shared_rss->queue); + queue = shared_rss->ind_tbl->queues; + remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true); + if (remaining) + return rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "shared rss indirection table has" + " references"); + mlx5_free(queue); + rte_spinlock_lock(&priv->shared_act_sl); + ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + &priv->rss_shared_actions, idx, shared_rss, next); + rte_spinlock_unlock(&priv->shared_act_sl); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + idx); return 0; } @@ -11132,31 +12890,39 @@ __flow_dv_action_rss_release(struct rte_eth_dev *dev, * rte_errno is set. */ static struct rte_flow_shared_action * -__flow_dv_action_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, - const struct rte_flow_action *action, - struct rte_flow_error *error) +flow_dv_action_create(struct rte_eth_dev *dev, + const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_action *action, + struct rte_flow_error *err) { - struct rte_flow_shared_action *shared_action = NULL; - struct mlx5_priv *priv = dev->data->dev_private; + uint32_t idx = 0; + uint32_t ret = 0; switch (action->type) { case RTE_FLOW_ACTION_TYPE_RSS: - shared_action = __flow_dv_action_rss_create(dev, conf, - action->conf, - error); + ret = __flow_dv_action_rss_create(dev, conf, action->conf, err); + idx = (MLX5_SHARED_ACTION_TYPE_RSS << + MLX5_SHARED_ACTION_TYPE_OFFSET) | ret; + break; + case RTE_FLOW_ACTION_TYPE_AGE: + ret = flow_dv_translate_create_aso_age(dev, action->conf, err); + idx = (MLX5_SHARED_ACTION_TYPE_AGE << + MLX5_SHARED_ACTION_TYPE_OFFSET) | ret; + if (ret) { + struct mlx5_aso_age_action *aso_age = + flow_aso_age_get_by_idx(dev, ret); + + if (!aso_age->age_params.context) + aso_age->age_params.context = + (void *)(uintptr_t)idx; + } break; default: - rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "action type not supported"); break; } - if (shared_action) { - __atomic_add_fetch(&shared_action->refcnt, 1, - __ATOMIC_RELAXED); - LIST_INSERT_HEAD(&priv->shared_actions, shared_action, next); - } - return shared_action; + return ret ? (struct rte_flow_shared_action *)(uintptr_t)idx : NULL; } /** @@ -11177,27 +12943,34 @@ __flow_dv_action_create(struct rte_eth_dev *dev, * 0 on success, otherwise negative errno value. */ static int -__flow_dv_action_destroy(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - struct rte_flow_error *error) +flow_dv_action_destroy(struct rte_eth_dev *dev, + struct rte_flow_shared_action *action, + struct rte_flow_error *error) { + uint32_t act_idx = (uint32_t)(uintptr_t)action; + uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); int ret; - switch (action->type) { - case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: - ret = __flow_dv_action_rss_release(dev, &action->rss, error); - break; + switch (type) { + case MLX5_SHARED_ACTION_TYPE_RSS: + return __flow_dv_action_rss_release(dev, idx, error); + case MLX5_SHARED_ACTION_TYPE_AGE: + ret = flow_dv_aso_age_release(dev, idx); + if (ret) + /* + * In this case, the last flow has a reference will + * actually release the age action. + */ + DRV_LOG(DEBUG, "Shared age action %" PRIu32 " was" + " released with references %d.", idx, ret); + return 0; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "action type not supported"); } - if (ret) - return ret; - LIST_REMOVE(action, next); - rte_free(action); - return 0; } /** @@ -11205,8 +12978,8 @@ __flow_dv_action_destroy(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] shared_rss - * The shared RSS action object to be updated. + * @param[in] idx + * The shared RSS action object ID to be updated. * @param[in] action_conf * RSS action specification used to modify *shared_rss*. * @param[out] error @@ -11218,18 +12991,26 @@ __flow_dv_action_destroy(struct rte_eth_dev *dev, * @note: currently only support update of RSS queues. */ static int -__flow_dv_action_rss_update(struct rte_eth_dev *dev, - struct mlx5_shared_action_rss *shared_rss, +__flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx, const struct rte_flow_action_rss *action_conf, struct rte_flow_error *error) { - size_t i; - int ret; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss = + mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); + int ret = 0; void *queue = NULL; - const uint8_t *rss_key; - uint32_t rss_key_len; + uint16_t *queue_old = NULL; uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t); + if (!shared_rss) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "invalid shared action to update"); + if (priv->obj_ops.ind_table_modify == NULL) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "cannot modify indirection table"); queue = mlx5_malloc(MLX5_MEM_ZERO, RTE_ALIGN_CEIL(queue_size, sizeof(void *)), 0, SOCKET_ID_ANY); @@ -11238,42 +13019,24 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); - if (action_conf->key) { - rss_key = action_conf->key; - rss_key_len = action_conf->key_len; + memcpy(queue, action_conf->queue, queue_size); + MLX5_ASSERT(shared_rss->ind_tbl); + rte_spinlock_lock(&shared_rss->action_rss_sl); + queue_old = shared_rss->ind_tbl->queues; + ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl, + queue, action_conf->queue_num, true); + if (ret) { + mlx5_free(queue); + ret = rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "cannot update indirection table"); } else { - rss_key = rss_hash_default_key; - rss_key_len = MLX5_RSS_HASH_KEY_LEN; - } - for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) { - uint32_t hrxq_idx; - uint64_t hash_fields = mlx5_rss_hash_fields[i]; - int tunnel; - - for (tunnel = 0; tunnel < 2; tunnel++) { - hrxq_idx = __flow_dv_action_rss_hrxq_lookup - (shared_rss, hash_fields, tunnel); - MLX5_ASSERT(hrxq_idx); - ret = mlx5_hrxq_modify - (dev, hrxq_idx, - rss_key, rss_key_len, - hash_fields, - action_conf->queue, action_conf->queue_num); - if (ret) { - mlx5_free(queue); - return rte_flow_error_set - (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "cannot update hash queue"); - } - } + mlx5_free(queue_old); + shared_rss->origin.queue = queue; + shared_rss->origin.queue_num = action_conf->queue_num; } - mlx5_free(shared_rss->queue); - shared_rss->queue = queue; - memcpy(shared_rss->queue, action_conf->queue, queue_size); - shared_rss->origin.queue = shared_rss->queue; - shared_rss->origin.queue_num = action_conf->queue_num; - return 0; + rte_spinlock_unlock(&shared_rss->action_rss_sl); + return ret; } /** @@ -11296,22 +13059,57 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, * 0 on success, otherwise negative errno value. */ static int -__flow_dv_action_update(struct rte_eth_dev *dev, +flow_dv_action_update(struct rte_eth_dev *dev, struct rte_flow_shared_action *action, const void *action_conf, - struct rte_flow_error *error) + struct rte_flow_error *err) { - switch (action->type) { - case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: - return __flow_dv_action_rss_update(dev, &action->rss, - action_conf, error); + uint32_t act_idx = (uint32_t)(uintptr_t)action; + uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + + switch (type) { + case MLX5_SHARED_ACTION_TYPE_RSS: + return __flow_dv_action_rss_update(dev, idx, action_conf, err); + default: + return rte_flow_error_set(err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "action type update not supported"); + } +} + +static int +flow_dv_action_query(struct rte_eth_dev *dev, + const struct rte_flow_shared_action *action, void *data, + struct rte_flow_error *error) +{ + struct mlx5_age_param *age_param; + struct rte_flow_query_age *resp; + uint32_t act_idx = (uint32_t)(uintptr_t)action; + uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + + switch (type) { + case MLX5_SHARED_ACTION_TYPE_AGE: + age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params; + resp = data; + resp->aged = __atomic_load_n(&age_param->state, + __ATOMIC_RELAXED) == AGE_TMOUT ? + 1 : 0; + resp->sec_since_last_hit_valid = !resp->aged; + if (resp->sec_since_last_hit_valid) + resp->sec_since_last_hit = __atomic_load_n + (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); + return 0; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "action type not supported"); + "action type query not supported"); } } + /** * Query a dv flow rule for its statistics via devx. * @@ -11388,30 +13186,33 @@ flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow, void *data, struct rte_flow_error *error) { struct rte_flow_query_age *resp = data; + struct mlx5_age_param *age_param; - if (flow->counter) { - struct mlx5_age_param *age_param = - flow_dv_counter_idx_get_age(dev, flow->counter); + if (flow->age) { + struct mlx5_aso_age_action *act = + flow_aso_age_get_by_idx(dev, flow->age); + + age_param = &act->age_params; + } else if (flow->counter) { + age_param = flow_dv_counter_idx_get_age(dev, flow->counter); if (!age_param || !age_param->timeout) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot read age data"); - resp->aged = __atomic_load_n(&age_param->state, - __ATOMIC_RELAXED) == - AGE_TMOUT ? 1 : 0; - resp->sec_since_last_hit_valid = !resp->aged; - if (resp->sec_since_last_hit_valid) - resp->sec_since_last_hit = - __atomic_load_n(&age_param->sec_since_last_hit, - __ATOMIC_RELAXED); - return 0; - } - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "age data not available"); + } else { + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "age data not available"); + } + resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) == + AGE_TMOUT ? 1 : 0; + resp->sec_since_last_hit_valid = !resp->aged; + if (resp->sec_since_last_hit_valid) + resp->sec_since_last_hit = __atomic_load_n + (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); + return 0; } /** @@ -11895,19 +13696,15 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) .match_mask = (void *)&mask, }; void *actions[2] = { 0 }; - struct mlx5_flow_tbl_resource *tbl = NULL, *dest_tbl = NULL; + struct mlx5_flow_tbl_resource *tbl = NULL; struct mlx5_devx_obj *dcs = NULL; void *matcher = NULL; void *flow = NULL; - int i, ret = -1; + int ret = -1; tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, 0, NULL); if (!tbl) goto err; - dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, false, - NULL, 0, 0, NULL); - if (!dest_tbl) - goto err; dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); if (!dcs) goto err; @@ -11915,10 +13712,7 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) &actions[0]); if (ret) goto err; - ret = mlx5_flow_os_create_flow_action_dest_flow_tbl - (dest_tbl->obj, &actions[1]); - if (ret) - goto err; + actions[1] = priv->drop_queue.hrxq->action; dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf); ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj, &matcher); @@ -11947,17 +13741,12 @@ err: "support detection"); ret = 0; } - for (i = 0; i < 2; i++) { - if (actions[i]) - claim_zero(mlx5_flow_os_destroy_flow_action - (actions[i])); - } + if (actions[0]) + claim_zero(mlx5_flow_os_destroy_flow_action(actions[0])); if (matcher) claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher)); if (tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); - if (dest_tbl) - flow_dv_tbl_resource_release(MLX5_SH(dev), dest_tbl); if (dcs) claim_zero(mlx5_devx_cmd_destroy(dcs)); return ret; @@ -12035,16 +13824,24 @@ flow_get_aged_flows(struct rte_eth_dev *dev, struct mlx5_age_info *age_info; struct mlx5_age_param *age_param; struct mlx5_flow_counter *counter; + struct mlx5_aso_age_action *act; int nb_flows = 0; if (nb_contexts && !context) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "Should assign at least one flow or" - " context to get if nb_contexts != 0"); + NULL, "empty context"); age_info = GET_PORT_AGE_INFO(priv); rte_spinlock_lock(&age_info->aged_sl); + LIST_FOREACH(act, &age_info->aged_aso, next) { + nb_flows++; + if (nb_contexts) { + context[nb_flows - 1] = + act->age_params.context; + if (!(--nb_contexts)) + break; + } + } TAILQ_FOREACH(counter, &age_info->aged_counters, next) { nb_flows++; if (nb_contexts) { @@ -12059,86 +13856,13 @@ flow_get_aged_flows(struct rte_eth_dev *dev, return nb_flows; } -/* - * Mutex-protected thunk to lock-free __flow_dv_translate(). - */ -static int -flow_dv_translate(struct rte_eth_dev *dev, - struct mlx5_flow *dev_flow, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - struct rte_flow_error *error) -{ - int ret; - - flow_dv_shared_lock(dev); - ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error); - flow_dv_shared_unlock(dev); - return ret; -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_apply(). - */ -static int -flow_dv_apply(struct rte_eth_dev *dev, - struct rte_flow *flow, - struct rte_flow_error *error) -{ - int ret; - - flow_dv_shared_lock(dev); - ret = __flow_dv_apply(dev, flow, error); - flow_dv_shared_unlock(dev); - return ret; -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_remove(). - */ -static void -flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) -{ - flow_dv_shared_lock(dev); - __flow_dv_remove(dev, flow); - flow_dv_shared_unlock(dev); -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_destroy(). - */ -static void -flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) -{ - flow_dv_shared_lock(dev); - __flow_dv_destroy(dev, flow); - flow_dv_shared_unlock(dev); -} - /* * Mutex-protected thunk to lock-free flow_dv_counter_alloc(). */ static uint32_t flow_dv_counter_allocate(struct rte_eth_dev *dev) { - uint32_t cnt; - - flow_dv_shared_lock(dev); - cnt = flow_dv_counter_alloc(dev, 0); - flow_dv_shared_unlock(dev); - return cnt; -} - -/* - * Mutex-protected thunk to lock-free flow_dv_counter_release(). - */ -static void -flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt) -{ - flow_dv_shared_lock(dev); - flow_dv_counter_release(dev, cnt); - flow_dv_shared_unlock(dev); + return flow_dv_counter_alloc(dev, 0); } /** @@ -12162,71 +13886,43 @@ static int flow_dv_action_validate(struct rte_eth_dev *dev, const struct rte_flow_shared_action_conf *conf, const struct rte_flow_action *action, - struct rte_flow_error *error) + struct rte_flow_error *err) { + struct mlx5_priv *priv = dev->data->dev_private; + RTE_SET_USED(conf); switch (action->type) { case RTE_FLOW_ACTION_TYPE_RSS: - return mlx5_validate_action_rss(dev, action, error); + /* + * priv->obj_ops is set according to driver capabilities. + * When DevX capabilities are + * sufficient, it is set to devx_obj_ops. + * Otherwise, it is set to ibv_obj_ops. + * ibv_obj_ops doesn't support ind_table_modify operation. + * In this case the shared RSS action can't be used. + */ + if (priv->obj_ops.ind_table_modify == NULL) + return rte_flow_error_set + (err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "shared RSS action not supported"); + return mlx5_validate_action_rss(dev, action, err); + case RTE_FLOW_ACTION_TYPE_AGE: + if (!priv->sh->aso_age_mng) + return rte_flow_error_set(err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "shared age action not supported"); + return flow_dv_validate_action_age(0, action, dev, err); default: - return rte_flow_error_set(error, ENOTSUP, + return rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "action type not supported"); } } -/* - * Mutex-protected thunk to lock-free __flow_dv_action_create(). - */ -static struct rte_flow_shared_action * -flow_dv_action_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, - const struct rte_flow_action *action, - struct rte_flow_error *error) -{ - struct rte_flow_shared_action *shared_action = NULL; - - flow_dv_shared_lock(dev); - shared_action = __flow_dv_action_create(dev, conf, action, error); - flow_dv_shared_unlock(dev); - return shared_action; -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_action_destroy(). - */ -static int -flow_dv_action_destroy(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - struct rte_flow_error *error) -{ - int ret; - - flow_dv_shared_lock(dev); - ret = __flow_dv_action_destroy(dev, action, error); - flow_dv_shared_unlock(dev); - return ret; -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_action_update(). - */ -static int -flow_dv_action_update(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - const void *action_conf, - struct rte_flow_error *error) -{ - int ret; - - flow_dv_shared_lock(dev); - ret = __flow_dv_action_update(dev, action, action_conf, - error); - flow_dv_shared_unlock(dev); - return ret; -} - static int flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags) { @@ -12234,18 +13930,18 @@ flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags) int ret = 0; if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) { - ret = mlx5_glue->dr_sync_domain(priv->sh->rx_domain, + ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain, flags); if (ret != 0) return ret; } if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) { - ret = mlx5_glue->dr_sync_domain(priv->sh->tx_domain, flags); + ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags); if (ret != 0) return ret; } if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) { - ret = mlx5_glue->dr_sync_domain(priv->sh->fdb_domain, flags); + ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags); if (ret != 0) return ret; } @@ -12272,6 +13968,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .action_create = flow_dv_action_create, .action_destroy = flow_dv_action_destroy, .action_update = flow_dv_action_update, + .action_query = flow_dv_action_query, .sync_domain = flow_dv_sync_domain, };