X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=223a7d0e365bde2506955f8c9191f4026ccb81a4;hb=5cc6764267e405364d3dee5d0dc512fcf82a7786;hp=62e1d19a8927a340f7c85cdcdcb6e0330a46c8ff;hpb=3f90d1a00a5444ac7067705ab292ced71cf25c3c;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 62e1d19a89..223a7d0e36 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include @@ -33,8 +33,9 @@ #include "mlx5_flow.h" #include "mlx5_flow_os.h" #include "mlx5_rxtx.h" +#include "rte_pmd_mlx5.h" -#ifdef HAVE_IBV_FLOW_DV_SUPPORT +#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 @@ -70,19 +71,22 @@ union flow_dv_attr { }; static int -flow_dv_tbl_resource_release(struct rte_eth_dev *dev, +flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh, struct mlx5_flow_tbl_resource *tbl); -static int -flow_dv_default_miss_resource_release(struct rte_eth_dev *dev); - static int flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, - uint32_t encap_decap_idx); + uint32_t encap_decap_idx); static int flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, uint32_t port_id); +static void +flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss); + +static int +flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, + uint32_t rix_jump); /** * Initialize flow attributes structure according to flow items' types. @@ -278,45 +282,6 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, } } -/** - * Acquire the synchronizing object to protect multithreaded access - * to shared dv context. Lock occurs only if context is actually - * shared, i.e. we have multiport IB device and representors are - * created. - * - * @param[in] dev - * Pointer to the rte_eth_dev structure. - */ -static void -flow_dv_shared_lock(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - - if (sh->refcnt > 1) { - int ret; - - ret = pthread_mutex_lock(&sh->dv_mutex); - MLX5_ASSERT(!ret); - (void)ret; - } -} - -static void -flow_dv_shared_unlock(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - - if (sh->refcnt > 1) { - int ret; - - ret = pthread_mutex_unlock(&sh->dv_mutex); - MLX5_ASSERT(!ret); - (void)ret; - } -} - /* Update VLAN's VID/PCP based on input rte_flow_action. * * @param[in] action @@ -466,6 +431,7 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, (int)dcopy->offset < 0 ? off_b : dcopy->offset; /* Convert entire record to big-endian format. */ actions[i].data1 = rte_cpu_to_be_32(actions[i].data1); + ++dcopy; } else { MLX5_ASSERT(item->spec); data = flow_dv_fetch_field((const uint8_t *)item->spec + @@ -994,7 +960,7 @@ flow_dv_convert_action_set_reg RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many items to modify"); MLX5_ASSERT(conf->id != REG_NON); - MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field)); + MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field)); actions[i] = (struct mlx5_modification_cmd) { .action_type = MLX5_MODIFICATION_TYPE_SET, .field = reg_to_field[conf->id], @@ -1256,6 +1222,7 @@ flow_dv_convert_action_set_meta if (reg < 0) return reg; + MLX5_ASSERT(reg != REG_NON); /* * In datapath code there is no endianness * coversions for perfromance reasons, all @@ -1358,6 +1325,476 @@ flow_dv_convert_action_modify_ipv6_dscp MLX5_MODIFICATION_TYPE_SET, error); } +static void +mlx5_flow_field_id_to_modify_info + (const struct rte_flow_action_modify_data *data, + struct field_modify_info *info, + uint32_t *mask, uint32_t *value, uint32_t width, + struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + uint32_t idx = 0; + switch (data->field) { + case RTE_FLOW_FIELD_START: + /* not supported yet */ + MLX5_ASSERT(false); + break; + case RTE_FLOW_FIELD_MAC_DST: + if (mask) { + if (data->offset < 32) { + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DMAC_47_16}; + if (width < 32) { + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + width = 0; + } else { + mask[idx] = RTE_BE32(0xffffffff); + width -= 32; + } + if (!width) + break; + ++idx; + } + info[idx] = (struct field_modify_info){2, 4 * idx, + MLX5_MODI_OUT_DMAC_15_0}; + mask[idx] = rte_cpu_to_be_32(0x0000ffff >> + (16 - width)); + } else { + if (data->offset < 32) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DMAC_47_16}; + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_DMAC_15_0}; + } + break; + case RTE_FLOW_FIELD_MAC_SRC: + if (mask) { + if (data->offset < 32) { + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SMAC_47_16}; + if (width < 32) { + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + width = 0; + } else { + mask[idx] = RTE_BE32(0xffffffff); + width -= 32; + } + if (!width) + break; + ++idx; + } + info[idx] = (struct field_modify_info){2, 4 * idx, + MLX5_MODI_OUT_SMAC_15_0}; + mask[idx] = rte_cpu_to_be_32(0x0000ffff >> + (16 - width)); + } else { + if (data->offset < 32) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SMAC_47_16}; + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_SMAC_15_0}; + } + break; + case RTE_FLOW_FIELD_VLAN_TYPE: + /* not supported yet */ + break; + case RTE_FLOW_FIELD_VLAN_ID: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_FIRST_VID}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x00000fff >> + (12 - width)); + break; + case RTE_FLOW_FIELD_MAC_TYPE: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_ETHERTYPE}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x0000ffff >> + (16 - width)); + break; + case RTE_FLOW_FIELD_IPV4_DSCP: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_IP_DSCP}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x0000003f >> + (6 - width)); + break; + case RTE_FLOW_FIELD_IPV4_TTL: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_IPV4_TTL}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x000000ff >> + (8 - width)); + break; + case RTE_FLOW_FIELD_IPV4_SRC: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV4}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + break; + case RTE_FLOW_FIELD_IPV4_DST: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV4}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + break; + case RTE_FLOW_FIELD_IPV6_DSCP: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_IP_DSCP}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x0000003f >> + (6 - width)); + break; + case RTE_FLOW_FIELD_IPV6_HOPLIMIT: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_IPV6_HOPLIMIT}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x000000ff >> + (8 - width)); + break; + case RTE_FLOW_FIELD_IPV6_SRC: + if (mask) { + if (data->offset < 32) { + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_127_96}; + if (width < 32) { + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + width = 0; + } else { + mask[idx] = RTE_BE32(0xffffffff); + width -= 32; + } + if (!width) + break; + ++idx; + } + if (data->offset < 64) { + info[idx] = (struct field_modify_info){4, + 4 * idx, + MLX5_MODI_OUT_SIPV6_95_64}; + if (width < 32) { + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + width = 0; + } else { + mask[idx] = RTE_BE32(0xffffffff); + width -= 32; + } + if (!width) + break; + ++idx; + } + if (data->offset < 96) { + info[idx] = (struct field_modify_info){4, + 8 * idx, + MLX5_MODI_OUT_SIPV6_63_32}; + if (width < 32) { + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + width = 0; + } else { + mask[idx] = RTE_BE32(0xffffffff); + width -= 32; + } + if (!width) + break; + ++idx; + } + info[idx] = (struct field_modify_info){4, 12 * idx, + MLX5_MODI_OUT_SIPV6_31_0}; + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + } else { + if (data->offset < 32) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_127_96}; + if (data->offset < 64) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_95_64}; + if (data->offset < 96) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_63_32}; + if (data->offset < 128) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_SIPV6_31_0}; + } + break; + case RTE_FLOW_FIELD_IPV6_DST: + if (mask) { + if (data->offset < 32) { + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_127_96}; + if (width < 32) { + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + width = 0; + } else { + mask[idx] = RTE_BE32(0xffffffff); + width -= 32; + } + if (!width) + break; + ++idx; + } + if (data->offset < 64) { + info[idx] = (struct field_modify_info){4, + 4 * idx, + MLX5_MODI_OUT_DIPV6_95_64}; + if (width < 32) { + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + width = 0; + } else { + mask[idx] = RTE_BE32(0xffffffff); + width -= 32; + } + if (!width) + break; + ++idx; + } + if (data->offset < 96) { + info[idx] = (struct field_modify_info){4, + 8 * idx, + MLX5_MODI_OUT_DIPV6_63_32}; + if (width < 32) { + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + width = 0; + } else { + mask[idx] = RTE_BE32(0xffffffff); + width -= 32; + } + if (!width) + break; + ++idx; + } + info[idx] = (struct field_modify_info){4, 12 * idx, + MLX5_MODI_OUT_DIPV6_31_0}; + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + } else { + if (data->offset < 32) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_127_96}; + if (data->offset < 64) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_95_64}; + if (data->offset < 96) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_63_32}; + if (data->offset < 128) + info[idx++] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_DIPV6_31_0}; + } + break; + case RTE_FLOW_FIELD_TCP_PORT_SRC: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_TCP_SPORT}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x0000ffff >> + (16 - width)); + break; + case RTE_FLOW_FIELD_TCP_PORT_DST: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_TCP_DPORT}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x0000ffff >> + (16 - width)); + break; + case RTE_FLOW_FIELD_TCP_SEQ_NUM: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_TCP_SEQ_NUM}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + break; + case RTE_FLOW_FIELD_TCP_ACK_NUM: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_OUT_TCP_ACK_NUM}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + break; + case RTE_FLOW_FIELD_TCP_FLAGS: + info[idx] = (struct field_modify_info){1, 0, + MLX5_MODI_OUT_TCP_FLAGS}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x0000003f >> + (6 - width)); + break; + case RTE_FLOW_FIELD_UDP_PORT_SRC: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_UDP_SPORT}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x0000ffff >> + (16 - width)); + break; + case RTE_FLOW_FIELD_UDP_PORT_DST: + info[idx] = (struct field_modify_info){2, 0, + MLX5_MODI_OUT_UDP_DPORT}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0x0000ffff >> + (16 - width)); + break; + case RTE_FLOW_FIELD_VXLAN_VNI: + /* not supported yet */ + break; + case RTE_FLOW_FIELD_GENEVE_VNI: + /* not supported yet*/ + break; + case RTE_FLOW_FIELD_GTP_TEID: + info[idx] = (struct field_modify_info){4, 0, + MLX5_MODI_GTP_TEID}; + if (mask) + mask[idx] = rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + break; + case RTE_FLOW_FIELD_TAG: + { + int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, + data->level, error); + if (reg < 0) + return; + MLX5_ASSERT(reg != REG_NON); + MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); + info[idx] = (struct field_modify_info){4, 0, + reg_to_field[reg]}; + if (mask) + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + } + break; + case RTE_FLOW_FIELD_MARK: + { + int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, + 0, error); + if (reg < 0) + return; + MLX5_ASSERT(reg != REG_NON); + MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); + info[idx] = (struct field_modify_info){4, 0, + reg_to_field[reg]}; + if (mask) + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + } + break; + case RTE_FLOW_FIELD_META: + { + int reg = flow_dv_get_metadata_reg(dev, attr, error); + if (reg < 0) + return; + MLX5_ASSERT(reg != REG_NON); + MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field)); + info[idx] = (struct field_modify_info){4, 0, + reg_to_field[reg]}; + if (mask) + mask[idx] = + rte_cpu_to_be_32(0xffffffff >> + (32 - width)); + } + break; + case RTE_FLOW_FIELD_POINTER: + for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) { + if (mask[idx]) { + memcpy(&value[idx], + (void *)(uintptr_t)data->value, 32); + value[idx] = rte_cpu_to_be_32(value[idx]); + break; + } + } + break; + case RTE_FLOW_FIELD_VALUE: + for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) { + if (mask[idx]) { + value[idx] = + rte_cpu_to_be_32((uint32_t)data->value); + break; + } + } + break; + default: + MLX5_ASSERT(false); + break; + } +} + +/** + * Convert modify_field action to DV specification. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_field + (struct rte_eth_dev *dev, + struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + const struct rte_flow_action_modify_field *conf = + (const struct rte_flow_action_modify_field *)(action->conf); + struct rte_flow_item item; + struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = { + {0, 0, 0} }; + struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = { + {0, 0, 0} }; + uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; + uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0}; + uint32_t type; + + if (conf->src.field == RTE_FLOW_FIELD_POINTER || + conf->src.field == RTE_FLOW_FIELD_VALUE) { + type = MLX5_MODIFICATION_TYPE_SET; + /** For SET fill the destination field (field) first. */ + mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask, + value, conf->width, dev, attr, error); + /** Then copy immediate value from source as per mask. */ + mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask, + value, conf->width, dev, attr, error); + item.spec = &value; + } else { + type = MLX5_MODIFICATION_TYPE_COPY; + /** For COPY fill the destination field (dcopy) without mask. */ + mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL, + value, conf->width, dev, attr, error); + /** Then construct the source field (field) with mask. */ + mlx5_flow_field_id_to_modify_info(&conf->src, field, mask, + value, conf->width, dev, attr, error); + } + item.mask = &mask; + return flow_dv_convert_modify_action(&item, + field, dcopy, resource, type, error); +} + /** * Validate MARK item. * @@ -1477,6 +1914,10 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, reg = flow_dv_get_metadata_reg(dev, attr, error); if (reg < 0) return reg; + if (reg == REG_NON) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "unavalable extended metadata register"); if (reg == REG_B) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, @@ -1484,13 +1925,20 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, "isn't supported"); if (reg != REG_A) nic_mask.data = priv->sh->dv_meta_mask; - } else if (attr->transfer) { - return rte_flow_error_set(error, ENOTSUP, + } else { + if (attr->transfer) + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, "extended metadata feature " "should be enabled when " "meta item is requested " "with e-switch mode "); + if (attr->ingress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "match on metadata for ingress " + "is not supported in legacy " + "metadata mode"); } if (!mask) mask = &rte_flow_item_meta_mask; @@ -1676,6 +2124,7 @@ flow_dv_validate_item_vlan(const struct rte_flow_item *item, const struct rte_flow_item_vlan nic_mask = { .tci = RTE_BE16(UINT16_MAX), .inner_type = RTE_BE16(UINT16_MAX), + .has_more_vlan = 1, }; const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); int ret; @@ -1792,6 +2241,72 @@ flow_dv_validate_item_gtp(struct rte_eth_dev *dev, MLX5_ITEM_RANGE_NOT_ACCEPTED, error); } +/** + * Validate GTP PSC item. + * + * @param[in] item + * Item specification. + * @param[in] last_item + * Previous validated item in the pattern items. + * @param[in] gtp_item + * Previous GTP item specification. + * @param[in] attr + * Pointer to flow attributes. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item, + uint64_t last_item, + const struct rte_flow_item *gtp_item, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + const struct rte_flow_item_gtp *gtp_spec; + const struct rte_flow_item_gtp *gtp_mask; + const struct rte_flow_item_gtp_psc *spec; + const struct rte_flow_item_gtp_psc *mask; + const struct rte_flow_item_gtp_psc nic_mask = { + .pdu_type = 0xFF, + .qfi = 0xFF, + }; + + if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP)) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "GTP PSC item must be preceded with GTP item"); + gtp_spec = gtp_item->spec; + gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask; + /* GTP spec and E flag is requested to match zero. */ + if (gtp_spec && + (gtp_mask->v_pt_rsv_flags & + ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG)) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "GTP E flag must be 1 to match GTP PSC"); + /* Check the flow is not created in group zero. */ + if (!attr->transfer && !attr->group) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "GTP PSC is not supported for group 0"); + /* GTP spec is here and E flag is requested to match zero. */ + if (!item->spec) + return 0; + spec = item->spec; + mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask; + if (spec->pdu_type > MLX5_GTP_EXT_MAX_PDU_TYPE) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, + "PDU type should be smaller than 16"); + return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_gtp_psc), + MLX5_ITEM_RANGE_NOT_ACCEPTED, error); +} + /** * Validate IPV4 item. * Use existing validation function mlx5_flow_validate_item_ipv4(), and @@ -2408,6 +2923,11 @@ flow_dv_validate_action_mark(struct rte_eth_dev *dev, const struct rte_flow_action_mark *mark = action->conf; int ret; + if (is_tunnel_offload_active(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "no mark action " + "if tunnel offload active"); /* Fall back if no extended metadata register support. */ if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) return mlx5_flow_validate_action_mark(action, action_flags, @@ -2484,6 +3004,10 @@ flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, reg = flow_dv_get_metadata_reg(dev, attr, error); if (reg < 0) return reg; + if (reg == REG_NON) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "unavalable extended metadata register"); if (reg != REG_A && reg != REG_B) { struct mlx5_priv *priv = dev->data->dev_private; @@ -2566,6 +3090,10 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to rte_eth_dev structure. + * @param[in] action + * Pointer to the action structure. + * @param[in] action_flags + * Holds the actions detected until now. * @param[out] error * Pointer to error structure. * @@ -2574,12 +3102,25 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, */ static int flow_dv_validate_action_count(struct rte_eth_dev *dev, + const struct rte_flow_action *action, + uint64_t action_flags, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_action_count *count; if (!priv->config.devx) goto notsup_err; + if (action_flags & MLX5_FLOW_ACTION_COUNT) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "duplicate count actions set"); + count = (const struct rte_flow_action_count *)action->conf; + if (count && count->shared && (action_flags & MLX5_FLOW_ACTION_AGE) && + !priv->sh->flow_hit_aso_en) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "old age and shared count combination is not supported"); #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS return 0; #endif @@ -2641,6 +3182,10 @@ flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, * Pointer to the rte_eth_dev structure. * @param[in] action_flags * Holds the actions detected until now. + * @param[in] action + * Pointer to the action structure. + * @param[in] item_flags + * Holds the items detected. * @param[in] attr * Pointer to flow attributes * @param[out] error @@ -2652,6 +3197,8 @@ flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, static int flow_dv_validate_action_decap(struct rte_eth_dev *dev, uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, const struct rte_flow_attr *attr, struct rte_flow_error *error) { @@ -2685,6 +3232,11 @@ flow_dv_validate_action_decap(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "decap action for VF representor " "not supported on NIC table"); + if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP && + !(item_flags & MLX5_FLOW_LAYER_VXLAN)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "VXLAN item should be present for VXLAN decap"); return 0; } @@ -2705,6 +3257,10 @@ const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,}; * Holds the actions detected until now. * @param[out] actions_n * pointer to the number of actions counter. + * @param[in] action + * Pointer to the action structure. + * @param[in] item_flags + * Holds the items detected. * @param[out] error * Pointer to error structure. * @@ -2717,7 +3273,8 @@ flow_dv_validate_action_raw_encap_decap const struct rte_flow_action_raw_decap *decap, const struct rte_flow_action_raw_encap *encap, const struct rte_flow_attr *attr, uint64_t *action_flags, - int *actions_n, struct rte_flow_error *error) + int *actions_n, const struct rte_flow_action *action, + uint64_t item_flags, struct rte_flow_error *error) { const struct mlx5_priv *priv = dev->data->dev_private; int ret; @@ -2752,8 +3309,8 @@ flow_dv_validate_action_raw_encap_decap "encap combination"); } if (decap) { - ret = flow_dv_validate_action_decap(dev, *action_flags, attr, - error); + ret = flow_dv_validate_action_decap(dev, *action_flags, action, + item_flags, attr, error); if (ret < 0) return ret; *action_flags |= MLX5_FLOW_ACTION_DECAP; @@ -2785,26 +3342,31 @@ flow_dv_validate_action_raw_encap_decap /** * Match encap_decap resource. * + * @param list + * Pointer to the hash list. * @param entry * Pointer to exist resource entry object. - * @param ctx + * @param key + * Key of the new entry. + * @param ctx_cb * Pointer to new encap_decap resource. * * @return - * 0 on matching, -1 otherwise. + * 0 on matching, none-zero otherwise. */ -static int -flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx) +int +flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused, + struct mlx5_hlist_entry *entry, + uint64_t key __rte_unused, void *cb_ctx) { - struct mlx5_flow_dv_encap_decap_resource *resource; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data; struct mlx5_flow_dv_encap_decap_resource *cache_resource; - resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx; cache_resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource, entry); - if (resource->entry.key == cache_resource->entry.key && - resource->reformat_type == cache_resource->reformat_type && + if (resource->reformat_type == cache_resource->reformat_type && resource->ft_type == cache_resource->ft_type && resource->flags == cache_resource->flags && resource->size == cache_resource->size && @@ -2815,6 +3377,63 @@ flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx) return -1; } +/** + * Allocate encap_decap resource. + * + * @param list + * Pointer to the hash list. + * @param entry + * Pointer to exist resource entry object. + * @param ctx_cb + * Pointer to new encap_decap resource. + * + * @return + * 0 on matching, none-zero otherwise. + */ +struct mlx5_hlist_entry * +flow_dv_encap_decap_create_cb(struct mlx5_hlist *list, + uint64_t key __rte_unused, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5dv_dr_domain *domain; + struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data; + struct mlx5_flow_dv_encap_decap_resource *cache_resource; + uint32_t idx; + int ret; + + if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + domain = sh->fdb_domain; + else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) + domain = sh->rx_domain; + else + domain = sh->tx_domain; + /* Register new encap/decap resource. */ + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], + &idx); + if (!cache_resource) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + return NULL; + } + *cache_resource = *resource; + cache_resource->idx = idx; + ret = mlx5_flow_os_create_flow_action_packet_reformat + (sh->ctx, domain, cache_resource, + &cache_resource->action); + if (ret) { + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx); + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create action"); + return NULL; + } + + return &cache_resource->entry; +} + /** * Find existing encap/decap resource or create and register a new one. * @@ -2839,80 +3458,47 @@ flow_dv_encap_decap_resource_register { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_encap_decap_resource *cache_resource; - struct mlx5dv_dr_domain *domain; struct mlx5_hlist_entry *entry; - union mlx5_flow_encap_decap_key encap_decap_key = { + union { + struct { + uint32_t ft_type:8; + uint32_t refmt_type:8; + /* + * Header reformat actions can be shared between + * non-root tables. One bit to indicate non-root + * table or not. + */ + uint32_t is_root:1; + uint32_t reserve:15; + }; + uint32_t v32; + } encap_decap_key = { { .ft_type = resource->ft_type, .refmt_type = resource->reformat_type, - .buf_size = resource->size, - .table_level = !!dev_flow->dv.group, - .cksum = 0, + .is_root = !!dev_flow->dv.group, + .reserve = 0, } }; - int ret; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = resource, + }; + uint64_t key64; resource->flags = dev_flow->dv.group ? 0 : 1; - if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) - domain = sh->fdb_domain; - else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) - domain = sh->rx_domain; - else - domain = sh->tx_domain; - encap_decap_key.cksum = __rte_raw_cksum(resource->buf, - resource->size, 0); - resource->entry.key = encap_decap_key.v64; - /* Lookup a matching resource from cache. */ - entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key, - flow_dv_encap_decap_resource_match, - (void *)resource); - if (entry) { - cache_resource = container_of(entry, - struct mlx5_flow_dv_encap_decap_resource, entry); - DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx; - dev_flow->dv.encap_decap = cache_resource; - return 0; - } - /* Register new encap/decap resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], - &dev_flow->handle->dvh.rix_encap_decap); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate resource memory"); - *cache_resource = *resource; - cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap; - ret = mlx5_flow_os_create_flow_action_packet_reformat - (sh->ctx, domain, cache_resource, - &cache_resource->action); - if (ret) { - mlx5_free(cache_resource); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create action"); - } - rte_atomic32_init(&cache_resource->refcnt); - rte_atomic32_inc(&cache_resource->refcnt); - if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry, - flow_dv_encap_decap_resource_match, - (void *)cache_resource)) { - claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->action)); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], - cache_resource->idx); - return rte_flow_error_set(error, EEXIST, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "action exist"); - } - dev_flow->dv.encap_decap = cache_resource; - DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); + key64 = __rte_raw_cksum(&encap_decap_key.v32, + sizeof(encap_decap_key.v32), 0); + if (resource->reformat_type != + MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 && + resource->size) + key64 = __rte_raw_cksum(resource->buf, resource->size, key64); + entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx); + if (!entry) + return -rte_errno; + resource = container_of(entry, typeof(*resource), entry); + dev_flow->dv.encap_decap = resource; + dev_flow->handle->dvh.rix_encap_decap = resource->idx; return 0; } @@ -2936,70 +3522,63 @@ flow_dv_jump_tbl_resource_register (struct rte_eth_dev *dev __rte_unused, struct mlx5_flow_tbl_resource *tbl, struct mlx5_flow *dev_flow, - struct rte_flow_error *error) + struct rte_flow_error *error __rte_unused) { struct mlx5_flow_tbl_data_entry *tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); - int cnt, ret; MLX5_ASSERT(tbl); - cnt = rte_atomic32_read(&tbl_data->jump.refcnt); - if (!cnt) { - ret = mlx5_flow_os_create_flow_action_dest_flow_tbl - (tbl->obj, &tbl_data->jump.action); - if (ret) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create jump action"); - DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", - (void *)&tbl_data->jump, cnt); - } else { - /* old jump should not make the table ref++. */ - flow_dv_tbl_resource_release(dev, &tbl_data->tbl); - MLX5_ASSERT(tbl_data->jump.action); - DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++", - (void *)&tbl_data->jump, cnt); - } - rte_atomic32_inc(&tbl_data->jump.refcnt); + MLX5_ASSERT(tbl_data->jump.action); dev_flow->handle->rix_jump = tbl_data->idx; dev_flow->dv.jump = &tbl_data->jump; return 0; } -/** - * Find existing default miss resource or create and register a new one. - * - * @param[in, out] dev - * Pointer to rte_eth_dev structure. - * @param[out] error - * pointer to error structure. - * - * @return - * 0 on success otherwise -errno and errno is set. - */ -static int -flow_dv_default_miss_resource_register(struct rte_eth_dev *dev, - struct rte_flow_error *error) +int +flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry, void *cb_ctx) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_default_miss_resource *cache_resource = - &sh->default_miss; - int cnt = rte_atomic32_read(&cache_resource->refcnt); - - if (!cnt) { - MLX5_ASSERT(cache_resource->action); - cache_resource->action = - mlx5_glue->dr_create_flow_action_default_miss(); - if (!cache_resource->action) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot create default miss action"); - DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++", - (void *)cache_resource->action, cnt); - } - rte_atomic32_inc(&cache_resource->refcnt); - return 0; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data; + struct mlx5_flow_dv_port_id_action_resource *res = + container_of(entry, typeof(*res), entry); + + return ref->port_id != res->port_id; +} + +struct mlx5_cache_entry * +flow_dv_port_id_create_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data; + struct mlx5_flow_dv_port_id_action_resource *cache; + uint32_t idx; + int ret; + + /* Register new port id action resource. */ + cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx); + if (!cache) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate port_id action cache memory"); + return NULL; + } + *cache = *ref; + ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain, + ref->port_id, + &cache->action); + if (ret) { + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx); + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot create action"); + return NULL; + } + cache->idx = idx; + return &cache->entry; } /** @@ -3025,52 +3604,73 @@ flow_dv_port_id_action_resource_register struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_port_id_action_resource *cache_resource; - uint32_t idx = 0; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_port_id_action_resource *cache; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = resource, + }; + + entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx); + if (!entry) + return -rte_errno; + cache = container_of(entry, typeof(*cache), entry); + dev_flow->dv.port_id_action = cache; + dev_flow->handle->rix_port_id_action = cache->idx; + return 0; +} + +int +flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data; + struct mlx5_flow_dv_push_vlan_action_resource *res = + container_of(entry, typeof(*res), entry); + + return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type; +} + +struct mlx5_cache_entry * +flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data; + struct mlx5_flow_dv_push_vlan_action_resource *cache; + struct mlx5dv_dr_domain *domain; + uint32_t idx; int ret; - /* Lookup a matching resource from cache. */ - ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list, - idx, cache_resource, next) { - if (resource->port_id == cache_resource->port_id) { - DRV_LOG(DEBUG, "port id action resource resource %p: " - "refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->rix_port_id_action = idx; - dev_flow->dv.port_id_action = cache_resource; - return 0; - } - } /* Register new port id action resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], - &dev_flow->handle->rix_port_id_action); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate resource memory"); - *cache_resource = *resource; - ret = mlx5_flow_os_create_flow_action_dest_port - (priv->sh->fdb_domain, resource->port_id, - &cache_resource->action); + cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx); + if (!cache) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate push_vlan action cache memory"); + return NULL; + } + *cache = *ref; + if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + domain = sh->fdb_domain; + else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) + domain = sh->rx_domain; + else + domain = sh->tx_domain; + ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag, + &cache->action); if (ret) { - mlx5_free(cache_resource); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create action"); - } - rte_atomic32_init(&cache_resource->refcnt); - rte_atomic32_inc(&cache_resource->refcnt); - ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list, - dev_flow->handle->rix_port_id_action, cache_resource, - next); - dev_flow->dv.port_id_action = cache_resource; - DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - return 0; + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot create push vlan action"); + return NULL; + } + cache->idx = idx; + return &cache->entry; } /** @@ -3096,62 +3696,23 @@ flow_dv_push_vlan_action_resource_register struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; - struct mlx5dv_dr_domain *domain; - uint32_t idx = 0; - int ret; + struct mlx5_flow_dv_push_vlan_action_resource *cache; + struct mlx5_cache_entry *entry; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = resource, + }; - /* Lookup a matching resource from cache. */ - ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN], - sh->push_vlan_action_list, idx, cache_resource, next) { - if (resource->vlan_tag == cache_resource->vlan_tag && - resource->ft_type == cache_resource->ft_type) { - DRV_LOG(DEBUG, "push-VLAN action resource resource %p: " - "refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.rix_push_vlan = idx; - dev_flow->dv.push_vlan_res = cache_resource; - return 0; - } - } - /* Register new push_vlan action resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], - &dev_flow->handle->dvh.rix_push_vlan); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate resource memory"); - *cache_resource = *resource; - if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) - domain = sh->fdb_domain; - else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) - domain = sh->rx_domain; - else - domain = sh->tx_domain; - ret = mlx5_flow_os_create_flow_action_push_vlan - (domain, resource->vlan_tag, - &cache_resource->action); - if (ret) { - mlx5_free(cache_resource); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create action"); - } - rte_atomic32_init(&cache_resource->refcnt); - rte_atomic32_inc(&cache_resource->refcnt); - ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN], - &sh->push_vlan_action_list, - dev_flow->handle->dvh.rix_push_vlan, - cache_resource, next); - dev_flow->dv.push_vlan_res = cache_resource; - DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); + entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx); + if (!entry) + return -rte_errno; + cache = container_of(entry, typeof(*cache), entry); + + dev_flow->handle->dvh.rix_push_vlan = cache->idx; + dev_flow->dv.push_vlan_res = cache; return 0; } + /** * Get the size of specific rte_flow_item_type hdr size * @@ -3607,8 +4168,6 @@ flow_dv_create_action_push_vlan(struct rte_eth_dev *dev, (dev, &res, dev_flow, error); } -static int fdb_mirror; - /** * Validate the modify-header actions. * @@ -3636,12 +4195,6 @@ flow_dv_validate_action_modify_hdr(const uint64_t action_flags, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can't have encap action before" " modify action"); - if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't support sample action before" - " modify action for E-Switch" - " mirroring"); return 0; } @@ -3929,6 +4482,214 @@ flow_dv_validate_action_modify_ttl(const uint64_t action_flags, return ret; } +static int +mlx5_flow_item_field_width(enum rte_flow_field_id field) +{ + switch (field) { + case RTE_FLOW_FIELD_START: + return 32; + case RTE_FLOW_FIELD_MAC_DST: + case RTE_FLOW_FIELD_MAC_SRC: + return 48; + case RTE_FLOW_FIELD_VLAN_TYPE: + return 16; + case RTE_FLOW_FIELD_VLAN_ID: + return 12; + case RTE_FLOW_FIELD_MAC_TYPE: + return 16; + case RTE_FLOW_FIELD_IPV4_DSCP: + return 6; + case RTE_FLOW_FIELD_IPV4_TTL: + return 8; + case RTE_FLOW_FIELD_IPV4_SRC: + case RTE_FLOW_FIELD_IPV4_DST: + return 32; + case RTE_FLOW_FIELD_IPV6_DSCP: + return 6; + case RTE_FLOW_FIELD_IPV6_HOPLIMIT: + return 8; + case RTE_FLOW_FIELD_IPV6_SRC: + case RTE_FLOW_FIELD_IPV6_DST: + return 128; + case RTE_FLOW_FIELD_TCP_PORT_SRC: + case RTE_FLOW_FIELD_TCP_PORT_DST: + return 16; + case RTE_FLOW_FIELD_TCP_SEQ_NUM: + case RTE_FLOW_FIELD_TCP_ACK_NUM: + return 32; + case RTE_FLOW_FIELD_TCP_FLAGS: + return 6; + case RTE_FLOW_FIELD_UDP_PORT_SRC: + case RTE_FLOW_FIELD_UDP_PORT_DST: + return 16; + case RTE_FLOW_FIELD_VXLAN_VNI: + case RTE_FLOW_FIELD_GENEVE_VNI: + return 24; + case RTE_FLOW_FIELD_GTP_TEID: + case RTE_FLOW_FIELD_TAG: + return 32; + case RTE_FLOW_FIELD_MARK: + return 24; + case RTE_FLOW_FIELD_META: + case RTE_FLOW_FIELD_POINTER: + case RTE_FLOW_FIELD_VALUE: + return 32; + default: + MLX5_ASSERT(false); + } + return 0; +} + +/** + * Validate the generic modify field actions. + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] attr + * Pointer to the flow attributes. + * @param[out] error + * Pointer to error structure. + * + * @return + * Number of header fields to modify (0 or more) on success, + * a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_field(struct rte_eth_dev *dev, + const uint64_t action_flags, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + int ret = 0; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; + const struct rte_flow_action_modify_field *action_modify_field = + action->conf; + uint32_t dst_width = + mlx5_flow_item_field_width(action_modify_field->dst.field); + uint32_t src_width = + mlx5_flow_item_field_width(action_modify_field->src.field); + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (ret) + return ret; + + if (action_modify_field->width == 0) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no bits are requested to be modified"); + else if (action_modify_field->width > dst_width || + action_modify_field->width > src_width) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot modify more bits than" + " the width of a field"); + if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE && + action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) { + if ((action_modify_field->dst.offset + + action_modify_field->width > dst_width) || + (action_modify_field->dst.offset % 32)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "destination offset is too big" + " or not aligned to 4 bytes"); + if (action_modify_field->dst.level && + action_modify_field->dst.field != RTE_FLOW_FIELD_TAG) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot modify inner headers"); + } + if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE && + action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) { + if (!attr->transfer && !attr->group) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "modify field action " + "is not supported for group 0"); + if ((action_modify_field->src.offset + + action_modify_field->width > src_width) || + (action_modify_field->src.offset % 32)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "source offset is too big" + " or not aligned to 4 bytes"); + if (action_modify_field->src.level && + action_modify_field->src.field != RTE_FLOW_FIELD_TAG) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot copy from inner headers"); + } + if (action_modify_field->dst.field == + action_modify_field->src.field) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "source and destination fields" + " cannot be the same"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE || + action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "immediate value or a pointer to it" + " cannot be used as a destination"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_START || + action_modify_field->src.field == RTE_FLOW_FIELD_START) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "modifications of an arbitrary" + " place in a packet is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE || + action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "modifications of the 802.1Q Tag" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI || + action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "modifications of the VXLAN Network" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI || + action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "modifications of the GENEVE Network" + " Identifier is not supported"); + if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK || + action_modify_field->src.field == RTE_FLOW_FIELD_MARK) { + if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || + !mlx5_flow_ext_mreg_supported(dev)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "cannot modify mark without extended" + " metadata register support"); + } + if (action_modify_field->operation != RTE_FLOW_MODIFY_SET) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "add and sub operations" + " are not supported"); + return (action_modify_field->width / 32) + + !!(action_modify_field->width % 32); +} + /** * Validate jump action. * @@ -3972,12 +4733,6 @@ flow_dv_validate_action_jump(struct rte_eth_dev *dev, return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "jump with meter not support"); - if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "E-Switch mirroring can't support" - " Sample action and jump action in" - " same flow now"); if (!action->conf) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, @@ -3985,7 +4740,7 @@ flow_dv_validate_action_jump(struct rte_eth_dev *dev, target_group = ((const struct rte_flow_action_jump *)action->conf)->group; ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table, - grp_info, error); + &grp_info, error); if (ret) return ret; if (attributes->group == target_group && @@ -4179,7 +4934,8 @@ flow_dv_validate_action_age(uint64_t action_flags, struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_age *age = action->conf; - if (!priv->config.devx || priv->sh->cmng.counter_fallback) + if (!priv->config.devx || (priv->sh->cmng.counter_fallback && + !priv->sh->aso_age_mng)) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -4270,41 +5026,81 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags, /** * Match modify-header resource. * + * @param list + * Pointer to the hash list. * @param entry * Pointer to exist resource entry object. + * @param key + * Key of the new entry. * @param ctx * Pointer to new modify-header resource. * * @return - * 0 on matching, -1 otherwise. + * 0 on matching, non-zero otherwise. */ -static int -flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx) +int +flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused, + struct mlx5_hlist_entry *entry, + uint64_t key __rte_unused, void *cb_ctx) { - struct mlx5_flow_dv_modify_hdr_resource *resource; - struct mlx5_flow_dv_modify_hdr_resource *cache_resource; - uint32_t actions_len; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data; + struct mlx5_flow_dv_modify_hdr_resource *resource = + container_of(entry, typeof(*resource), entry); + uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type); - resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx; - cache_resource = container_of(entry, - struct mlx5_flow_dv_modify_hdr_resource, - entry); - actions_len = resource->actions_num * sizeof(resource->actions[0]); - if (resource->entry.key == cache_resource->entry.key && - resource->ft_type == cache_resource->ft_type && - resource->actions_num == cache_resource->actions_num && - resource->flags == cache_resource->flags && - !memcmp((const void *)resource->actions, - (const void *)cache_resource->actions, - actions_len)) - return 0; - return -1; + key_len += ref->actions_num * sizeof(ref->actions[0]); + return ref->actions_num != resource->actions_num || + memcmp(&ref->ft_type, &resource->ft_type, key_len); +} + +struct mlx5_hlist_entry * +flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5dv_dr_domain *ns; + struct mlx5_flow_dv_modify_hdr_resource *entry; + struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data; + int ret; + uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]); + uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type); + + entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0, + SOCKET_ID_ANY); + if (!entry) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + return NULL; + } + rte_memcpy(&entry->ft_type, + RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)), + key_len + data_len); + if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + ns = sh->fdb_domain; + else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) + ns = sh->tx_domain; + else + ns = sh->rx_domain; + ret = mlx5_flow_os_create_flow_action_modify_header + (sh->ctx, ns, entry, + data_len, &entry->action); + if (ret) { + mlx5_free(entry); + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create modification action"); + return NULL; + } + return &entry->entry; } /** * Validate the sample action. * - * @param[in] action_flags + * @param[in, out] action_flags * Holds the actions detected until now. * @param[in] action * Pointer to the sample action. @@ -4312,6 +5108,16 @@ flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx) * Pointer to the Ethernet device structure. * @param[in] attr * Attributes of flow that includes this action. + * @param[in] item_flags + * Holds the items detected. + * @param[in] rss + * Pointer to the RSS action. + * @param[out] sample_rss + * Pointer to the RSS action in sample action list. + * @param[out] count + * Pointer to the COUNT action in sample action list. + * @param[out] fdb_mirror_limit + * Pointer to the FDB mirror limitation flag. * @param[out] error * Pointer to error structure. * @@ -4319,10 +5125,15 @@ flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx) * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -flow_dv_validate_action_sample(uint64_t action_flags, +flow_dv_validate_action_sample(uint64_t *action_flags, const struct rte_flow_action *action, struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + uint64_t item_flags, + const struct rte_flow_action_rss *rss, + const struct rte_flow_action_rss **sample_rss, + const struct rte_flow_action_count **count, + int *fdb_mirror_limit, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; @@ -4333,7 +5144,6 @@ flow_dv_validate_action_sample(uint64_t action_flags, uint16_t queue_index = 0xFFFF; int actions_n = 0; int ret; - fdb_mirror = 0; if (!sample) return rte_flow_error_set(error, EINVAL, @@ -4348,17 +5158,17 @@ flow_dv_validate_action_sample(uint64_t action_flags, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "sample action not supported"); - if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + if (*action_flags & MLX5_FLOW_ACTION_SAMPLE) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Multiple sample actions not " "supported"); - if (action_flags & MLX5_FLOW_ACTION_METER) + if (*action_flags & MLX5_FLOW_ACTION_METER) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "wrong action order, meter should " "be after sample action"); - if (action_flags & MLX5_FLOW_ACTION_JUMP) + if (*action_flags & MLX5_FLOW_ACTION_JUMP) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "wrong action order, jump should " @@ -4382,6 +5192,28 @@ flow_dv_validate_action_sample(uint64_t action_flags, sub_action_flags |= MLX5_FLOW_ACTION_QUEUE; ++actions_n; break; + case RTE_FLOW_ACTION_TYPE_RSS: + *sample_rss = act->conf; + ret = mlx5_flow_validate_action_rss(act, + sub_action_flags, + dev, attr, + item_flags, + error); + if (ret < 0) + return ret; + if (rss && *sample_rss && + ((*sample_rss)->level != rss->level || + (*sample_rss)->types != rss->types)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "Can't use the different RSS types " + "or level in the same flow"); + if (*sample_rss != NULL && (*sample_rss)->queue_num) + queue_index = (*sample_rss)->queue[0]; + sub_action_flags |= MLX5_FLOW_ACTION_RSS; + ++actions_n; + break; case RTE_FLOW_ACTION_TYPE_MARK: ret = flow_dv_validate_action_mark(dev, act, sub_action_flags, @@ -4396,10 +5228,15 @@ flow_dv_validate_action_sample(uint64_t action_flags, ++actions_n; break; case RTE_FLOW_ACTION_TYPE_COUNT: - ret = flow_dv_validate_action_count(dev, error); + ret = flow_dv_validate_action_count + (dev, act, + *action_flags | sub_action_flags, + error); if (ret < 0) return ret; + *count = act->conf; sub_action_flags |= MLX5_FLOW_ACTION_COUNT; + *action_flags |= MLX5_FLOW_ACTION_COUNT; ++actions_n; break; case RTE_FLOW_ACTION_TYPE_PORT_ID: @@ -4416,7 +5253,7 @@ flow_dv_validate_action_sample(uint64_t action_flags, case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: ret = flow_dv_validate_action_raw_encap_decap (dev, NULL, act->conf, attr, &sub_action_flags, - &actions_n, error); + &actions_n, action, item_flags, error); if (ret < 0) return ret; ++actions_n; @@ -4430,7 +5267,8 @@ flow_dv_validate_action_sample(uint64_t action_flags, } } if (attr->ingress && !attr->transfer) { - if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE)) + if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE | + MLX5_FLOW_ACTION_RSS))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -4451,18 +5289,25 @@ flow_dv_validate_action_sample(uint64_t action_flags, "E-Switch doesn't support " "any optional action " "for sampling"); - fdb_mirror = 1; if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "unsupported action QUEUE"); + if (sub_action_flags & MLX5_FLOW_ACTION_RSS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported action QUEUE"); if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "E-Switch must has a dest " "port for mirroring"); + if (!priv->config.hca_attr.reg_c_preserve && + priv->representor_id != -1) + *fdb_mirror_limit = 1; } /* Continue validation for Xcap actions.*/ if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) && @@ -4509,19 +5354,15 @@ flow_dv_modify_hdr_resource_register { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_modify_hdr_resource *cache_resource; - struct mlx5dv_dr_domain *ns; - uint32_t actions_len; + uint32_t key_len = sizeof(*resource) - + offsetof(typeof(*resource), ft_type) + + resource->actions_num * sizeof(resource->actions[0]); struct mlx5_hlist_entry *entry; - union mlx5_flow_modify_hdr_key hdr_mod_key = { - { - .ft_type = resource->ft_type, - .actions_num = resource->actions_num, - .group = dev_flow->dv.group, - .cksum = 0, - } + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = resource, }; - int ret; + uint64_t key64; resource->flags = dev_flow->dv.group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; @@ -4530,66 +5371,12 @@ flow_dv_modify_hdr_resource_register return rte_flow_error_set(error, EOVERFLOW, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many modify header items"); - if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) - ns = sh->fdb_domain; - else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) - ns = sh->tx_domain; - else - ns = sh->rx_domain; - /* Lookup a matching resource from cache. */ - actions_len = resource->actions_num * sizeof(resource->actions[0]); - hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0); - resource->entry.key = hdr_mod_key.v64; - entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key, - flow_dv_modify_hdr_resource_match, - (void *)resource); - if (entry) { - cache_resource = container_of(entry, - struct mlx5_flow_dv_modify_hdr_resource, - entry); - DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->handle->dvh.modify_hdr = cache_resource; - return 0; - - } - /* Register new modify-header resource. */ - cache_resource = mlx5_malloc(MLX5_MEM_ZERO, - sizeof(*cache_resource) + actions_len, 0, - SOCKET_ID_ANY); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate resource memory"); - *cache_resource = *resource; - rte_memcpy(cache_resource->actions, resource->actions, actions_len); - ret = mlx5_flow_os_create_flow_action_modify_header - (sh->ctx, ns, cache_resource, - actions_len, &cache_resource->action); - if (ret) { - mlx5_free(cache_resource); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create action"); - } - rte_atomic32_init(&cache_resource->refcnt); - rte_atomic32_inc(&cache_resource->refcnt); - if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry, - flow_dv_modify_hdr_resource_match, - (void *)cache_resource)) { - claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->action)); - mlx5_free(cache_resource); - return rte_flow_error_set(error, EEXIST, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "action exist"); - } - dev_flow->handle->dvh.modify_hdr = cache_resource; - DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); + key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0); + entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx); + if (!entry) + return -rte_errno; + resource = container_of(entry, typeof(*resource), entry); + dev_flow->handle->dvh.modify_hdr = resource; return 0; } @@ -5107,7 +5894,7 @@ flow_dv_counter_remove_from_age(struct rte_eth_dev *dev, * Index to the counter handler. */ static void -flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter) +flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_counter_pool *pool = NULL; @@ -5171,11 +5958,11 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, const struct mlx5_flow_tunnel *tunnel, const struct rte_flow_attr *attributes, - struct flow_grp_info grp_info, + const struct flow_grp_info *grp_info, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - uint32_t priority_max = priv->config.flow_prio - 1; + uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes); int ret = 0; #ifndef HAVE_MLX5DV_DR @@ -5196,8 +5983,8 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, if (!table) ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; #endif - if (attributes->priority != MLX5_FLOW_PRIO_RSVD && - attributes->priority >= priority_max) + if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && + attributes->priority > lowest_priority) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, NULL, @@ -5262,10 +6049,17 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, uint16_t ether_type = 0; int actions_n = 0; uint8_t item_ipv6_proto = 0; + int fdb_mirror_limit = 0; + int modify_after_mirror = 0; + const struct rte_flow_item *geneve_item = NULL; const struct rte_flow_item *gre_item = NULL; + const struct rte_flow_item *gtp_item = NULL; const struct rte_flow_action_raw_decap *decap; const struct rte_flow_action_raw_encap *encap; - const struct rte_flow_action_rss *rss; + const struct rte_flow_action_rss *rss = NULL; + const struct rte_flow_action_rss *sample_rss = NULL; + const struct rte_flow_action_count *count = NULL; + const struct rte_flow_action_count *sample_count = NULL; const struct rte_flow_item_tcp nic_tcp_mask = { .hdr = { .tcp_flags = 0xFF, @@ -5302,7 +6096,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, struct mlx5_dev_config *dev_conf = &priv->config; uint16_t queue_index = 0xFFFF; const struct rte_flow_item_vlan *vlan_m = NULL; - int16_t rw_act_num = 0; + uint32_t rw_act_num = 0; uint64_t is_root; const struct mlx5_flow_tunnel *tunnel; struct flow_grp_info grp_info = { @@ -5324,9 +6118,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, } else { tunnel = NULL; } + if (tunnel && priv->representor) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "decap not supported " + "for VF representor"); grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate (dev, tunnel, attr, items, actions); - ret = flow_dv_validate_attributes(dev, tunnel, attr, grp_info, error); + ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error); if (ret < 0) return ret; is_root = (uint64_t)ret; @@ -5359,7 +6158,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, break; case RTE_FLOW_ITEM_TYPE_ETH: ret = mlx5_flow_validate_item_eth(items, item_flags, - error); + true, error); if (ret < 0) return ret; last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : @@ -5538,8 +6337,19 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + geneve_item = items; last_item = MLX5_FLOW_LAYER_GENEVE; break; + case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: + ret = mlx5_flow_validate_item_geneve_opt(items, + last_item, + geneve_item, + dev, + error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_LAYER_GENEVE_OPT; + break; case RTE_FLOW_ITEM_TYPE_MPLS: ret = mlx5_flow_validate_item_mpls(dev, items, item_flags, @@ -5595,8 +6405,17 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + gtp_item = items; last_item = MLX5_FLOW_LAYER_GTP; break; + case RTE_FLOW_ITEM_TYPE_GTP_PSC: + ret = flow_dv_validate_item_gtp_psc(items, last_item, + gtp_item, attr, + error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_LAYER_GTP_PSC; + break; case RTE_FLOW_ITEM_TYPE_ECPRI: /* Capacity will be checked in the translate stage. */ ret = mlx5_flow_validate_item_ecpri(items, item_flags, @@ -5653,6 +6472,9 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; action_flags |= MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK_EXT; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; + } else { action_flags |= MLX5_FLOW_ACTION_FLAG; ++actions_n; @@ -5672,6 +6494,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; action_flags |= MLX5_FLOW_ACTION_MARK | MLX5_FLOW_ACTION_MARK_EXT; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; } else { action_flags |= MLX5_FLOW_ACTION_MARK; ++actions_n; @@ -5687,6 +6511,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_META; rw_act_num += MLX5_ACT_NUM_SET_META; break; @@ -5699,6 +6525,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_TAG; rw_act_num += MLX5_ACT_NUM_SET_TAG; break; @@ -5729,6 +6557,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + if (rss && sample_rss && + (sample_rss->level != rss->level || + sample_rss->types != rss->types)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "Can't use the different RSS types " + "or level in the same flow"); if (rss != NULL && rss->queue_num) queue_index = rss->queue[0]; action_flags |= MLX5_FLOW_ACTION_RSS; @@ -5744,9 +6580,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; break; case RTE_FLOW_ACTION_TYPE_COUNT: - ret = flow_dv_validate_action_count(dev, error); + ret = flow_dv_validate_action_count(dev, actions, + action_flags, + error); if (ret < 0) return ret; + count = actions->conf; action_flags |= MLX5_FLOW_ACTION_COUNT; ++actions_n; break; @@ -5803,6 +6642,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: ret = flow_dv_validate_action_decap(dev, action_flags, + actions, item_flags, attr, error); if (ret < 0) return ret; @@ -5812,7 +6652,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: ret = flow_dv_validate_action_raw_encap_decap (dev, NULL, actions->conf, attr, &action_flags, - &actions_n, error); + &actions_n, actions, item_flags, error); if (ret < 0) return ret; break; @@ -5830,7 +6670,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, (dev, decap ? decap : &empty_decap, encap, attr, &action_flags, &actions_n, - error); + actions, item_flags, error); if (ret < 0) return ret; break; @@ -5849,6 +6689,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? MLX5_FLOW_ACTION_SET_MAC_SRC : MLX5_FLOW_ACTION_SET_MAC_DST; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; /* * Even if the source and destination MAC addresses have * overlap in the header with 4B alignment, the convert @@ -5869,6 +6711,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? MLX5_FLOW_ACTION_SET_IPV4_SRC : @@ -5892,6 +6736,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? MLX5_FLOW_ACTION_SET_IPV6_SRC : @@ -5909,6 +6755,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? MLX5_FLOW_ACTION_SET_TP_SRC : @@ -5926,6 +6774,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_TTL ? MLX5_FLOW_ACTION_SET_TTL : @@ -5939,6 +6789,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret) return ret; + if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && + fdb_mirror_limit) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "sample and jump action combination is not supported"); ++actions_n; action_flags |= MLX5_FLOW_ACTION_JUMP; break; @@ -5954,6 +6810,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? MLX5_FLOW_ACTION_INC_TCP_SEQ : @@ -5972,6 +6830,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? MLX5_FLOW_ACTION_INC_TCP_ACK : @@ -5996,12 +6856,39 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Meter action will add one more TAG action. */ rw_act_num += MLX5_ACT_NUM_SET_TAG; break; + case MLX5_RTE_FLOW_ACTION_TYPE_AGE: + if (!attr->transfer && !attr->group) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Shared ASO age action is not supported for group 0"); + action_flags |= MLX5_FLOW_ACTION_AGE; + ++actions_n; + break; case RTE_FLOW_ACTION_TYPE_AGE: ret = flow_dv_validate_action_age(action_flags, actions, dev, error); if (ret < 0) return ret; + /* + * Validate the regular AGE action (using counter) + * mutual exclusion with share counter actions. + */ + if (!priv->sh->flow_hit_aso_en) { + if (count && count->shared) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "old age and shared count combination is not supported"); + if (sample_count) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "old age action and count must be in the same sub flow"); + } action_flags |= MLX5_FLOW_ACTION_AGE; ++actions_n; break; @@ -6016,6 +6903,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; rw_act_num += MLX5_ACT_NUM_SET_DSCP; break; @@ -6030,13 +6919,19 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) + modify_after_mirror = 1; action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; rw_act_num += MLX5_ACT_NUM_SET_DSCP; break; case RTE_FLOW_ACTION_TYPE_SAMPLE: - ret = flow_dv_validate_action_sample(action_flags, + ret = flow_dv_validate_action_sample(&action_flags, actions, dev, - attr, error); + attr, item_flags, + rss, &sample_rss, + &sample_count, + &fdb_mirror_limit, + error); if (ret < 0) return ret; action_flags |= MLX5_FLOW_ACTION_SAMPLE; @@ -6053,6 +6948,20 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET; break; + case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: + ret = flow_dv_validate_action_modify_field(dev, + action_flags, + actions, + attr, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_ACTION_MODIFY_FIELD)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD; + rw_act_num += ret; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -6112,8 +7021,11 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, * Validate the drop action mutual exclusion with other actions. * Drop action is mutually-exclusive with any other action, except for * Count action. + * Drop action compatibility with tunnel offload was already validated. */ - if ((action_flags & MLX5_FLOW_ACTION_DROP) && + if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH | + MLX5_FLOW_ACTION_TUNNEL_MATCH)); + else if ((action_flags & MLX5_FLOW_ACTION_DROP) && (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -6207,13 +7119,18 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && mlx5_flow_ext_mreg_supported(dev)) rw_act_num += MLX5_ACT_NUM_SET_TAG; - if ((uint32_t)rw_act_num > + if (rw_act_num > flow_dv_modify_hdr_action_max(dev, is_root)) { return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many header modify" " actions to support"); } + /* Eswitch egress mirror and modify flow has limitation on CX5 */ + if (fdb_mirror_limit && modify_after_mirror) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "sample before modify action is not supported"); return 0; } @@ -6247,9 +7164,11 @@ flow_dv_prepare(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow; struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + MLX5_ASSERT(wks); /* In case of corrupting the memory. */ - if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { + if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { rte_flow_error_set(error, ENOSPC, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "not free temporary device flow"); @@ -6263,8 +7182,9 @@ flow_dv_prepare(struct rte_eth_dev *dev, "not enough memory to create flow handle"); return NULL; } - /* No multi-thread supporting. */ - dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; + MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows)); + dev_flow = &wks->flows[wks->flow_idx++]; + memset(dev_flow, 0, sizeof(*dev_flow)); dev_flow->handle = dev_handle; dev_flow->handle_idx = handle_idx; /* @@ -6276,12 +7196,6 @@ flow_dv_prepare(struct rte_eth_dev *dev, */ dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) - MLX5_ST_SZ_BYTES(fte_match_set_misc4); - /* - * The matching value needs to be cleared to 0 before using. In the - * past, it will be automatically cleared when using rte_*alloc - * API. The time consumption will be almost the same as before. - */ - memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param)); dev_flow->ingress = attr->ingress; dev_flow->dv.transfer = attr->transfer; return dev_flow; @@ -6371,9 +7285,10 @@ flow_dv_translate_item_eth(void *matcher, void *key, .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", .type = RTE_BE16(0xffff), + .has_vlan = 0, }; - void *headers_m; - void *headers_v; + void *hdrs_m; + void *hdrs_v; char *l24_v; unsigned int i; @@ -6382,38 +7297,26 @@ flow_dv_translate_item_eth(void *matcher, void *key, if (!eth_m) eth_m = &nic_mask; if (inner) { - headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, inner_headers); - headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); + hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); } else { - headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); - headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); + hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); } - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16), + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16), ð_m->dst, sizeof(eth_m->dst)); /* The value must be in the range of the mask. */ - l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16); + l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16); for (i = 0; i < sizeof(eth_m->dst); ++i) l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i]; - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16), + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16), ð_m->src, sizeof(eth_m->src)); - l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16); + l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16); /* The value must be in the range of the mask. */ for (i = 0; i < sizeof(eth_m->dst); ++i) l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i]; - if (eth_v->type) { - /* When ethertype is present set mask for tagged VLAN. */ - MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); - /* Set value for tagged VLAN if ethertype is 802.1Q. */ - if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) || - eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) { - MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, - 1); - /* Return here to avoid setting match on ethertype. */ - return; - } - } /* * HW supports match on one Ethertype, the Ethertype following the last * VLAN tag of the packet (see PRM). @@ -6422,19 +7325,42 @@ flow_dv_translate_item_eth(void *matcher, void *key, * ethertype, and use ip_version field instead. * eCPRI over Ether layer will use type value 0xAEFE. */ - if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) && - eth_m->type == 0xFFFF) { - flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); - } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) && - eth_m->type == 0xFFFF) { - flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); - } else { - MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, - rte_be_to_cpu_16(eth_m->type)); - l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, - ethertype); - *(uint16_t *)(l24_v) = eth_m->type & eth_v->type; + if (eth_m->type == 0xFFFF) { + /* Set cvlan_tag mask for any single\multi\un-tagged case. */ + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1); + switch (eth_v->type) { + case RTE_BE16(RTE_ETHER_TYPE_VLAN): + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1); + return; + case RTE_BE16(RTE_ETHER_TYPE_QINQ): + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1); + return; + case RTE_BE16(RTE_ETHER_TYPE_IPV4): + flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4); + return; + case RTE_BE16(RTE_ETHER_TYPE_IPV6): + flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6); + return; + default: + break; + } + } + if (eth_m->has_vlan) { + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1); + if (eth_v->has_vlan) { + /* + * Here, when also has_more_vlan field in VLAN item is + * not set, only single-tagged packets will be matched. + */ + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1); + return; + } } + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype, + rte_be_to_cpu_16(eth_m->type)); + l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype); + *(uint16_t *)(l24_v) = eth_m->type & eth_v->type; } /** @@ -6459,19 +7385,19 @@ flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, { const struct rte_flow_item_vlan *vlan_m = item->mask; const struct rte_flow_item_vlan *vlan_v = item->spec; - void *headers_m; - void *headers_v; + void *hdrs_m; + void *hdrs_v; uint16_t tci_m; uint16_t tci_v; if (inner) { - headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, inner_headers); - headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); + hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); } else { - headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); - headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); + hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); /* * This is workaround, masks are not supported, * and pre-validated. @@ -6484,37 +7410,54 @@ flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, * When VLAN item exists in flow, mark packet as tagged, * even if TCI is not specified. */ - MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); - MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); + if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) { + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1); + } if (!vlan_v) return; if (!vlan_m) vlan_m = &rte_flow_item_vlan_mask; tci_m = rte_be_to_cpu_16(vlan_m->tci); tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci); - MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m); - MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v); - MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12); - MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12); - MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13); - MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13); /* * HW is optimized for IPv4/IPv6. In such cases, avoid setting * ethertype, and use ip_version field instead. */ - if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) && - vlan_m->inner_type == 0xFFFF) { - flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); - } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) && - vlan_m->inner_type == 0xFFFF) { - flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); - } else { - MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, - rte_be_to_cpu_16(vlan_m->inner_type)); - MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, - rte_be_to_cpu_16(vlan_m->inner_type & - vlan_v->inner_type)); + if (vlan_m->inner_type == 0xFFFF) { + switch (vlan_v->inner_type) { + case RTE_BE16(RTE_ETHER_TYPE_VLAN): + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0); + return; + case RTE_BE16(RTE_ETHER_TYPE_IPV4): + flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4); + return; + case RTE_BE16(RTE_ETHER_TYPE_IPV6): + flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6); + return; + default: + break; + } + } + if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) { + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1); + /* Only one vlan_tag bit can be set. */ + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0); + return; } + MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype, + rte_be_to_cpu_16(vlan_m->inner_type)); + MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype, + rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type)); } /** @@ -6526,8 +7469,6 @@ flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, * Flow matcher value. * @param[in] item * Flow pattern to translate. - * @param[in] item_flags - * Bit-fields that holds the items detected until now. * @param[in] inner * Item is inner pattern. * @param[in] group @@ -6536,7 +7477,6 @@ flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, static void flow_dv_translate_item_ipv4(void *matcher, void *key, const struct rte_flow_item *item, - const uint64_t item_flags, int inner, uint32_t group) { const struct rte_flow_item_ipv4 *ipv4_m = item->mask; @@ -6566,13 +7506,6 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); } flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); - /* - * On outer header (which must contains L2), or inner header with L2, - * set cvlan_tag mask bit to mark this packet as untagged. - * This should be done even if item->spec is empty. - */ - if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) - MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); if (!ipv4_v) return; if (!ipv4_m) @@ -6619,8 +7552,6 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, * Flow matcher value. * @param[in] item * Flow pattern to translate. - * @param[in] item_flags - * Bit-fields that holds the items detected until now. * @param[in] inner * Item is inner pattern. * @param[in] group @@ -6629,7 +7560,6 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, static void flow_dv_translate_item_ipv6(void *matcher, void *key, const struct rte_flow_item *item, - const uint64_t item_flags, int inner, uint32_t group) { const struct rte_flow_item_ipv6 *ipv6_m = item->mask; @@ -6668,13 +7598,6 @@ flow_dv_translate_item_ipv6(void *matcher, void *key, headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); } flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); - /* - * On outer header (which must contains L2), or inner header with L2, - * set cvlan_tag mask bit to mark this packet as untagged. - * This should be done even if item->spec is empty. - */ - if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) - MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); if (!ipv6_v) return; if (!ipv6_m) @@ -7247,6 +8170,164 @@ flow_dv_translate_item_geneve(void *matcher, void *key, MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); } +/** + * Create Geneve TLV option resource. + * + * @param dev[in, out] + * Pointer to rte_eth_dev structure. + * @param[in, out] tag_be24 + * Tag value in big endian then R-shift 8. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ + +int +flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + struct mlx5_geneve_tlv_option_resource *geneve_opt_resource = + sh->geneve_tlv_option_resource; + struct mlx5_devx_obj *obj; + const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec; + int ret = 0; + + if (!geneve_opt_v) + return -1; + rte_spinlock_lock(&sh->geneve_tlv_opt_sl); + if (geneve_opt_resource != NULL) { + if (geneve_opt_resource->option_class == + geneve_opt_v->option_class && + geneve_opt_resource->option_type == + geneve_opt_v->option_type && + geneve_opt_resource->length == + geneve_opt_v->option_len) { + /* We already have GENVE TLV option obj allocated. */ + __atomic_fetch_add(&geneve_opt_resource->refcnt, 1, + __ATOMIC_RELAXED); + } else { + ret = rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Only one GENEVE TLV option supported"); + goto exit; + } + } else { + /* Create a GENEVE TLV object and resource. */ + obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->ctx, + geneve_opt_v->option_class, + geneve_opt_v->option_type, + geneve_opt_v->option_len); + if (!obj) { + ret = rte_flow_error_set(error, ENODATA, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to create GENEVE TLV Devx object"); + goto exit; + } + sh->geneve_tlv_option_resource = + mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*geneve_opt_resource), + 0, SOCKET_ID_ANY); + if (!sh->geneve_tlv_option_resource) { + claim_zero(mlx5_devx_cmd_destroy(obj)); + ret = rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "GENEVE TLV object memory allocation failed"); + goto exit; + } + geneve_opt_resource = sh->geneve_tlv_option_resource; + geneve_opt_resource->obj = obj; + geneve_opt_resource->option_class = geneve_opt_v->option_class; + geneve_opt_resource->option_type = geneve_opt_v->option_type; + geneve_opt_resource->length = geneve_opt_v->option_len; + __atomic_store_n(&geneve_opt_resource->refcnt, 1, + __ATOMIC_RELAXED); + } +exit: + rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); + return ret; +} + +/** + * Add Geneve TLV option item to matcher. + * + * @param[in, out] dev + * Pointer to rte_eth_dev structure. + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[out] error + * Pointer to error structure. + */ +static int +flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher, + void *key, const struct rte_flow_item *item, + struct rte_flow_error *error) +{ + const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask; + const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec; + void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); + void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); + void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, + misc_parameters_3); + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + rte_be32_t opt_data_key = 0, opt_data_mask = 0; + int ret = 0; + + if (!geneve_opt_v) + return -1; + if (!geneve_opt_m) + geneve_opt_m = &rte_flow_item_geneve_opt_mask; + ret = flow_dev_geneve_tlv_option_resource_register(dev, item, + error); + if (ret) { + DRV_LOG(ERR, "Failed to create geneve_tlv_obj"); + return ret; + } + /* + * Set the option length in GENEVE header if not requested. + * The GENEVE TLV option length is expressed by the option length field + * in the GENEVE header. + * If the option length was not requested but the GENEVE TLV option item + * is present we set the option length field implicitly. + */ + if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) { + MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len, + MLX5_GENEVE_OPTLEN_MASK); + MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, + geneve_opt_v->option_len + 1); + } + /* Set the data. */ + if (geneve_opt_v->data) { + memcpy(&opt_data_key, geneve_opt_v->data, + RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4), + sizeof(opt_data_key))); + MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <= + sizeof(opt_data_key)); + memcpy(&opt_data_mask, geneve_opt_m->data, + RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4), + sizeof(opt_data_mask))); + MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <= + sizeof(opt_data_mask)); + MLX5_SET(fte_match_set_misc3, misc3_m, + geneve_tlv_option_0_data, + rte_be_to_cpu_32(opt_data_mask)); + MLX5_SET(fte_match_set_misc3, misc3_v, + geneve_tlv_option_0_data, + rte_be_to_cpu_32(opt_data_key & opt_data_mask)); + } + return ret; +} + /** * Add MPLS item to matcher and to the value. * @@ -7502,6 +8583,7 @@ flow_dv_translate_item_meta(struct rte_eth_dev *dev, reg = flow_dv_get_metadata_reg(dev, attr, NULL); if (reg < 0) return; + MLX5_ASSERT(reg != REG_NON); /* * In datapath code there is no endianness * coversions for perfromance reasons, all @@ -7644,13 +8726,16 @@ flow_dv_translate_item_source_vport(void *matcher, void *key, * Flow matcher value. * @param[in] item * Flow pattern to translate. + * @param[in] + * Flow attributes. * * @return * 0 on success, a negative errno value otherwise. */ static int flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, - void *key, const struct rte_flow_item *item) + void *key, const struct rte_flow_item *item, + const struct rte_flow_attr *attr) { const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; @@ -7662,14 +8747,34 @@ flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, priv = mlx5_port_to_eswitch_info(id, item == NULL); if (!priv) return -rte_errno; - /* Translate to vport field or to metadata, depending on mode. */ - if (priv->vport_meta_mask) + /* + * Translate to vport field or to metadata, depending on mode. + * Kernel can use either misc.source_port or half of C0 metadata + * register. + */ + if (priv->vport_meta_mask) { + /* + * Provide the hint for SW steering library + * to insert the flow into ingress domain and + * save the extra vport match. + */ + if (mask == 0xffff && priv->vport_id == 0xffff && + priv->pf_bond < 0 && attr->transfer) + flow_dv_translate_item_source_vport + (matcher, key, priv->vport_id, mask); + /* + * We should always set the vport metadata register, + * otherwise the SW steering library can drop + * the rule if wire vport metadata value is not zero, + * it depends on kernel configuration. + */ flow_dv_translate_item_meta_vport(matcher, key, priv->vport_meta_tag, priv->vport_meta_mask); - else + } else { flow_dv_translate_item_source_vport(matcher, key, priv->vport_id, mask); + } return 0; } @@ -7837,6 +8942,82 @@ flow_dv_translate_item_gtp(void *matcher, void *key, rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid)); } +/** + * Add GTP PSC item to matcher. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + */ +static int +flow_dv_translate_item_gtp_psc(void *matcher, void *key, + const struct rte_flow_item *item) +{ + const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask; + const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec; + void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, + misc_parameters_3); + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + union { + uint32_t w32; + struct { + uint16_t seq_num; + uint8_t npdu_num; + uint8_t next_ext_header_type; + }; + } dw_2; + uint8_t gtp_flags; + + /* Always set E-flag match on one, regardless of GTP item settings. */ + gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags); + gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG; + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags); + gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags); + gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG; + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags); + /*Set next extension header type. */ + dw_2.seq_num = 0; + dw_2.npdu_num = 0; + dw_2.next_ext_header_type = 0xff; + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2, + rte_cpu_to_be_32(dw_2.w32)); + dw_2.seq_num = 0; + dw_2.npdu_num = 0; + dw_2.next_ext_header_type = 0x85; + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2, + rte_cpu_to_be_32(dw_2.w32)); + if (gtp_psc_v) { + union { + uint32_t w32; + struct { + uint8_t len; + uint8_t type_flags; + uint8_t qfi; + uint8_t reserved; + }; + } dw_0; + + /*Set extension header PDU type and Qos. */ + if (!gtp_psc_m) + gtp_psc_m = &rte_flow_item_gtp_psc_mask; + dw_0.w32 = 0; + dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->pdu_type); + dw_0.qfi = gtp_psc_m->qfi; + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0, + rte_cpu_to_be_32(dw_0.w32)); + dw_0.w32 = 0; + dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->pdu_type & + gtp_psc_m->pdu_type); + dw_0.qfi = gtp_psc_v->qfi & gtp_psc_m->qfi; + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0, + rte_cpu_to_be_32(dw_0.w32)); + } + return 0; +} + /** * Add eCPRI item to matcher and to the value. * @@ -7858,6 +9039,7 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_item_ecpri *ecpri_m = item->mask; const struct rte_flow_item_ecpri *ecpri_v = item->spec; + struct rte_ecpri_common_hdr common; void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_4); void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4); @@ -7886,7 +9068,7 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, prog_sample_field_value_0); /* Already big endian (network order) in the header. */ *(uint32_t *)dw_m = ecpri_m->hdr.common.u32; - *(uint32_t *)dw_v = ecpri_v->hdr.common.u32; + *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32; /* Sample#0, used for matching type, offset 0. */ MLX5_SET(fte_match_set_misc4, misc4_m, prog_sample_field_id_0, samples[0]); @@ -7898,7 +9080,8 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, * Some wildcard rules only matching type field should be supported. */ if (ecpri_m->hdr.dummy[0]) { - switch (ecpri_v->hdr.common.type) { + common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32); + switch (common.type) { case RTE_ECPRI_MSG_TYPE_IQ_DATA: case RTE_ECPRI_MSG_TYPE_RTC_CTRL: case RTE_ECPRI_MSG_TYPE_DLY_MSR: @@ -7907,7 +9090,8 @@ flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v, prog_sample_field_value_1); *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0]; - *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0]; + *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] & + ecpri_m->hdr.dummy[0]; /* Sample#1, to match message body, offset 4. */ MLX5_SET(fte_match_set_misc4, misc4_m, prog_sample_field_id_1, samples[1]); @@ -7962,58 +9146,21 @@ flow_dv_matcher_enable(uint32_t *match_criteria) return match_criteria_enable; } - -/** - * Get a flow table. - * - * @param[in, out] dev - * Pointer to rte_eth_dev structure. - * @param[in] table_id - * Table id to use. - * @param[in] egress - * Direction of the table. - * @param[in] transfer - * E-Switch or NIC flow. - * @param[out] error - * pointer to error structure. - * - * @return - * Returns tables resource based on the index, NULL in case of failed. - */ -static struct mlx5_flow_tbl_resource * -flow_dv_tbl_resource_get(struct rte_eth_dev *dev, - uint32_t table_id, uint8_t egress, - uint8_t transfer, - bool external, - const struct mlx5_flow_tunnel *tunnel, - uint32_t group_id, - struct rte_flow_error *error) +struct mlx5_hlist_entry * +flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_tbl_resource *tbl; - union mlx5_flow_tbl_key table_key = { - { - .table_id = table_id, - .reserved = 0, - .domain = !!transfer, - .direction = !!egress, - } - }; - struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, - table_key.v64); + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data; + struct rte_flow_error *error = ctx->error; + union mlx5_flow_tbl_key key = { .v64 = key64 }; + struct mlx5_flow_tbl_resource *tbl; + void *domain; uint32_t idx = 0; int ret; - void *domain; - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - tbl = &tbl_data->tbl; - rte_atomic32_inc(&tbl->refcnt); - return tbl; - } tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); if (!tbl_data) { rte_flow_error_set(error, ENOMEM, @@ -8023,18 +9170,24 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, return NULL; } tbl_data->idx = idx; - tbl_data->tunnel = tunnel; - tbl_data->group_id = group_id; - tbl_data->external = external; + tbl_data->tunnel = tt_prm->tunnel; + tbl_data->group_id = tt_prm->group_id; + tbl_data->external = !!tt_prm->external; + tbl_data->tunnel_offload = is_tunnel_offload_active(dev); + tbl_data->is_egress = !!key.direction; + tbl_data->is_transfer = !!key.domain; + tbl_data->dummy = !!key.dummy; + tbl_data->table_id = key.table_id; tbl = &tbl_data->tbl; - pos = &tbl_data->entry; - if (transfer) + if (key.dummy) + return &tbl_data->entry; + if (key.domain) domain = sh->fdb_domain; - else if (egress) + else if (key.direction) domain = sh->tx_domain; else domain = sh->rx_domain; - ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj); + ret = mlx5_flow_os_create_flow_tbl(domain, key.table_id, &tbl->obj); if (ret) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -8042,31 +9195,153 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); return NULL; } - /* - * No multi-threads now, but still better to initialize the reference - * count before insert it into the hash list. - */ - rte_atomic32_init(&tbl->refcnt); - /* Jump action reference count is initialized here. */ - rte_atomic32_init(&tbl_data->jump.refcnt); - pos->key = table_key.v64; - ret = mlx5_hlist_insert(sh->flow_tbls, pos); - if (ret < 0) { - rte_flow_error_set(error, -ret, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot insert flow table data entry"); - mlx5_flow_os_destroy_flow_tbl(tbl->obj); - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); + if (key.table_id) { + ret = mlx5_flow_os_create_flow_action_dest_flow_tbl + (tbl->obj, &tbl_data->jump.action); + if (ret) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot create flow jump action"); + mlx5_flow_os_destroy_flow_tbl(tbl->obj); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); + return NULL; + } } - rte_atomic32_inc(&tbl->refcnt); - return tbl; + MKSTR(matcher_name, "%s_%s_%u_matcher_cache", + key.domain ? "FDB" : "NIC", key.direction ? "egress" : "ingress", + key.table_id); + mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh, + flow_dv_matcher_create_cb, + flow_dv_matcher_match_cb, + flow_dv_matcher_remove_cb); + return &tbl_data->entry; +} + +int +flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused, + struct mlx5_hlist_entry *entry, uint64_t key64, + void *cb_ctx __rte_unused) +{ + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + union mlx5_flow_tbl_key key = { .v64 = key64 }; + + return tbl_data->table_id != key.table_id || + tbl_data->dummy != key.dummy || + tbl_data->is_transfer != key.domain || + tbl_data->is_egress != key.direction; } /** - * Release a flow table. + * Get a flow table. * - * @param[in] dev + * @param[in, out] dev * Pointer to rte_eth_dev structure. + * @param[in] table_id + * Table id to use. + * @param[in] egress + * Direction of the table. + * @param[in] transfer + * E-Switch or NIC flow. + * @param[in] dummy + * Dummy entry for dv API. + * @param[out] error + * pointer to error structure. + * + * @return + * Returns tables resource based on the index, NULL in case of failed. + */ +struct mlx5_flow_tbl_resource * +flow_dv_tbl_resource_get(struct rte_eth_dev *dev, + uint32_t table_id, uint8_t egress, + uint8_t transfer, + bool external, + const struct mlx5_flow_tunnel *tunnel, + uint32_t group_id, uint8_t dummy, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + union mlx5_flow_tbl_key table_key = { + { + .table_id = table_id, + .dummy = dummy, + .domain = !!transfer, + .direction = !!egress, + } + }; + struct mlx5_flow_tbl_tunnel_prm tt_prm = { + .tunnel = tunnel, + .group_id = group_id, + .external = external, + }; + struct mlx5_flow_cb_ctx ctx = { + .dev = dev, + .error = error, + .data = &tt_prm, + }; + struct mlx5_hlist_entry *entry; + struct mlx5_flow_tbl_data_entry *tbl_data; + + entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx); + if (!entry) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot get table"); + return NULL; + } + DRV_LOG(DEBUG, "Table_id %u tunnel %u group %u registered.", + table_id, tunnel ? tunnel->tunnel_id : 0, group_id); + tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + return &tbl_data->tbl; +} + +void +flow_dv_tbl_remove_cb(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + + MLX5_ASSERT(entry && sh); + if (tbl_data->jump.action) + mlx5_flow_os_destroy_flow_action(tbl_data->jump.action); + if (tbl_data->tbl.obj) + mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj); + if (tbl_data->tunnel_offload && tbl_data->external) { + struct mlx5_hlist_entry *he; + struct mlx5_hlist *tunnel_grp_hash; + struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; + union tunnel_tbl_key tunnel_key = { + .tunnel_id = tbl_data->tunnel ? + tbl_data->tunnel->tunnel_id : 0, + .group = tbl_data->group_id + }; + uint32_t table_id = tbl_data->table_id; + + tunnel_grp_hash = tbl_data->tunnel ? + tbl_data->tunnel->groups : + thub->groups; + he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL); + if (he) + mlx5_hlist_unregister(tunnel_grp_hash, he); + DRV_LOG(DEBUG, + "Table_id %u tunnel %u group %u released.", + table_id, + tbl_data->tunnel ? + tbl_data->tunnel->tunnel_id : 0, + tbl_data->group_id); + } + mlx5_cache_list_destroy(&tbl_data->matchers); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx); +} + +/** + * Release a flow table. + * + * @param[in] sh + * Pointer to device shared structure. * @param[in] tbl * Table resource to be released. * @@ -8074,63 +9349,72 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, * Returns 0 if table was released, else return 1; */ static int -flow_dv_tbl_resource_release(struct rte_eth_dev *dev, +flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh, struct mlx5_flow_tbl_resource *tbl) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_flow_tbl_data_entry *tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); if (!tbl) return 0; - if (rte_atomic32_dec_and_test(&tbl->refcnt)) { - struct mlx5_hlist_entry *pos = &tbl_data->entry; - - mlx5_flow_os_destroy_flow_tbl(tbl->obj); - tbl->obj = NULL; - if (is_tunnel_offload_active(dev) && tbl_data->external) { - struct mlx5_hlist_entry *he; - struct mlx5_hlist *tunnel_grp_hash; - struct mlx5_flow_tunnel_hub *thub = - mlx5_tunnel_hub(dev); - union tunnel_tbl_key tunnel_key = { - .tunnel_id = tbl_data->tunnel ? - tbl_data->tunnel->tunnel_id : 0, - .group = tbl_data->group_id - }; - union mlx5_flow_tbl_key table_key = { - .v64 = pos->key - }; - uint32_t table_id = table_key.table_id; - - tunnel_grp_hash = tbl_data->tunnel ? - tbl_data->tunnel->groups : - thub->groups; - he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val); - if (he) { - struct tunnel_tbl_entry *tte; - tte = container_of(he, typeof(*tte), hash); - MLX5_ASSERT(tte->flow_table == table_id); - mlx5_hlist_remove(tunnel_grp_hash, he); - mlx5_free(tte); - } - mlx5_flow_id_release(mlx5_tunnel_hub(dev)->table_ids, - tunnel_flow_tbl_to_id(table_id)); - DRV_LOG(DEBUG, - "port %u release table_id %#x tunnel %u group %u", - dev->data->port_id, table_id, - tbl_data->tunnel ? - tbl_data->tunnel->tunnel_id : 0, - tbl_data->group_id); - } - /* remove the entry from the hash list and free memory. */ - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP], - tbl_data->idx); - return 0; + return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry); +} + +int +flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_matcher *ref = ctx->data; + struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur), + entry); + + return cur->crc != ref->crc || + cur->priority != ref->priority || + memcmp((const void *)cur->mask.buf, + (const void *)ref->mask.buf, ref->mask.size); +} + +struct mlx5_cache_entry * +flow_dv_matcher_create_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_matcher *ref = ctx->data; + struct mlx5_flow_dv_matcher *cache; + struct mlx5dv_flow_matcher_attr dv_attr = { + .type = IBV_FLOW_ATTR_NORMAL, + .match_mask = (void *)&ref->mask, + }; + struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl, + typeof(*tbl), tbl); + int ret; + + cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY); + if (!cache) { + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot create matcher"); + return NULL; } - return 1; + *cache = *ref; + dv_attr.match_criteria_enable = + flow_dv_matcher_enable(cache->mask.buf); + dv_attr.priority = ref->priority; + if (tbl->is_egress) + dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; + ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj, + &cache->matcher_object); + if (ret) { + mlx5_free(cache); + rte_flow_error_set(ctx->error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot create matcher"); + return NULL; + } + return &cache->entry; } /** @@ -8152,90 +9436,87 @@ flow_dv_tbl_resource_release(struct rte_eth_dev *dev, */ static int flow_dv_matcher_register(struct rte_eth_dev *dev, - struct mlx5_flow_dv_matcher *matcher, + struct mlx5_flow_dv_matcher *ref, union mlx5_flow_tbl_key *key, struct mlx5_flow *dev_flow, + const struct mlx5_flow_tunnel *tunnel, + uint32_t group_id, struct rte_flow_error *error) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_matcher *cache_matcher; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .match_mask = (void *)&matcher->mask, - }; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher *cache; struct mlx5_flow_tbl_resource *tbl; struct mlx5_flow_tbl_data_entry *tbl_data; - int ret; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = ref, + }; - tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, - key->domain, false, NULL, 0, error); + /** + * tunnel offload API requires this registration for cases when + * tunnel match rule was inserted before tunnel set rule. + */ + tbl = flow_dv_tbl_resource_get(dev, key->table_id, + key->direction, key->domain, + dev_flow->external, tunnel, + group_id, 0, error); if (!tbl) return -rte_errno; /* No need to refill the error info */ tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); - /* Lookup from cache. */ - LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) { - if (matcher->crc == cache_matcher->crc && - matcher->priority == cache_matcher->priority && - !memcmp((const void *)matcher->mask.buf, - (const void *)cache_matcher->mask.buf, - cache_matcher->mask.size)) { - DRV_LOG(DEBUG, - "%s group %u priority %hd use %s " - "matcher %p: refcnt %d++", - key->domain ? "FDB" : "NIC", key->table_id, - cache_matcher->priority, - key->direction ? "tx" : "rx", - (void *)cache_matcher, - rte_atomic32_read(&cache_matcher->refcnt)); - rte_atomic32_inc(&cache_matcher->refcnt); - dev_flow->handle->dvh.matcher = cache_matcher; - /* old matcher should not make the table ref++. */ - flow_dv_tbl_resource_release(dev, tbl); - return 0; - } - } - /* Register new matcher. */ - cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0, - SOCKET_ID_ANY); - if (!cache_matcher) { - flow_dv_tbl_resource_release(dev, tbl); + ref->tbl = tbl; + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate matcher memory"); + "cannot allocate ref memory"); } - *cache_matcher = *matcher; - dv_attr.match_criteria_enable = - flow_dv_matcher_enable(cache_matcher->mask.buf); - dv_attr.priority = matcher->priority; - if (key->direction) - dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj, - &cache_matcher->matcher_object); - if (ret) { - mlx5_free(cache_matcher); -#ifdef HAVE_MLX5DV_DR - flow_dv_tbl_resource_release(dev, tbl); -#endif - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create matcher"); - } - /* Save the table information */ - cache_matcher->tbl = tbl; - rte_atomic32_init(&cache_matcher->refcnt); - /* only matcher ref++, table ref++ already done above in get API. */ - rte_atomic32_inc(&cache_matcher->refcnt); - LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); - dev_flow->handle->dvh.matcher = cache_matcher; - DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", - key->domain ? "FDB" : "NIC", key->table_id, - cache_matcher->priority, - key->direction ? "tx" : "rx", (void *)cache_matcher, - rte_atomic32_read(&cache_matcher->refcnt)); + cache = container_of(entry, typeof(*cache), entry); + dev_flow->handle->dvh.matcher = cache; return 0; } +struct mlx5_hlist_entry * +flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct rte_flow_error *error = ctx; + struct mlx5_flow_dv_tag_resource *entry; + uint32_t idx = 0; + int ret; + + entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx); + if (!entry) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + return NULL; + } + entry->idx = idx; + entry->tag_id = key; + ret = mlx5_flow_os_create_flow_action_tag(key, + &entry->action); + if (ret) { + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx); + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create action"); + return NULL; + } + return &entry->entry; +} + +int +flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused, + struct mlx5_hlist_entry *entry, uint64_t key, + void *cb_ctx __rte_unused) +{ + struct mlx5_flow_dv_tag_resource *tag = + container_of(entry, struct mlx5_flow_dv_tag_resource, entry); + + return key != tag->tag_id; +} + /** * Find existing tag resource or create and register a new one. * @@ -8259,54 +9540,32 @@ flow_dv_tag_resource_register struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_flow_dv_tag_resource *cache_resource; struct mlx5_hlist_entry *entry; - int ret; - /* Lookup a matching resource from cache. */ - entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24); + entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error); if (entry) { cache_resource = container_of (entry, struct mlx5_flow_dv_tag_resource, entry); - rte_atomic32_inc(&cache_resource->refcnt); dev_flow->handle->dvh.rix_tag = cache_resource->idx; dev_flow->dv.tag_resource = cache_resource; - DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); return 0; } - /* Register new resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], - &dev_flow->handle->dvh.rix_tag); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate resource memory"); - cache_resource->entry.key = (uint64_t)tag_be24; - ret = mlx5_flow_os_create_flow_action_tag(tag_be24, - &cache_resource->action); - if (ret) { - mlx5_free(cache_resource); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create action"); - } - rte_atomic32_init(&cache_resource->refcnt); - rte_atomic32_inc(&cache_resource->refcnt); - if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) { - mlx5_flow_os_destroy_flow_action(cache_resource->action); - mlx5_free(cache_resource); - return rte_flow_error_set(error, EEXIST, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot insert tag"); - } - dev_flow->dv.tag_resource = cache_resource; - DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - return 0; + return -rte_errno; +} + +void +flow_dv_tag_remove_cb(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_dv_tag_resource *tag = + container_of(entry, struct mlx5_flow_dv_tag_resource, entry); + + MLX5_ASSERT(tag && sh && tag->action); + claim_zero(mlx5_flow_os_destroy_flow_action(tag->action)); + DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx); } /** @@ -8325,24 +9584,14 @@ flow_dv_tag_release(struct rte_eth_dev *dev, uint32_t tag_idx) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_flow_dv_tag_resource *tag; tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); if (!tag) return 0; DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", - dev->data->port_id, (void *)tag, - rte_atomic32_read(&tag->refcnt)); - if (rte_atomic32_dec_and_test(&tag->refcnt)) { - claim_zero(mlx5_flow_os_destroy_flow_action(tag->action)); - mlx5_hlist_remove(sh->tag_table, &tag->entry); - DRV_LOG(DEBUG, "port %u tag %p: removed", - dev->data->port_id, (void *)tag); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); - return 0; - } - return 1; + dev->data->port_id, (void *)tag, tag->entry.ref_cnt); + return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry); } /** @@ -8432,6 +9681,7 @@ flow_dv_translate_create_counter(struct rte_eth_dev *dev, __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED); return counter; } + /** * Add Tx queue matcher * @@ -8549,7 +9799,7 @@ flow_dv_hashfields_set(struct mlx5_flow *dev_flow, } /** - * Create an Rx Hash queue. + * Prepare an Rx Hash queue. * * @param dev * Pointer to Ethernet device. @@ -8564,61 +9814,101 @@ flow_dv_hashfields_set(struct mlx5_flow *dev_flow, * The Verbs/DevX object initialised, NULL otherwise and rte_errno is set. */ static struct mlx5_hrxq * -flow_dv_handle_rx_queue(struct rte_eth_dev *dev, - struct mlx5_flow *dev_flow, - struct mlx5_flow_rss_desc *rss_desc, - uint32_t *hrxq_idx) +flow_dv_hrxq_prepare(struct rte_eth_dev *dev, + struct mlx5_flow *dev_flow, + struct mlx5_flow_rss_desc *rss_desc, + uint32_t *hrxq_idx) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_handle *dh = dev_flow->handle; struct mlx5_hrxq *hrxq; MLX5_ASSERT(rss_desc->queue_num); - *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, rss_desc->queue_num); - if (!*hrxq_idx) { - *hrxq_idx = mlx5_hrxq_new - (dev, rss_desc->key, MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, rss_desc->queue_num, - !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL), - false); - if (!*hrxq_idx) - return NULL; - } + rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; + rss_desc->hash_fields = dev_flow->hash_fields; + rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL); + rss_desc->shared_rss = 0; + *hrxq_idx = mlx5_hrxq_get(dev, rss_desc); + if (!*hrxq_idx) + return NULL; hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], *hrxq_idx); return hrxq; } /** - * Find existing sample resource or create and register a new one. + * Release sample sub action resource. * * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in] attr - * Attributes of flow that includes this item. - * @param[in] resource - * Pointer to sample resource. - * @parm[in, out] dev_flow - * Pointer to the dev_flow. - * @param[in, out] sample_dv_actions - * Pointer to sample actions list. - * @param[out] error - * pointer to error structure. - * - * @return - * 0 on success otherwise -errno and errno is set. + * @param[in] act_res + * Pointer to sample sub action resource. */ -static int -flow_dv_sample_resource_register(struct rte_eth_dev *dev, - const struct rte_flow_attr *attr, - struct mlx5_flow_dv_sample_resource *resource, - struct mlx5_flow *dev_flow, - void **sample_dv_actions, - struct rte_flow_error *error) +static void +flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev, + struct mlx5_flow_sub_actions_idx *act_res) +{ + if (act_res->rix_hrxq) { + mlx5_hrxq_release(dev, act_res->rix_hrxq); + act_res->rix_hrxq = 0; + } + if (act_res->rix_encap_decap) { + flow_dv_encap_decap_resource_release(dev, + act_res->rix_encap_decap); + act_res->rix_encap_decap = 0; + } + if (act_res->rix_port_id_action) { + flow_dv_port_id_action_resource_release(dev, + act_res->rix_port_id_action); + act_res->rix_port_id_action = 0; + } + if (act_res->rix_tag) { + flow_dv_tag_release(dev, act_res->rix_tag); + act_res->rix_tag = 0; + } + if (act_res->rix_jump) { + flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump); + act_res->rix_jump = 0; + } +} + +int +flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; + struct mlx5_flow_dv_sample_resource *resource = ctx->data; + struct mlx5_flow_dv_sample_resource *cache_resource = + container_of(entry, typeof(*cache_resource), entry); + + if (resource->ratio == cache_resource->ratio && + resource->ft_type == cache_resource->ft_type && + resource->ft_id == cache_resource->ft_id && + resource->set_action == cache_resource->set_action && + !memcmp((void *)&resource->sample_act, + (void *)&cache_resource->sample_act, + sizeof(struct mlx5_flow_sub_actions_list))) { + /* + * Existing sample action should release the prepared + * sub-actions reference counter. + */ + flow_dv_sample_sub_actions_release(dev, + &resource->sample_idx); + return 0; + } + return 1; +} + +struct mlx5_cache_entry * +flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) { + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; + struct mlx5_flow_dv_sample_resource *resource = ctx->data; + void **sample_dv_actions = resource->sub_actions; struct mlx5_flow_dv_sample_resource *cache_resource; struct mlx5dv_dr_flow_sampler_attr sampler_attr; struct mlx5_priv *priv = dev->data->dev_private; @@ -8627,41 +9917,28 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev, uint32_t idx = 0; const uint32_t next_ft_step = 1; uint32_t next_ft_id = resource->ft_id + next_ft_step; + uint8_t is_egress = 0; + uint8_t is_transfer = 0; + struct rte_flow_error *error = ctx->error; - /* Lookup a matching resource from cache. */ - ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list, - idx, cache_resource, next) { - if (resource->ratio == cache_resource->ratio && - resource->ft_type == cache_resource->ft_type && - resource->ft_id == cache_resource->ft_id && - resource->set_action == cache_resource->set_action && - !memcmp((void *)&resource->sample_act, - (void *)&cache_resource->sample_act, - sizeof(struct mlx5_flow_sub_actions_list))) { - DRV_LOG(DEBUG, "sample resource %p: refcnt %d++", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, - __ATOMIC_RELAXED)); - __atomic_fetch_add(&cache_resource->refcnt, 1, - __ATOMIC_RELAXED); - dev_flow->handle->dvh.rix_sample = idx; - dev_flow->dv.sample_res = cache_resource; - return 0; - } - } /* Register new sample resource. */ - cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], - &dev_flow->handle->dvh.rix_sample); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, + cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx); + if (!cache_resource) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); + return NULL; + } *cache_resource = *resource; /* Create normal path table level */ + if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + is_transfer = 1; + else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) + is_egress = 1; tbl = flow_dv_tbl_resource_get(dev, next_ft_id, - attr->egress, attr->transfer, - dev_flow->external, NULL, 0, error); + is_egress, is_transfer, + true, NULL, 0, 0, error); if (!tbl) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -8672,18 +9949,16 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev, } cache_resource->normal_path_tbl = tbl; if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) { - cache_resource->default_miss = - mlx5_glue->dr_create_flow_action_default_miss(); - if (!cache_resource->default_miss) { + if (!sh->default_miss_action) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot create default miss " - "action"); + "default miss action was not " + "created"); goto error; } sample_dv_actions[resource->sample_act.actions_num++] = - cache_resource->default_miss; + sh->default_miss_action; } /* Create a DR sample action */ sampler_attr.sample_ratio = cache_resource->ratio; @@ -8692,61 +9967,35 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev, sampler_attr.sample_actions = (struct mlx5dv_dr_action **) &sample_dv_actions[0]; sampler_attr.action = cache_resource->set_action; - cache_resource->verbs_action = - mlx5_glue->dr_create_flow_action_sampler(&sampler_attr); - if (!cache_resource->verbs_action) { + if (mlx5_os_flow_dr_create_flow_action_sampler + (&sampler_attr, &cache_resource->verbs_action)) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create sample action"); goto error; } - __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED); - ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list, - dev_flow->handle->dvh.rix_sample, cache_resource, - next); - dev_flow->dv.sample_res = cache_resource; - DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)); - return 0; + cache_resource->idx = idx; + cache_resource->dev = dev; + return &cache_resource->entry; error: - if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) { - if (cache_resource->default_miss) - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->default_miss)); - } else { - if (cache_resource->sample_idx.rix_hrxq && - !mlx5_hrxq_release(dev, - cache_resource->sample_idx.rix_hrxq)) - cache_resource->sample_idx.rix_hrxq = 0; - if (cache_resource->sample_idx.rix_tag && - !flow_dv_tag_release(dev, - cache_resource->sample_idx.rix_tag)) - cache_resource->sample_idx.rix_tag = 0; - if (cache_resource->sample_idx.cnt) { - flow_dv_counter_release(dev, - cache_resource->sample_idx.cnt); - cache_resource->sample_idx.cnt = 0; - } - } + if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB) + flow_dv_sample_sub_actions_release(dev, + &cache_resource->sample_idx); if (cache_resource->normal_path_tbl) - flow_dv_tbl_resource_release(dev, + flow_dv_tbl_resource_release(MLX5_SH(dev), cache_resource->normal_path_tbl); - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], - dev_flow->handle->dvh.rix_sample); - dev_flow->handle->dvh.rix_sample = 0; - return -rte_errno; + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx); + return NULL; + } /** - * Find existing destination array resource or create and register a new one. + * Find existing sample resource or create and register a new one. * * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in] attr - * Attributes of flow that includes this item. * @param[in] resource - * Pointer to destination array resource. + * Pointer to sample resource. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -8756,54 +10005,92 @@ error: * 0 on success otherwise -errno and errno is set. */ static int -flow_dv_dest_array_resource_register(struct rte_eth_dev *dev, - const struct rte_flow_attr *attr, - struct mlx5_flow_dv_dest_array_resource *resource, +flow_dv_sample_resource_register(struct rte_eth_dev *dev, + struct mlx5_flow_dv_sample_resource *resource, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { + struct mlx5_flow_dv_sample_resource *cache_resource; + struct mlx5_cache_entry *entry; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_cb_ctx ctx = { + .dev = dev, + .error = error, + .data = resource, + }; + + entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx); + if (!entry) + return -rte_errno; + cache_resource = container_of(entry, typeof(*cache_resource), entry); + dev_flow->handle->dvh.rix_sample = cache_resource->idx; + dev_flow->dv.sample_res = cache_resource; + return 0; +} + +int +flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry, void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_dv_dest_array_resource *resource = ctx->data; + struct rte_eth_dev *dev = ctx->dev; + struct mlx5_flow_dv_dest_array_resource *cache_resource = + container_of(entry, typeof(*cache_resource), entry); + uint32_t idx = 0; + + if (resource->num_of_dest == cache_resource->num_of_dest && + resource->ft_type == cache_resource->ft_type && + !memcmp((void *)cache_resource->sample_act, + (void *)resource->sample_act, + (resource->num_of_dest * + sizeof(struct mlx5_flow_sub_actions_list)))) { + /* + * Existing sample action should release the prepared + * sub-actions reference counter. + */ + for (idx = 0; idx < resource->num_of_dest; idx++) + flow_dv_sample_sub_actions_release(dev, + &resource->sample_idx[idx]); + return 0; + } + return 1; +} + +struct mlx5_cache_entry * +flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) +{ + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; struct mlx5_flow_dv_dest_array_resource *cache_resource; + struct mlx5_flow_dv_dest_array_resource *resource = ctx->data; struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 }; struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM]; struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_flow_sub_actions_list *sample_act; struct mlx5dv_dr_domain *domain; - uint32_t idx = 0; + uint32_t idx = 0, res_idx = 0; + struct rte_flow_error *error = ctx->error; + uint64_t action_flags; + int ret; - /* Lookup a matching resource from cache. */ - ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY], - sh->dest_array_list, - idx, cache_resource, next) { - if (resource->num_of_dest == cache_resource->num_of_dest && - resource->ft_type == cache_resource->ft_type && - !memcmp((void *)cache_resource->sample_act, - (void *)resource->sample_act, - (resource->num_of_dest * - sizeof(struct mlx5_flow_sub_actions_list)))) { - DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, - __ATOMIC_RELAXED)); - __atomic_fetch_add(&cache_resource->refcnt, 1, - __ATOMIC_RELAXED); - dev_flow->handle->dvh.rix_dest_array = idx; - dev_flow->dv.dest_array_res = cache_resource; - return 0; - } - } /* Register new destination array resource. */ cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY], - &dev_flow->handle->dvh.rix_dest_array); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, + &res_idx); + if (!cache_resource) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); + return NULL; + } *cache_resource = *resource; - if (attr->transfer) + if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) domain = sh->fdb_domain; - else if (attr->ingress) + else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) domain = sh->rx_domain; else domain = sh->tx_domain; @@ -8821,45 +10108,51 @@ flow_dv_dest_array_resource_register(struct rte_eth_dev *dev, } dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST; sample_act = &resource->sample_act[idx]; - if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) { + action_flags = sample_act->action_flags; + switch (action_flags) { + case MLX5_FLOW_ACTION_QUEUE: dest_attr[idx]->dest = sample_act->dr_queue_action; - } else if (sample_act->action_flags == - (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) { + break; + case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP): dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT; dest_attr[idx]->dest_reformat = &dest_reformat[idx]; dest_attr[idx]->dest_reformat->reformat = sample_act->dr_encap_action; dest_attr[idx]->dest_reformat->dest = sample_act->dr_port_id_action; - } else if (sample_act->action_flags == - MLX5_FLOW_ACTION_PORT_ID) { + break; + case MLX5_FLOW_ACTION_PORT_ID: dest_attr[idx]->dest = sample_act->dr_port_id_action; + break; + case MLX5_FLOW_ACTION_JUMP: + dest_attr[idx]->dest = sample_act->dr_jump_action; + break; + default: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported actions type"); + goto error; } } /* create a dest array actioin */ - cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array + ret = mlx5_os_flow_dr_create_flow_action_dest_array (domain, cache_resource->num_of_dest, - dest_attr); - if (!cache_resource->action) { + dest_attr, + &cache_resource->action); + if (ret) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create destination array action"); goto error; } - __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED); - ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY], - &sh->dest_array_list, - dev_flow->handle->dvh.rix_dest_array, cache_resource, - next); - dev_flow->dv.dest_array_res = cache_resource; - DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)); + cache_resource->idx = res_idx; + cache_resource->dev = dev; for (idx = 0; idx < resource->num_of_dest; idx++) mlx5_free(dest_attr[idx]); - return 0; + return &cache_resource->entry; error: for (idx = 0; idx < resource->num_of_dest; idx++) { struct mlx5_flow_sub_actions_idx *act_res = @@ -8876,14 +10169,55 @@ error: !flow_dv_port_id_action_resource_release(dev, act_res->rix_port_id_action)) act_res->rix_port_id_action = 0; + if (act_res->rix_jump && + !flow_dv_jump_tbl_resource_release(dev, + act_res->rix_jump)) + act_res->rix_jump = 0; if (dest_attr[idx]) mlx5_free(dest_attr[idx]); } - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], - dev_flow->handle->dvh.rix_dest_array); - dev_flow->handle->dvh.rix_dest_array = 0; - return -rte_errno; + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx); + return NULL; +} + +/** + * Find existing destination array resource or create and register a new one. + * + * @param[in, out] dev + * Pointer to rte_eth_dev structure. + * @param[in] resource + * Pointer to destination array resource. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ +static int +flow_dv_dest_array_resource_register(struct rte_eth_dev *dev, + struct mlx5_flow_dv_dest_array_resource *resource, + struct mlx5_flow *dev_flow, + struct rte_flow_error *error) +{ + struct mlx5_flow_dv_dest_array_resource *cache_resource; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_cache_entry *entry; + struct mlx5_flow_cb_ctx ctx = { + .dev = dev, + .error = error, + .data = resource, + }; + + entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx); + if (!entry) + return -rte_errno; + cache_resource = container_of(entry, typeof(*cache_resource), entry); + dev_flow->handle->dvh.rix_dest_array = cache_resource->idx; + dev_flow->dv.dest_array_res = cache_resource; + return 0; } /** @@ -8892,7 +10226,7 @@ error: * @param[in] dev * Pointer to rte_eth_dev structure. * @param[in] action - * Pointer to action structure. + * Pointer to sample action structure. * @param[in, out] dev_flow * Pointer to the mlx5_flow. * @param[in] attr @@ -8911,7 +10245,7 @@ error: */ static int flow_dv_translate_action_sample(struct rte_eth_dev *dev, - const struct rte_flow_action *action, + const struct rte_flow_action_sample *action, struct mlx5_flow *dev_flow, const struct rte_flow_attr *attr, uint32_t *num_of_dest, @@ -8920,21 +10254,20 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - const struct rte_flow_action_sample *sample_action; const struct rte_flow_action *sub_actions; - const struct rte_flow_action_queue *queue; struct mlx5_flow_sub_actions_list *sample_act; struct mlx5_flow_sub_actions_idx *sample_idx; - struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) - priv->rss_desc) - [!!priv->flow_nested_idx]; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + struct rte_flow *flow = dev_flow->flow; + struct mlx5_flow_rss_desc *rss_desc; uint64_t action_flags = 0; + MLX5_ASSERT(wks); + rss_desc = &wks->rss_desc; sample_act = &res->sample_act; sample_idx = &res->sample_idx; - sample_action = (const struct rte_flow_action_sample *)action->conf; - res->ratio = sample_action->ratio; - sub_actions = sample_action->actions; + res->ratio = action->ratio; + sub_actions = action->actions; for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) { int type = sub_actions->type; uint32_t pre_rix = 0; @@ -8942,14 +10275,15 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, switch (type) { case RTE_FLOW_ACTION_TYPE_QUEUE: { + const struct rte_flow_action_queue *queue; struct mlx5_hrxq *hrxq; uint32_t hrxq_idx; queue = sub_actions->conf; rss_desc->queue_num = 1; rss_desc->queue[0] = queue->index; - hrxq = flow_dv_handle_rx_queue(dev, dev_flow, - rss_desc, &hrxq_idx); + hrxq = flow_dv_hrxq_prepare(dev, dev_flow, + rss_desc, &hrxq_idx); if (!hrxq) return rte_flow_error_set (error, rte_errno, @@ -8968,6 +10302,45 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, MLX5_FLOW_FATE_QUEUE; break; } + case RTE_FLOW_ACTION_TYPE_RSS: + { + struct mlx5_hrxq *hrxq; + uint32_t hrxq_idx; + const struct rte_flow_action_rss *rss; + const uint8_t *rss_key; + + rss = sub_actions->conf; + memcpy(rss_desc->queue, rss->queue, + rss->queue_num * sizeof(uint16_t)); + rss_desc->queue_num = rss->queue_num; + /* NULL RSS key indicates default RSS key. */ + rss_key = !rss->key ? rss_hash_default_key : rss->key; + memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); + /* + * rss->level and rss.types should be set in advance + * when expanding items for RSS. + */ + flow_dv_hashfields_set(dev_flow, rss_desc); + hrxq = flow_dv_hrxq_prepare(dev, dev_flow, + rss_desc, &hrxq_idx); + if (!hrxq) + return rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot create fate queue"); + sample_act->dr_queue_action = hrxq->action; + sample_idx->rix_hrxq = hrxq_idx; + sample_actions[sample_act->actions_num++] = + hrxq->action; + (*num_of_dest)++; + action_flags |= MLX5_FLOW_ACTION_RSS; + if (action_flags & MLX5_FLOW_ACTION_MARK) + dev_flow->handle->rix_hrxq = hrxq_idx; + dev_flow->handle->fate_action = + MLX5_FLOW_FATE_QUEUE; + break; + } case RTE_FLOW_ACTION_TYPE_MARK: { uint32_t tag_be = mlx5_flow_mark_set @@ -8996,21 +10369,22 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, } case RTE_FLOW_ACTION_TYPE_COUNT: { - uint32_t counter; - - counter = flow_dv_translate_create_counter(dev, - dev_flow, sub_actions->conf, 0); - if (!counter) - return rte_flow_error_set + if (!flow->counter) { + flow->counter = + flow_dv_translate_create_counter(dev, + dev_flow, sub_actions->conf, + 0); + if (!flow->counter) + return rte_flow_error_set (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot create counter" - " object."); - sample_idx->cnt = counter; + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot create counter" + " object."); + } sample_act->dr_cnt_action = (flow_dv_counter_get_by_idx(dev, - counter, NULL))->action; + flow->counter, NULL))->action; sample_actions[sample_act->actions_num++] = sample_act->dr_cnt_action; action_flags |= MLX5_FLOW_ACTION_COUNT; @@ -9091,6 +10465,8 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, res->set_action = action_ctx.set_action; } else if (attr->ingress) { res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX; + } else { + res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX; } return 0; } @@ -9102,8 +10478,6 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, * Pointer to rte_eth_dev structure. * @param[in, out] dev_flow * Pointer to the mlx5_flow. - * @param[in] attr - * Pointer to the flow attributes. * @param[in] num_of_dest * The num of destination. * @param[in, out] res @@ -9123,7 +10497,6 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev, static int flow_dv_create_action_sample(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, - const struct rte_flow_attr *attr, uint32_t num_of_dest, struct mlx5_flow_dv_sample_resource *res, struct mlx5_flow_dv_dest_array_resource *mdest_res, @@ -9131,23 +10504,23 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, uint64_t action_flags, struct rte_flow_error *error) { - struct mlx5_priv *priv = dev->data->dev_private; /* update normal path action resource into last index of array */ uint32_t dest_index = MLX5_MAX_DEST_NUM - 1; struct mlx5_flow_sub_actions_list *sample_act = &mdest_res->sample_act[dest_index]; - struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) - priv->rss_desc) - [!!priv->flow_nested_idx]; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + struct mlx5_flow_rss_desc *rss_desc; uint32_t normal_idx = 0; struct mlx5_hrxq *hrxq; uint32_t hrxq_idx; + MLX5_ASSERT(wks); + rss_desc = &wks->rss_desc; if (num_of_dest > 1) { if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) { /* Handle QP action for mirroring */ - hrxq = flow_dv_handle_rx_queue(dev, dev_flow, - rss_desc, &hrxq_idx); + hrxq = flow_dv_hrxq_prepare(dev, dev_flow, + rss_desc, &hrxq_idx); if (!hrxq) return rte_flow_error_set (error, rte_errno, @@ -9175,6 +10548,14 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, sample_act->dr_port_id_action = dev_flow->dv.port_id_action->action; } + if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) { + normal_idx++; + mdest_res->sample_idx[dest_index].rix_jump = + dev_flow->handle->rix_jump; + sample_act->dr_jump_action = + dev_flow->dv.jump->action; + dev_flow->handle->rix_jump = 0; + } sample_act->actions_num = normal_idx; /* update sample action resource into first index of array */ mdest_res->ft_type = res->ft_type; @@ -9183,15 +10564,15 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, memcpy(&mdest_res->sample_act[0], &res->sample_act, sizeof(struct mlx5_flow_sub_actions_list)); mdest_res->num_of_dest = num_of_dest; - if (flow_dv_dest_array_resource_register(dev, attr, mdest_res, + if (flow_dv_dest_array_resource_register(dev, mdest_res, dev_flow, error)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can't create sample " "action"); } else { - if (flow_dv_sample_resource_register(dev, attr, res, dev_flow, - sample_actions, error)) + res->sub_actions = sample_actions; + if (flow_dv_sample_resource_register(dev, res, dev_flow, error)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -9200,6 +10581,273 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, return 0; } +/** + * Remove an ASO age action from age actions list. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] age + * Pointer to the aso age action handler. + */ +static void +flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev, + struct mlx5_aso_age_action *age) +{ + struct mlx5_age_info *age_info; + struct mlx5_age_param *age_param = &age->age_params; + struct mlx5_priv *priv = dev->data->dev_private; + uint16_t expected = AGE_CANDIDATE; + + age_info = GET_PORT_AGE_INFO(priv); + if (!__atomic_compare_exchange_n(&age_param->state, &expected, + AGE_FREE, false, __ATOMIC_RELAXED, + __ATOMIC_RELAXED)) { + /** + * We need the lock even it is age timeout, + * since age action may still in process. + */ + rte_spinlock_lock(&age_info->aged_sl); + LIST_REMOVE(age, next); + rte_spinlock_unlock(&age_info->aged_sl); + __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED); + } +} + +/** + * Release an ASO age action. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] age_idx + * Index of ASO age action to release. + * @param[in] flow + * True if the release operation is during flow destroy operation. + * False if the release operation is during action destroy operation. + * + * @return + * 0 when age action was removed, otherwise the number of references. + */ +static int +flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx); + uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED); + + if (!ret) { + flow_dv_aso_age_remove_from_age(dev, age); + rte_spinlock_lock(&mng->free_sl); + LIST_INSERT_HEAD(&mng->free, age, next); + rte_spinlock_unlock(&mng->free_sl); + } + return ret; +} + +/** + * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * 0 on success, otherwise negative errno value and rte_errno is set. + */ +static int +flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + void *old_pools = mng->pools; + uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE; + uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize; + void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); + + if (!pools) { + rte_errno = ENOMEM; + return -ENOMEM; + } + if (old_pools) { + memcpy(pools, old_pools, + mng->n * sizeof(struct mlx5_flow_counter_pool *)); + mlx5_free(old_pools); + } else { + /* First ASO flow hit allocation - starting ASO data-path. */ + int ret = mlx5_aso_queue_start(priv->sh); + + if (ret) { + mlx5_free(pools); + return ret; + } + } + mng->n = resize; + mng->pools = pools; + return 0; +} + +/** + * Create and initialize a new ASO aging pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] age_free + * Where to put the pointer of a new age action. + * + * @return + * The age actions pool pointer and @p age_free is set on success, + * NULL otherwise and rte_errno is set. + */ +static struct mlx5_aso_age_pool * +flow_dv_age_pool_create(struct rte_eth_dev *dev, + struct mlx5_aso_age_action **age_free) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + struct mlx5_aso_age_pool *pool = NULL; + struct mlx5_devx_obj *obj = NULL; + uint32_t i; + + obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->ctx, + priv->sh->pdn); + if (!obj) { + rte_errno = ENODATA; + DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX."); + return NULL; + } + pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); + if (!pool) { + claim_zero(mlx5_devx_cmd_destroy(obj)); + rte_errno = ENOMEM; + return NULL; + } + pool->flow_hit_aso_obj = obj; + pool->time_of_last_age_check = MLX5_CURR_TIME_SEC; + rte_spinlock_lock(&mng->resize_sl); + pool->index = mng->next; + /* Resize pools array if there is no room for the new pool in it. */ + if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) { + claim_zero(mlx5_devx_cmd_destroy(obj)); + mlx5_free(pool); + rte_spinlock_unlock(&mng->resize_sl); + return NULL; + } + mng->pools[pool->index] = pool; + mng->next++; + rte_spinlock_unlock(&mng->resize_sl); + /* Assign the first action in the new pool, the rest go to free list. */ + *age_free = &pool->actions[0]; + for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) { + pool->actions[i].offset = i; + LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next); + } + return pool; +} + +/** + * Allocate a ASO aging bit. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] error + * Pointer to the error structure. + * + * @return + * Index to ASO age action on success, 0 otherwise and rte_errno is set. + */ +static uint32_t +flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const struct mlx5_aso_age_pool *pool; + struct mlx5_aso_age_action *age_free = NULL; + struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; + + MLX5_ASSERT(mng); + /* Try to get the next free age action bit. */ + rte_spinlock_lock(&mng->free_sl); + age_free = LIST_FIRST(&mng->free); + if (age_free) { + LIST_REMOVE(age_free, next); + } else if (!flow_dv_age_pool_create(dev, &age_free)) { + rte_spinlock_unlock(&mng->free_sl); + rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "failed to create ASO age pool"); + return 0; /* 0 is an error. */ + } + rte_spinlock_unlock(&mng->free_sl); + pool = container_of + ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL]) + (age_free - age_free->offset), const struct mlx5_aso_age_pool, + actions); + if (!age_free->dr_action) { + int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0, + error); + + if (reg_c < 0) { + rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "failed to get reg_c " + "for ASO flow hit"); + return 0; /* 0 is an error. */ + } +#ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO + age_free->dr_action = mlx5_glue->dv_create_flow_action_aso + (priv->sh->rx_domain, + pool->flow_hit_aso_obj->obj, age_free->offset, + MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET, + (reg_c - REG_C_0)); +#endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ + if (!age_free->dr_action) { + rte_errno = errno; + rte_spinlock_lock(&mng->free_sl); + LIST_INSERT_HEAD(&mng->free, age_free, next); + rte_spinlock_unlock(&mng->free_sl); + rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "failed to create ASO " + "flow hit action"); + return 0; /* 0 is an error. */ + } + } + __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED); + return pool->index | ((age_free->offset + 1) << 16); +} + +/** + * Create a age action using ASO mechanism. + * + * @param[in] dev + * Pointer to rte_eth_dev structure. + * @param[in] age + * Pointer to the aging action configuration. + * @param[out] error + * Pointer to the error structure. + * + * @return + * Index to flow counter on success, 0 otherwise. + */ +static uint32_t +flow_dv_translate_create_aso_age(struct rte_eth_dev *dev, + const struct rte_flow_action_age *age, + struct rte_flow_error *error) +{ + uint32_t age_idx = 0; + struct mlx5_aso_age_action *aso_age; + + age_idx = flow_dv_aso_age_alloc(dev, error); + if (!age_idx) + return 0; + aso_age = flow_aso_age_get_by_idx(dev, age_idx); + aso_age->age_params.context = age->context; + aso_age->age_params.timeout = age->timeout; + aso_age->age_params.port_id = dev->data->port_id; + __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0, + __ATOMIC_RELAXED); + __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE, + __ATOMIC_RELAXED); + return age_idx; +} + /** * Fill the flow with DV spec, lock free * (mutex should be acquired by caller). @@ -9221,24 +10869,22 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -__flow_dv_translate(struct rte_eth_dev *dev, - struct mlx5_flow *dev_flow, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - struct rte_flow_error *error) +flow_dv_translate(struct rte_eth_dev *dev, + struct mlx5_flow *dev_flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_config *dev_conf = &priv->config; struct rte_flow *flow = dev_flow->flow; struct mlx5_flow_handle *handle = dev_flow->handle; - struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) - priv->rss_desc) - [!!priv->flow_nested_idx]; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + struct mlx5_flow_rss_desc *rss_desc; uint64_t item_flags = 0; uint64_t last_item = 0; uint64_t action_flags = 0; - uint64_t priority = attr->priority; struct mlx5_flow_dv_matcher matcher = { .mask = { .size = sizeof(matcher.mask.buf) - @@ -9267,6 +10913,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, struct mlx5_flow_dv_dest_array_resource mdest_res; struct mlx5_flow_dv_sample_resource sample_res; void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0}; + const struct rte_flow_action_sample *sample = NULL; struct mlx5_flow_sub_actions_list *sample_act; uint32_t sample_act_pos = UINT32_MAX; uint32_t num_of_dest = 0; @@ -9278,8 +10925,16 @@ __flow_dv_translate(struct rte_eth_dev *dev, .external = !!dev_flow->external, .transfer = !!attr->transfer, .fdb_def_rule = !!priv->fdb_def_rule, + .skip_scale = dev_flow->skip_scale & + (1 << MLX5_SCALE_FLOW_GROUP_BIT), }; + if (!wks) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "failed to push flow workspace"); + rss_desc = &wks->rss_desc; memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource)); memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource)); mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : @@ -9296,24 +10951,26 @@ __flow_dv_translate(struct rte_eth_dev *dev, grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate (dev, tunnel, attr, items, actions); ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table, - grp_info, error); + &grp_info, error); if (ret) return ret; dev_flow->dv.group = table; if (attr->transfer) mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; - if (priority == MLX5_FLOW_PRIO_RSVD) - priority = dev_conf->flow_prio - 1; /* number of actions must be set to 0 in case of dirty stack. */ mhdr_res->actions_num = 0; if (is_flow_tunnel_match_rule(dev, attr, items, actions)) { /* * do not add decap action if match rule drops packet * HW rejects rules with decap & drop + * + * if tunnel match rule was inserted before matching tunnel set + * rule flow table used in the match rule must be registered. + * current implementation handles that in the + * flow_dv_match_register() at the function end. */ bool add_decap = true; const struct rte_flow_action *ptr = actions; - struct mlx5_flow_tbl_resource *tbl; for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) { if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) { @@ -9330,20 +10987,6 @@ __flow_dv_translate(struct rte_eth_dev *dev, dev_flow->dv.encap_decap->action; action_flags |= MLX5_FLOW_ACTION_DECAP; } - /* - * bind table_id with for tunnel match rule. - * Tunnel set rule establishes that bind in JUMP action handler. - * Required for scenario when application creates tunnel match - * rule before tunnel set rule. - */ - tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, - attr->transfer, - !!dev_flow->external, tunnel, - attr->group, error); - if (!tbl) - return rte_flow_error_set - (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, - actions, "cannot register tunnel group"); } for (; !actions_end ; actions++) { const struct rte_flow_action_queue *queue; @@ -9352,6 +10995,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, const uint8_t *rss_key; const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; + struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; @@ -9486,9 +11130,43 @@ __flow_dv_translate(struct rte_eth_dev *dev, * when expanding items for RSS. */ action_flags |= MLX5_FLOW_ACTION_RSS; - dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; + dev_flow->handle->fate_action = rss_desc->shared_rss ? + MLX5_FLOW_FATE_SHARED_RSS : + MLX5_FLOW_FATE_QUEUE; + break; + case MLX5_RTE_FLOW_ACTION_TYPE_AGE: + flow->age = (uint32_t)(uintptr_t)(action->conf); + age_act = flow_aso_age_get_by_idx(dev, flow->age); + __atomic_fetch_add(&age_act->refcnt, 1, + __ATOMIC_RELAXED); + dev_flow->dv.actions[actions_n++] = age_act->dr_action; + action_flags |= MLX5_FLOW_ACTION_AGE; break; case RTE_FLOW_ACTION_TYPE_AGE: + if (priv->sh->flow_hit_aso_en && attr->group) { + /* + * Create one shared age action, to be used + * by all sub-flows. + */ + if (!flow->age) { + flow->age = + flow_dv_translate_create_aso_age + (dev, action->conf, + error); + if (!flow->age) + return rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "can't create ASO age action"); + } + dev_flow->dv.actions[actions_n++] = + (flow_aso_age_get_by_idx + (dev, flow->age))->dr_action; + action_flags |= MLX5_FLOW_ACTION_AGE; + break; + } + /* Fall-through */ case RTE_FLOW_ACTION_TYPE_COUNT: if (!dev_conf->devx) { return rte_flow_error_set @@ -9612,16 +11290,21 @@ __flow_dv_translate(struct rte_eth_dev *dev, jump_group = ((const struct rte_flow_action_jump *) action->conf)->group; grp_info.std_tbl_fix = 0; + if (dev_flow->skip_scale & + (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT)) + grp_info.skip_scale = 1; + else + grp_info.skip_scale = 0; ret = mlx5_flow_group_to_table(dev, tunnel, jump_group, &table, - grp_info, error); + &grp_info, error); if (ret) return ret; tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, attr->transfer, !!dev_flow->external, - tunnel, jump_group, + tunnel, jump_group, 0, error); if (!tbl) return rte_flow_error_set @@ -9631,7 +11314,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, "cannot create jump action."); if (flow_dv_jump_tbl_resource_register (dev, tbl, dev_flow, error)) { - flow_dv_tbl_resource_release(dev, tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); return rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_ACTION, @@ -9642,6 +11325,8 @@ __flow_dv_translate(struct rte_eth_dev *dev, dev_flow->dv.jump->action; action_flags |= MLX5_FLOW_ACTION_JUMP; dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP; + sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP; + num_of_dest++; break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: @@ -9783,15 +11468,8 @@ __flow_dv_translate(struct rte_eth_dev *dev, break; case RTE_FLOW_ACTION_TYPE_SAMPLE: sample_act_pos = actions_n; - ret = flow_dv_translate_action_sample(dev, - actions, - dev_flow, attr, - &num_of_dest, - sample_actions, - &sample_res, - error); - if (ret < 0) - return ret; + sample = (const struct rte_flow_action_sample *) + action->conf; actions_n++; action_flags |= MLX5_FLOW_ACTION_SAMPLE; /* put encap action into group if work with port id */ @@ -9800,6 +11478,12 @@ __flow_dv_translate(struct rte_eth_dev *dev, sample_act->action_flags |= MLX5_FLOW_ACTION_ENCAP; break; + case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: + if (flow_dv_convert_action_modify_field + (dev, mhdr_res, actions, attr, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD; + break; case RTE_FLOW_ACTION_TYPE_END: actions_end = true; if (mhdr_res->actions_num) { @@ -9811,46 +11495,27 @@ __flow_dv_translate(struct rte_eth_dev *dev, handle->dvh.modify_hdr->action; } if (action_flags & MLX5_FLOW_ACTION_COUNT) { - flow->counter = - flow_dv_translate_create_counter(dev, - dev_flow, count, age); - - if (!flow->counter) - return rte_flow_error_set + /* + * Create one count action, to be used + * by all sub-flows. + */ + if (!flow->counter) { + flow->counter = + flow_dv_translate_create_counter + (dev, dev_flow, count, + age); + if (!flow->counter) + return rte_flow_error_set (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot create counter" - " object."); + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "cannot create counter" + " object."); + } dev_flow->dv.actions[actions_n] = (flow_dv_counter_get_by_idx(dev, flow->counter, NULL))->action; actions_n++; } - if (action_flags & MLX5_FLOW_ACTION_SAMPLE) { - ret = flow_dv_create_action_sample(dev, - dev_flow, attr, - num_of_dest, - &sample_res, - &mdest_res, - sample_actions, - action_flags, - error); - if (ret < 0) - return rte_flow_error_set - (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "cannot create sample action"); - if (num_of_dest > 1) { - dev_flow->dv.actions[sample_act_pos] = - dev_flow->dv.dest_array_res->action; - } else { - dev_flow->dv.actions[sample_act_pos] = - dev_flow->dv.sample_res->verbs_action; - } - } - break; default: break; } @@ -9858,33 +11523,6 @@ __flow_dv_translate(struct rte_eth_dev *dev, modify_action_position == UINT32_MAX) modify_action_position = actions_n++; } - /* - * For multiple destination (sample action with ratio=1), the encap - * action and port id action will be combined into group action. - * So need remove the original these actions in the flow and only - * use the sample action instead of. - */ - if (num_of_dest > 1 && sample_act->dr_port_id_action) { - int i; - void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0}; - - for (i = 0; i < actions_n; i++) { - if ((sample_act->dr_encap_action && - sample_act->dr_encap_action == - dev_flow->dv.actions[i]) || - (sample_act->dr_port_id_action && - sample_act->dr_port_id_action == - dev_flow->dv.actions[i])) - continue; - temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i]; - } - memcpy((void *)dev_flow->dv.actions, - (void *)temp_actions, - tmp_actions_n * sizeof(void *)); - actions_n = tmp_actions_n; - } - dev_flow->dv.actions_n = actions_n; - dev_flow->act_flags = action_flags; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); int item_type = items->type; @@ -9895,8 +11533,8 @@ __flow_dv_translate(struct rte_eth_dev *dev, NULL, "item not supported"); switch (item_type) { case RTE_FLOW_ITEM_TYPE_PORT_ID: - flow_dv_translate_item_port_id(dev, match_mask, - match_value, items); + flow_dv_translate_item_port_id + (dev, match_mask, match_value, items, attr); last_item = MLX5_FLOW_ITEM_PORT_ID; break; case RTE_FLOW_ITEM_TYPE_ETH: @@ -9926,7 +11564,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, mlx5_flow_tunnel_ip_check(items, next_protocol, &item_flags, &tunnel); flow_dv_translate_item_ipv4(match_mask, match_value, - items, item_flags, tunnel, + items, tunnel, dev_flow->dv.group); matcher.priority = MLX5_PRIORITY_MAP_L3; last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : @@ -9949,7 +11587,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, mlx5_flow_tunnel_ip_check(items, next_protocol, &item_flags, &tunnel); flow_dv_translate_item_ipv6(match_mask, match_value, - items, item_flags, tunnel, + items, tunnel, dev_flow->dv.group); matcher.priority = MLX5_PRIORITY_MAP_L3; last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : @@ -10039,6 +11677,17 @@ __flow_dv_translate(struct rte_eth_dev *dev, matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); last_item = MLX5_FLOW_LAYER_GENEVE; break; + case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: + ret = flow_dv_translate_item_geneve_opt(dev, match_mask, + match_value, + items, error); + if (ret) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "cannot create GENEVE TLV option"); + flow->geneve_tlv_option = 1; + last_item = MLX5_FLOW_LAYER_GENEVE_OPT; + break; case RTE_FLOW_ITEM_TYPE_MPLS: flow_dv_translate_item_mpls(match_mask, match_value, items, last_item, tunnel); @@ -10087,6 +11736,16 @@ __flow_dv_translate(struct rte_eth_dev *dev, matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); last_item = MLX5_FLOW_LAYER_GTP; break; + case RTE_FLOW_ITEM_TYPE_GTP_PSC: + ret = flow_dv_translate_item_gtp_psc(match_mask, + match_value, + items); + if (ret) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "cannot create GTP PSC item"); + last_item = MLX5_FLOW_LAYER_GTP_PSC; + break; case RTE_FLOW_ITEM_TYPE_ECPRI: if (!mlx5_flex_parser_ecpri_exist(dev)) { /* Create it only the first time to be used. */ @@ -10122,7 +11781,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && (priv->representor || priv->master)) { if (flow_dv_translate_item_port_id(dev, match_mask, - match_value, NULL)) + match_value, NULL, attr)) return -rte_errno; } #ifdef RTE_LIBRTE_MLX5_DEBUG @@ -10136,16 +11795,83 @@ __flow_dv_translate(struct rte_eth_dev *dev, handle->layers |= item_flags; if (action_flags & MLX5_FLOW_ACTION_RSS) flow_dv_hashfields_set(dev_flow, rss_desc); + /* If has RSS action in the sample action, the Sample/Mirror resource + * should be registered after the hash filed be update. + */ + if (action_flags & MLX5_FLOW_ACTION_SAMPLE) { + ret = flow_dv_translate_action_sample(dev, + sample, + dev_flow, attr, + &num_of_dest, + sample_actions, + &sample_res, + error); + if (ret < 0) + return ret; + ret = flow_dv_create_action_sample(dev, + dev_flow, + num_of_dest, + &sample_res, + &mdest_res, + sample_actions, + action_flags, + error); + if (ret < 0) + return rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot create sample action"); + if (num_of_dest > 1) { + dev_flow->dv.actions[sample_act_pos] = + dev_flow->dv.dest_array_res->action; + } else { + dev_flow->dv.actions[sample_act_pos] = + dev_flow->dv.sample_res->verbs_action; + } + } + /* + * For multiple destination (sample action with ratio=1), the encap + * action and port id action will be combined into group action. + * So need remove the original these actions in the flow and only + * use the sample action instead of. + */ + if (num_of_dest > 1 && + (sample_act->dr_port_id_action || sample_act->dr_jump_action)) { + int i; + void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0}; + + for (i = 0; i < actions_n; i++) { + if ((sample_act->dr_encap_action && + sample_act->dr_encap_action == + dev_flow->dv.actions[i]) || + (sample_act->dr_port_id_action && + sample_act->dr_port_id_action == + dev_flow->dv.actions[i]) || + (sample_act->dr_jump_action && + sample_act->dr_jump_action == + dev_flow->dv.actions[i])) + continue; + temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i]; + } + memcpy((void *)dev_flow->dv.actions, + (void *)temp_actions, + tmp_actions_n * sizeof(void *)); + actions_n = tmp_actions_n; + } + dev_flow->dv.actions_n = actions_n; + dev_flow->act_flags = action_flags; /* Register matcher. */ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, matcher.mask.size); - matcher.priority = mlx5_flow_adjust_priority(dev, priority, - matcher.priority); + matcher.priority = mlx5_get_matcher_priority(dev, attr, + matcher.priority); /* reserved field no needs to be set to 0 here. */ tbl_key.domain = attr->transfer; tbl_key.direction = attr->egress; tbl_key.table_id = dev_flow->dv.group; - if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error)) + if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, + tunnel, attr->group, error)) return -rte_errno; return 0; } @@ -10205,8 +11931,10 @@ __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields) * and tunnel. * - * @param[in] action - * Shred RSS action holding hash RX queue objects. + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] idx + * Shared RSS action ID holding hash RX queue objects. * @param[in] hash_fields * Defines combination of packet fields to participate in RX hash. * @param[in] tunnel @@ -10216,11 +11944,15 @@ __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action, * Valid hash RX queue index, otherwise 0. */ static uint32_t -__flow_dv_action_rss_hrxq_lookup(const struct mlx5_shared_action_rss *action, +__flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx, const uint64_t hash_fields, const int tunnel) { - const uint32_t *hrxqs = tunnel ? action->hrxq : action->hrxq_tunnel; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss = + mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); + const uint32_t *hrxqs = tunnel ? shared_rss->hrxq : + shared_rss->hrxq_tunnel; switch (hash_fields & ~IBV_RX_HASH_INNER) { case MLX5_RSS_HASH_IPV4: @@ -10242,66 +11974,6 @@ __flow_dv_action_rss_hrxq_lookup(const struct mlx5_shared_action_rss *action, } } -/** - * Retrieves hash RX queue suitable for the *flow*. - * If shared action configured for *flow* suitable hash RX queue will be - * retrieved from attached shared action. - * - * @param[in] flow - * Shred RSS action holding hash RX queue objects. - * @param[in] dev_flow - * Pointer to the sub flow. - * @param[out] hrxq - * Pointer to retrieved hash RX queue object. - * - * @return - * Valid hash RX queue index, otherwise 0 and rte_errno is set. - */ -static uint32_t -__flow_dv_rss_get_hrxq(struct rte_eth_dev *dev, struct rte_flow *flow, - struct mlx5_flow *dev_flow, - struct mlx5_hrxq **hrxq) -{ - struct mlx5_priv *priv = dev->data->dev_private; - uint32_t hrxq_idx; - - if (flow->shared_rss) { - hrxq_idx = __flow_dv_action_rss_hrxq_lookup - (flow->shared_rss, dev_flow->hash_fields, - !!(dev_flow->handle->layers & - MLX5_FLOW_LAYER_TUNNEL)); - if (hrxq_idx) { - *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], - hrxq_idx); - rte_atomic32_inc(&(*hrxq)->refcnt); - } - } else { - struct mlx5_flow_rss_desc *rss_desc = - &((struct mlx5_flow_rss_desc *)priv->rss_desc) - [!!priv->flow_nested_idx]; - - MLX5_ASSERT(rss_desc->queue_num); - hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, rss_desc->queue_num); - if (!hrxq_idx) { - hrxq_idx = mlx5_hrxq_new(dev, - rss_desc->key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, - rss_desc->queue_num, - !!(dev_flow->handle->layers & - MLX5_FLOW_LAYER_TUNNEL), - false); - } - *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], - hrxq_idx); - } - return hrxq_idx; -} - /** * Apply the flow to the NIC, lock free, * (mutex should be acquired by caller). @@ -10317,8 +11989,8 @@ __flow_dv_rss_get_hrxq(struct rte_eth_dev *dev, struct rte_flow *flow, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -__flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, - struct rte_flow_error *error) +flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, + struct rte_flow_error *error) { struct mlx5_flow_dv_workspace *dv; struct mlx5_flow_handle *dh; @@ -10329,9 +12001,12 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, int n; int err; int idx; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc; - for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) { - dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; + MLX5_ASSERT(wks); + for (idx = wks->flow_idx - 1; idx >= 0; idx--) { + dev_flow = &wks->flows[idx]; dv = &dev_flow->dv; dh = dev_flow->handle; dv_h = &dh->dvh; @@ -10340,31 +12015,17 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, if (dv->transfer) { dv->actions[n++] = priv->sh->esw_drop_action; } else { - struct mlx5_hrxq *drop_hrxq; - drop_hrxq = mlx5_drop_action_create(dev); - if (!drop_hrxq) { - rte_flow_error_set - (error, errno, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "cannot get drop hash queue"); - goto error; - } - /* - * Drop queues will be released by the specify - * mlx5_drop_action_destroy() function. Assign - * the special index to hrxq to mark the queue - * has been allocated. - */ - dh->rix_hrxq = UINT32_MAX; - dv->actions[n++] = drop_hrxq->action; + MLX5_ASSERT(priv->drop_queue.hrxq); + dv->actions[n++] = + priv->drop_queue.hrxq->action; } - } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && - !dv_h->rix_sample && !dv_h->rix_dest_array) { - struct mlx5_hrxq *hrxq = NULL; - uint32_t hrxq_idx = __flow_dv_rss_get_hrxq - (dev, flow, dev_flow, &hrxq); + } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE && + !dv_h->rix_sample && !dv_h->rix_dest_array)) { + struct mlx5_hrxq *hrxq; + uint32_t hrxq_idx; + hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc, + &hrxq_idx); if (!hrxq) { rte_flow_error_set (error, rte_errno, @@ -10374,17 +12035,37 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, } dh->rix_hrxq = hrxq_idx; dv->actions[n++] = hrxq->action; + } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { + struct mlx5_hrxq *hrxq = NULL; + uint32_t hrxq_idx; + + hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev, + rss_desc->shared_rss, + dev_flow->hash_fields, + !!(dh->layers & + MLX5_FLOW_LAYER_TUNNEL)); + if (hrxq_idx) + hrxq = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_HRXQ], + hrxq_idx); + if (!hrxq) { + rte_flow_error_set + (error, rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot get hash queue"); + goto error; + } + dh->rix_srss = rss_desc->shared_rss; + dv->actions[n++] = hrxq->action; } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) { - if (flow_dv_default_miss_resource_register - (dev, error)) { + if (!priv->sh->default_miss_action) { rte_flow_error_set (error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot create default miss resource"); - goto error_default_miss; + "default miss action not be created."); + goto error; } - dh->rix_default_fate = MLX5_FLOW_FATE_DEFAULT_MISS; - dv->actions[n++] = priv->sh->default_miss.action; + dv->actions[n++] = priv->sh->default_miss_action; } err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object, (void *)&dv->value, n, @@ -10409,21 +12090,15 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, } return 0; error: - if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) - flow_dv_default_miss_resource_release(dev); -error_default_miss: err = rte_errno; /* Save rte_errno before cleanup. */ SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dh, next) { /* hrxq is union, don't clear it if the flag is not set. */ - if (dh->rix_hrxq) { - if (dh->fate_action == MLX5_FLOW_FATE_DROP) { - mlx5_drop_action_destroy(dev); - dh->rix_hrxq = 0; - } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { - mlx5_hrxq_release(dev, dh->rix_hrxq); - dh->rix_hrxq = 0; - } + if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) { + mlx5_hrxq_release(dev, dh->rix_hrxq); + dh->rix_hrxq = 0; + } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { + dh->rix_srss = 0; } if (dh->vf_vlan.tag && dh->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); @@ -10432,13 +12107,24 @@ error_default_miss: return -rte_errno; } +void +flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry) +{ + struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache), + entry); + + claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object)); + mlx5_free(cache); +} + /** * Release the flow matcher. * * @param dev * Pointer to Ethernet device. - * @param handle - * Pointer to mlx5_flow_handle. + * @param port_id + * Index to port ID action resource. * * @return * 1 while a reference on it exists, 0 when freed. @@ -10448,23 +12134,34 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher; + struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl, + typeof(*tbl), tbl); + int ret; MLX5_ASSERT(matcher->matcher_object); - DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", - dev->data->port_id, (void *)matcher, - rte_atomic32_read(&matcher->refcnt)); - if (rte_atomic32_dec_and_test(&matcher->refcnt)) { - claim_zero(mlx5_flow_os_destroy_flow_matcher - (matcher->matcher_object)); - LIST_REMOVE(matcher, next); - /* table ref-- in release interface. */ - flow_dv_tbl_resource_release(dev, matcher->tbl); - mlx5_free(matcher); - DRV_LOG(DEBUG, "port %u matcher %p: removed", - dev->data->port_id, (void *)matcher); - return 0; - } - return 1; + ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry); + flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl); + return ret; +} + +/** + * Release encap_decap resource. + * + * @param list + * Pointer to the hash list. + * @param entry + * Pointer to exist resource entry object. + */ +void +flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_dv_encap_decap_resource *res = + container_of(entry, typeof(*res), entry); + + claim_zero(mlx5_flow_os_destroy_flow_action(res->action)); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx); } /** @@ -10483,28 +12180,15 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, uint32_t encap_decap_idx) { struct mlx5_priv *priv = dev->data->dev_private; - uint32_t idx = encap_decap_idx; struct mlx5_flow_dv_encap_decap_resource *cache_resource; cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], - idx); + encap_decap_idx); if (!cache_resource) return 0; MLX5_ASSERT(cache_resource->action); - DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { - claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->action)); - mlx5_hlist_remove(priv->sh->encaps_decaps, - &cache_resource->entry); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx); - DRV_LOG(DEBUG, "encap/decap resource %p: removed", - (void *)cache_resource); - return 0; - } - return 1; + return mlx5_hlist_unregister(priv->sh->encaps_decaps, + &cache_resource->entry); } /** @@ -10512,69 +12196,35 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, * * @param dev * Pointer to Ethernet device. - * @param handle - * Pointer to mlx5_flow_handle. + * @param rix_jump + * Index to the jump action resource. * * @return * 1 while a reference on it exists, 0 when freed. */ static int flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, - struct mlx5_flow_handle *handle) + uint32_t rix_jump) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_jump_tbl_resource *cache_resource; struct mlx5_flow_tbl_data_entry *tbl_data; tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP], - handle->rix_jump); + rix_jump); if (!tbl_data) return 0; - cache_resource = &tbl_data->jump; - MLX5_ASSERT(cache_resource->action); - DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { - claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->action)); - /* jump action memory free is inside the table release. */ - flow_dv_tbl_resource_release(dev, &tbl_data->tbl); - DRV_LOG(DEBUG, "jump table resource %p: removed", - (void *)cache_resource); - return 0; - } - return 1; + return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl); } -/** - * Release a default miss resource. - * - * @param dev - * Pointer to Ethernet device. - * @return - * 1 while a reference on it exists, 0 when freed. - */ -static int -flow_dv_default_miss_resource_release(struct rte_eth_dev *dev) +void +flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused, + struct mlx5_hlist_entry *entry) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_default_miss_resource *cache_resource = - &sh->default_miss; + struct mlx5_flow_dv_modify_hdr_resource *res = + container_of(entry, typeof(*res), entry); - MLX5_ASSERT(cache_resource->action); - DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--", - (void *)cache_resource->action, - rte_atomic32_read(&cache_resource->refcnt)); - if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->action)); - DRV_LOG(DEBUG, "default miss resource %p: removed", - (void *)cache_resource->action); - return 0; - } - return 1; + claim_zero(mlx5_flow_os_destroy_flow_action(res->action)); + mlx5_free(entry); } /** @@ -10593,24 +12243,22 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_modify_hdr_resource *cache_resource = - handle->dvh.modify_hdr; + struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr; - MLX5_ASSERT(cache_resource->action); - DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { - claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->action)); - mlx5_hlist_remove(priv->sh->modify_cmds, - &cache_resource->entry); - mlx5_free(cache_resource); - DRV_LOG(DEBUG, "modify-header resource %p: removed", - (void *)cache_resource); - return 0; - } - return 1; + MLX5_ASSERT(entry->action); + return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry); +} + +void +flow_dv_port_id_remove_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_dv_port_id_action_resource *cache = + container_of(entry, typeof(*cache), entry); + + claim_zero(mlx5_flow_os_destroy_flow_action(cache->action)); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx); } /** @@ -10629,29 +12277,45 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, uint32_t port_id) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_port_id_action_resource *cache_resource; - uint32_t idx = port_id; + struct mlx5_flow_dv_port_id_action_resource *cache; - cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], - idx); - if (!cache_resource) + cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id); + if (!cache) return 0; - MLX5_ASSERT(cache_resource->action); - DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { - claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->action)); - ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID], - &priv->sh->port_id_action_list, idx, - cache_resource, next); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx); - DRV_LOG(DEBUG, "port id action resource %p: removed", - (void *)cache_resource); - return 0; - } - return 1; + MLX5_ASSERT(cache->action); + return mlx5_cache_unregister(&priv->sh->port_id_action_list, + &cache->entry); +} + +/** + * Release shared RSS action resource. + * + * @param dev + * Pointer to Ethernet device. + * @param srss + * Shared RSS action index. + */ +static void +flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss; + + shared_rss = mlx5_ipool_get + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss); + __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED); +} + +void +flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_dv_push_vlan_action_resource *cache = + container_of(entry, typeof(*cache), entry); + + claim_zero(mlx5_flow_os_destroy_flow_action(cache->action)); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx); } /** @@ -10670,29 +12334,15 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_dv_push_vlan_action_resource *cache; uint32_t idx = handle->dvh.rix_push_vlan; - struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; - cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], - idx); - if (!cache_resource) - return 0; - MLX5_ASSERT(cache_resource->action); - DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { - claim_zero(mlx5_flow_os_destroy_flow_action - (cache_resource->action)); - ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], - &priv->sh->push_vlan_action_list, idx, - cache_resource, next); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); - DRV_LOG(DEBUG, "push vlan action resource %p: removed", - (void *)cache_resource); + cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); + if (!cache) return 0; - } - return 1; + MLX5_ASSERT(cache->action); + return mlx5_cache_unregister(&priv->sh->push_vlan_action_list, + &cache->entry); } /** @@ -10710,22 +12360,16 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev, if (!handle->rix_fate) return; switch (handle->fate_action) { - case MLX5_FLOW_FATE_DROP: - mlx5_drop_action_destroy(dev); - break; case MLX5_FLOW_FATE_QUEUE: mlx5_hrxq_release(dev, handle->rix_hrxq); break; case MLX5_FLOW_FATE_JUMP: - flow_dv_jump_tbl_resource_release(dev, handle); + flow_dv_jump_tbl_resource_release(dev, handle->rix_jump); break; case MLX5_FLOW_FATE_PORT_ID: flow_dv_port_id_action_resource_release(dev, handle->rix_port_id_action); break; - case MLX5_FLOW_FATE_DEFAULT_MISS: - flow_dv_default_miss_resource_release(dev); - break; default: DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action); break; @@ -10733,6 +12377,29 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev, handle->rix_fate = 0; } +void +flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry) +{ + struct mlx5_flow_dv_sample_resource *cache_resource = + container_of(entry, typeof(*cache_resource), entry); + struct rte_eth_dev *dev = cache_resource->dev; + struct mlx5_priv *priv = dev->data->dev_private; + + if (cache_resource->verbs_action) + claim_zero(mlx5_flow_os_destroy_flow_action + (cache_resource->verbs_action)); + if (cache_resource->normal_path_tbl) + flow_dv_tbl_resource_release(MLX5_SH(dev), + cache_resource->normal_path_tbl); + flow_dv_sample_sub_actions_release(dev, + &cache_resource->sample_idx); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], + cache_resource->idx); + DRV_LOG(DEBUG, "sample resource %p: removed", + (void *)cache_resource); +} + /** * Release an sample resource. * @@ -10749,54 +12416,38 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev, struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - uint32_t idx = handle->dvh.rix_sample; struct mlx5_flow_dv_sample_resource *cache_resource; cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE], - idx); + handle->dvh.rix_sample); if (!cache_resource) return 0; MLX5_ASSERT(cache_resource->verbs_action); - DRV_LOG(DEBUG, "sample resource %p: refcnt %d--", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)); - if (__atomic_sub_fetch(&cache_resource->refcnt, 1, - __ATOMIC_RELAXED) == 0) { - if (cache_resource->verbs_action) - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->verbs_action)); - if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) { - if (cache_resource->default_miss) - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->default_miss)); - } - if (cache_resource->normal_path_tbl) - flow_dv_tbl_resource_release(dev, - cache_resource->normal_path_tbl); - } - if (cache_resource->sample_idx.rix_hrxq && - !mlx5_hrxq_release(dev, - cache_resource->sample_idx.rix_hrxq)) - cache_resource->sample_idx.rix_hrxq = 0; - if (cache_resource->sample_idx.rix_tag && - !flow_dv_tag_release(dev, - cache_resource->sample_idx.rix_tag)) - cache_resource->sample_idx.rix_tag = 0; - if (cache_resource->sample_idx.cnt) { - flow_dv_counter_release(dev, - cache_resource->sample_idx.cnt); - cache_resource->sample_idx.cnt = 0; - } - if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) { - ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE], - &priv->sh->sample_action_list, idx, - cache_resource, next); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx); - DRV_LOG(DEBUG, "sample resource %p: removed", - (void *)cache_resource); - return 0; - } - return 1; + return mlx5_cache_unregister(&priv->sh->sample_action_list, + &cache_resource->entry); +} + +void +flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused, + struct mlx5_cache_entry *entry) +{ + struct mlx5_flow_dv_dest_array_resource *cache_resource = + container_of(entry, typeof(*cache_resource), entry); + struct rte_eth_dev *dev = cache_resource->dev; + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t i = 0; + + MLX5_ASSERT(cache_resource->action); + if (cache_resource->action) + claim_zero(mlx5_flow_os_destroy_flow_action + (cache_resource->action)); + for (; i < cache_resource->num_of_dest; i++) + flow_dv_sample_sub_actions_release(dev, + &cache_resource->sample_idx[i]); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], + cache_resource->idx); + DRV_LOG(DEBUG, "destination array resource %p: removed", + (void *)cache_resource); } /** @@ -10812,59 +12463,38 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev, */ static int flow_dv_dest_array_resource_release(struct rte_eth_dev *dev, - struct mlx5_flow_handle *handle) + struct mlx5_flow_handle *handle) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_dv_dest_array_resource *cache_resource; - struct mlx5_flow_sub_actions_idx *mdest_act_res; - uint32_t idx = handle->dvh.rix_dest_array; - uint32_t i = 0; + struct mlx5_flow_dv_dest_array_resource *cache; - cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], - idx); - if (!cache_resource) + cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], + handle->dvh.rix_dest_array); + if (!cache) return 0; - MLX5_ASSERT(cache_resource->action); - DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--", - (void *)cache_resource, - __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)); - if (__atomic_sub_fetch(&cache_resource->refcnt, 1, - __ATOMIC_RELAXED) == 0) { - if (cache_resource->action) - claim_zero(mlx5_glue->destroy_flow_action - (cache_resource->action)); - for (; i < cache_resource->num_of_dest; i++) { - mdest_act_res = &cache_resource->sample_idx[i]; - if (mdest_act_res->rix_hrxq) { - mlx5_hrxq_release(dev, - mdest_act_res->rix_hrxq); - mdest_act_res->rix_hrxq = 0; - } - if (mdest_act_res->rix_encap_decap) { - flow_dv_encap_decap_resource_release(dev, - mdest_act_res->rix_encap_decap); - mdest_act_res->rix_encap_decap = 0; - } - if (mdest_act_res->rix_port_id_action) { - flow_dv_port_id_action_resource_release(dev, - mdest_act_res->rix_port_id_action); - mdest_act_res->rix_port_id_action = 0; - } - if (mdest_act_res->rix_tag) { - flow_dv_tag_release(dev, - mdest_act_res->rix_tag); - mdest_act_res->rix_tag = 0; - } + MLX5_ASSERT(cache->action); + return mlx5_cache_unregister(&priv->sh->dest_array_list, + &cache->entry); +} + +static void +flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + struct mlx5_geneve_tlv_option_resource *geneve_opt_resource = + sh->geneve_tlv_option_resource; + rte_spinlock_lock(&sh->geneve_tlv_opt_sl); + if (geneve_opt_resource) { + if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1, + __ATOMIC_RELAXED))) { + claim_zero(mlx5_devx_cmd_destroy + (geneve_opt_resource->obj)); + mlx5_free(sh->geneve_tlv_option_resource); + sh->geneve_tlv_option_resource = NULL; } - ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], - &priv->sh->dest_array_list, idx, - cache_resource, next); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx); - DRV_LOG(DEBUG, "destination array resource %p: removed", - (void *)cache_resource); - return 0; } - return 1; + rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); } /** @@ -10877,7 +12507,7 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev, * Pointer to flow structure. */ static void -__flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) +flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dh; uint32_t handle_idx; @@ -10895,9 +12525,7 @@ __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow)); dh->drv_flow = NULL; } - if (dh->fate_action == MLX5_FLOW_FATE_DROP || - dh->fate_action == MLX5_FLOW_FATE_QUEUE || - dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) + if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) flow_dv_fate_resource_release(dev, dh); if (dh->vf_vlan.tag && dh->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); @@ -10915,20 +12543,17 @@ __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) * Pointer to flow structure. */ static void -__flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) +flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct rte_flow_shared_action *shared; struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + uint32_t srss = 0; if (!flow) return; - __flow_dv_remove(dev, flow); - shared = mlx5_flow_get_shared_rss(flow); - if (shared) - __atomic_sub_fetch(&shared->refcnt, 1, __ATOMIC_RELAXED); + flow_dv_remove(dev, flow); if (flow->counter) { - flow_dv_counter_release(dev, flow->counter); + flow_dv_counter_free(dev, flow->counter); flow->counter = 0; } if (flow->meter) { @@ -10940,6 +12565,12 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) mlx5_flow_meter_detach(fm); flow->meter = 0; } + if (flow->age) + flow_dv_aso_age_release(dev, flow->age); + if (flow->geneve_tlv_option) { + flow_dv_geneve_tlv_option_resource_release(dev); + flow->geneve_tlv_option = 0; + } while (flow->dev_handles) { uint32_t tmp_idx = flow->dev_handles; @@ -10965,10 +12596,15 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) if (dev_handle->dvh.rix_tag) flow_dv_tag_release(dev, dev_handle->dvh.rix_tag); - flow_dv_fate_resource_release(dev, dev_handle); + if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS) + flow_dv_fate_resource_release(dev, dev_handle); + else if (!srss) + srss = dev_handle->rix_srss; mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } + if (srss) + flow_dv_shared_rss_action_release(dev, srss); } /** @@ -11016,10 +12652,10 @@ __flow_dv_hrxqs_release(struct rte_eth_dev *dev, */ static int __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, - struct mlx5_shared_action_rss *action) + struct mlx5_shared_action_rss *shared_rss) { - return __flow_dv_hrxqs_release(dev, &action->hrxq) + - __flow_dv_hrxqs_release(dev, &action->hrxq_tunnel); + return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq) + + __flow_dv_hrxqs_release(dev, &shared_rss->hrxq_tunnel); } /** @@ -11029,6 +12665,8 @@ __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to the Ethernet device structure. + * @param[in] action_idx + * Shared RSS action ipool index. * @param[in, out] action * Partially initialized shared RSS action. * @param[out] error @@ -11040,24 +12678,35 @@ __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev, */ static int __flow_dv_action_rss_setup(struct rte_eth_dev *dev, - struct mlx5_shared_action_rss *action, - struct rte_flow_error *error) + uint32_t action_idx, + struct mlx5_shared_action_rss *shared_rss, + struct rte_flow_error *error) { + struct mlx5_flow_rss_desc rss_desc = { 0 }; size_t i; int err; + if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) { + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot setup indirection table"); + } + memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN); + rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN; + rss_desc.const_q = shared_rss->origin.queue; + rss_desc.queue_num = shared_rss->origin.queue_num; + /* Set non-zero value to indicate a shared RSS. */ + rss_desc.shared_rss = action_idx; + rss_desc.ind_tbl = shared_rss->ind_tbl; for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) { uint32_t hrxq_idx; uint64_t hash_fields = mlx5_rss_hash_fields[i]; int tunnel; for (tunnel = 0; tunnel < 2; tunnel++) { - hrxq_idx = mlx5_hrxq_new(dev, action->origin.key, - MLX5_RSS_HASH_KEY_LEN, - hash_fields, - action->origin.queue, - action->origin.queue_num, - tunnel, true); + rss_desc.tunnel = tunnel; + rss_desc.hash_fields = hash_fields; + hrxq_idx = mlx5_hrxq_get(dev, &rss_desc); if (!hrxq_idx) { rte_flow_error_set (error, rte_errno, @@ -11066,14 +12715,16 @@ __flow_dv_action_rss_setup(struct rte_eth_dev *dev, goto error_hrxq_new; } err = __flow_dv_action_rss_hrxq_set - (action, hash_fields, tunnel, hrxq_idx); + (shared_rss, hash_fields, tunnel, hrxq_idx); MLX5_ASSERT(!err); } } return 0; error_hrxq_new: err = rte_errno; - __flow_dv_action_rss_hrxqs_release(dev, action); + __flow_dv_action_rss_hrxqs_release(dev, shared_rss); + if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true)) + shared_rss->ind_tbl = NULL; rte_errno = err; return -rte_errno; } @@ -11092,35 +12743,52 @@ error_hrxq_new: * error only. * * @return - * A valid shared action handle in case of success, NULL otherwise and + * A valid shared action ID in case of success, 0 otherwise and * rte_errno is set. */ -static struct rte_flow_shared_action * +static uint32_t __flow_dv_action_rss_create(struct rte_eth_dev *dev, const struct rte_flow_shared_action_conf *conf, const struct rte_flow_action_rss *rss, struct rte_flow_error *error) { - struct rte_flow_shared_action *shared_action = NULL; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss = NULL; void *queue = NULL; - struct mlx5_shared_action_rss *shared_rss; struct rte_flow_action_rss *origin; const uint8_t *rss_key; uint32_t queue_size = rss->queue_num * sizeof(uint16_t); + uint32_t idx; RTE_SET_USED(conf); queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)), 0, SOCKET_ID_ANY); - shared_action = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*shared_action), 0, - SOCKET_ID_ANY); - if (!shared_action || !queue) { + shared_rss = mlx5_ipool_zmalloc + (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx); + if (!shared_rss || !queue) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + goto error_rss_init; + } + if (idx > (1u << MLX5_SHARED_ACTION_TYPE_OFFSET)) { + rte_flow_error_set(error, E2BIG, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "rss action number out of range"); + goto error_rss_init; + } + shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*shared_rss->ind_tbl), + 0, SOCKET_ID_ANY); + if (!shared_rss->ind_tbl) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); goto error_rss_init; } - shared_rss = &shared_action->rss; - shared_rss->queue = queue; + memcpy(queue, rss->queue, queue_size); + shared_rss->ind_tbl->queues = queue; + shared_rss->ind_tbl->queues_n = rss->queue_num; origin = &shared_rss->origin; origin->func = rss->func; origin->level = rss->level; @@ -11131,17 +12799,27 @@ __flow_dv_action_rss_create(struct rte_eth_dev *dev, memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN); origin->key = &shared_rss->key[0]; origin->key_len = MLX5_RSS_HASH_KEY_LEN; - memcpy(shared_rss->queue, rss->queue, queue_size); - origin->queue = shared_rss->queue; + origin->queue = queue; origin->queue_num = rss->queue_num; - if (__flow_dv_action_rss_setup(dev, shared_rss, error)) + if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error)) goto error_rss_init; - shared_action->type = MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS; - return shared_action; + rte_spinlock_init(&shared_rss->action_rss_sl); + __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED); + rte_spinlock_lock(&priv->shared_act_sl); + ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + &priv->rss_shared_actions, idx, shared_rss, next); + rte_spinlock_unlock(&priv->shared_act_sl); + return idx; error_rss_init: - mlx5_free(shared_action); - mlx5_free(queue); - return NULL; + if (shared_rss) { + if (shared_rss->ind_tbl) + mlx5_free(shared_rss->ind_tbl); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + idx); + } + if (queue) + mlx5_free(queue); + return 0; } /** @@ -11150,8 +12828,8 @@ error_rss_init: * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] shared_rss - * The shared RSS action object to be removed. + * @param[in] idx + * The shared RSS action object ID to be removed. * @param[out] error * Perform verbose error reporting if not NULL. Initialized in case of * error only. @@ -11160,31 +12838,48 @@ error_rss_init: * 0 on success, otherwise negative errno value. */ static int -__flow_dv_action_rss_release(struct rte_eth_dev *dev, - struct mlx5_shared_action_rss *shared_rss, - struct rte_flow_error *error) +__flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx, + struct rte_flow_error *error) { - struct rte_flow_shared_action *shared_action = NULL; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss = + mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); uint32_t old_refcnt = 1; - int remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss); + int remaining; + uint16_t *queue = NULL; - if (remaining) { - return rte_flow_error_set(error, ETOOMANYREFS, + if (!shared_rss) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "invalid shared action"); + remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss); + if (remaining) + return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "shared rss hrxq has references"); - } - shared_action = container_of(shared_rss, - struct rte_flow_shared_action, rss); - if (!__atomic_compare_exchange_n(&shared_action->refcnt, &old_refcnt, - 0, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { - return rte_flow_error_set(error, ETOOMANYREFS, + if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt, + 0, 0, __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED)) + return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "shared rss has references"); - } - rte_free(shared_rss->queue); + queue = shared_rss->ind_tbl->queues; + remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true); + if (remaining) + return rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "shared rss indirection table has" + " references"); + mlx5_free(queue); + rte_spinlock_lock(&priv->shared_act_sl); + ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + &priv->rss_shared_actions, idx, shared_rss, next); + rte_spinlock_unlock(&priv->shared_act_sl); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], + idx); return 0; } @@ -11208,31 +12903,39 @@ __flow_dv_action_rss_release(struct rte_eth_dev *dev, * rte_errno is set. */ static struct rte_flow_shared_action * -__flow_dv_action_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, - const struct rte_flow_action *action, - struct rte_flow_error *error) +flow_dv_action_create(struct rte_eth_dev *dev, + const struct rte_flow_shared_action_conf *conf, + const struct rte_flow_action *action, + struct rte_flow_error *err) { - struct rte_flow_shared_action *shared_action = NULL; - struct mlx5_priv *priv = dev->data->dev_private; + uint32_t idx = 0; + uint32_t ret = 0; switch (action->type) { case RTE_FLOW_ACTION_TYPE_RSS: - shared_action = __flow_dv_action_rss_create(dev, conf, - action->conf, - error); + ret = __flow_dv_action_rss_create(dev, conf, action->conf, err); + idx = (MLX5_SHARED_ACTION_TYPE_RSS << + MLX5_SHARED_ACTION_TYPE_OFFSET) | ret; + break; + case RTE_FLOW_ACTION_TYPE_AGE: + ret = flow_dv_translate_create_aso_age(dev, action->conf, err); + idx = (MLX5_SHARED_ACTION_TYPE_AGE << + MLX5_SHARED_ACTION_TYPE_OFFSET) | ret; + if (ret) { + struct mlx5_aso_age_action *aso_age = + flow_aso_age_get_by_idx(dev, ret); + + if (!aso_age->age_params.context) + aso_age->age_params.context = + (void *)(uintptr_t)idx; + } break; default: - rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "action type not supported"); break; } - if (shared_action) { - __atomic_add_fetch(&shared_action->refcnt, 1, - __ATOMIC_RELAXED); - LIST_INSERT_HEAD(&priv->shared_actions, shared_action, next); - } - return shared_action; + return ret ? (struct rte_flow_shared_action *)(uintptr_t)idx : NULL; } /** @@ -11253,27 +12956,34 @@ __flow_dv_action_create(struct rte_eth_dev *dev, * 0 on success, otherwise negative errno value. */ static int -__flow_dv_action_destroy(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - struct rte_flow_error *error) +flow_dv_action_destroy(struct rte_eth_dev *dev, + struct rte_flow_shared_action *action, + struct rte_flow_error *error) { + uint32_t act_idx = (uint32_t)(uintptr_t)action; + uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); int ret; - switch (action->type) { - case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: - ret = __flow_dv_action_rss_release(dev, &action->rss, error); - break; + switch (type) { + case MLX5_SHARED_ACTION_TYPE_RSS: + return __flow_dv_action_rss_release(dev, idx, error); + case MLX5_SHARED_ACTION_TYPE_AGE: + ret = flow_dv_aso_age_release(dev, idx); + if (ret) + /* + * In this case, the last flow has a reference will + * actually release the age action. + */ + DRV_LOG(DEBUG, "Shared age action %" PRIu32 " was" + " released with references %d.", idx, ret); + return 0; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "action type not supported"); } - if (ret) - return ret; - LIST_REMOVE(action, next); - rte_free(action); - return 0; } /** @@ -11281,8 +12991,8 @@ __flow_dv_action_destroy(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] shared_rss - * The shared RSS action object to be updated. + * @param[in] idx + * The shared RSS action object ID to be updated. * @param[in] action_conf * RSS action specification used to modify *shared_rss*. * @param[out] error @@ -11294,18 +13004,26 @@ __flow_dv_action_destroy(struct rte_eth_dev *dev, * @note: currently only support update of RSS queues. */ static int -__flow_dv_action_rss_update(struct rte_eth_dev *dev, - struct mlx5_shared_action_rss *shared_rss, +__flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx, const struct rte_flow_action_rss *action_conf, struct rte_flow_error *error) { - size_t i; - int ret; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_shared_action_rss *shared_rss = + mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); + int ret = 0; void *queue = NULL; - const uint8_t *rss_key; - uint32_t rss_key_len; + uint16_t *queue_old = NULL; uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t); + if (!shared_rss) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "invalid shared action to update"); + if (priv->obj_ops.ind_table_modify == NULL) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "cannot modify indirection table"); queue = mlx5_malloc(MLX5_MEM_ZERO, RTE_ALIGN_CEIL(queue_size, sizeof(void *)), 0, SOCKET_ID_ANY); @@ -11314,42 +13032,24 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); - if (action_conf->key) { - rss_key = action_conf->key; - rss_key_len = action_conf->key_len; + memcpy(queue, action_conf->queue, queue_size); + MLX5_ASSERT(shared_rss->ind_tbl); + rte_spinlock_lock(&shared_rss->action_rss_sl); + queue_old = shared_rss->ind_tbl->queues; + ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl, + queue, action_conf->queue_num, true); + if (ret) { + mlx5_free(queue); + ret = rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "cannot update indirection table"); } else { - rss_key = rss_hash_default_key; - rss_key_len = MLX5_RSS_HASH_KEY_LEN; - } - for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) { - uint32_t hrxq_idx; - uint64_t hash_fields = mlx5_rss_hash_fields[i]; - int tunnel; - - for (tunnel = 0; tunnel < 2; tunnel++) { - hrxq_idx = __flow_dv_action_rss_hrxq_lookup - (shared_rss, hash_fields, tunnel); - MLX5_ASSERT(hrxq_idx); - ret = mlx5_hrxq_modify - (dev, hrxq_idx, - rss_key, rss_key_len, - hash_fields, - action_conf->queue, action_conf->queue_num); - if (ret) { - mlx5_free(queue); - return rte_flow_error_set - (error, rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "cannot update hash queue"); - } - } + mlx5_free(queue_old); + shared_rss->origin.queue = queue; + shared_rss->origin.queue_num = action_conf->queue_num; } - mlx5_free(shared_rss->queue); - shared_rss->queue = queue; - memcpy(shared_rss->queue, action_conf->queue, queue_size); - shared_rss->origin.queue = shared_rss->queue; - shared_rss->origin.queue_num = action_conf->queue_num; - return 0; + rte_spinlock_unlock(&shared_rss->action_rss_sl); + return ret; } /** @@ -11372,22 +13072,57 @@ __flow_dv_action_rss_update(struct rte_eth_dev *dev, * 0 on success, otherwise negative errno value. */ static int -__flow_dv_action_update(struct rte_eth_dev *dev, +flow_dv_action_update(struct rte_eth_dev *dev, struct rte_flow_shared_action *action, const void *action_conf, - struct rte_flow_error *error) + struct rte_flow_error *err) { - switch (action->type) { - case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS: - return __flow_dv_action_rss_update(dev, &action->rss, - action_conf, error); + uint32_t act_idx = (uint32_t)(uintptr_t)action; + uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + + switch (type) { + case MLX5_SHARED_ACTION_TYPE_RSS: + return __flow_dv_action_rss_update(dev, idx, action_conf, err); + default: + return rte_flow_error_set(err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "action type update not supported"); + } +} + +static int +flow_dv_action_query(struct rte_eth_dev *dev, + const struct rte_flow_shared_action *action, void *data, + struct rte_flow_error *error) +{ + struct mlx5_age_param *age_param; + struct rte_flow_query_age *resp; + uint32_t act_idx = (uint32_t)(uintptr_t)action; + uint32_t type = act_idx >> MLX5_SHARED_ACTION_TYPE_OFFSET; + uint32_t idx = act_idx & ((1u << MLX5_SHARED_ACTION_TYPE_OFFSET) - 1); + + switch (type) { + case MLX5_SHARED_ACTION_TYPE_AGE: + age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params; + resp = data; + resp->aged = __atomic_load_n(&age_param->state, + __ATOMIC_RELAXED) == AGE_TMOUT ? + 1 : 0; + resp->sec_since_last_hit_valid = !resp->aged; + if (resp->sec_since_last_hit_valid) + resp->sec_since_last_hit = __atomic_load_n + (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); + return 0; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "action type not supported"); + "action type query not supported"); } } + /** * Query a dv flow rule for its statistics via devx. * @@ -11464,30 +13199,33 @@ flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow, void *data, struct rte_flow_error *error) { struct rte_flow_query_age *resp = data; + struct mlx5_age_param *age_param; - if (flow->counter) { - struct mlx5_age_param *age_param = - flow_dv_counter_idx_get_age(dev, flow->counter); + if (flow->age) { + struct mlx5_aso_age_action *act = + flow_aso_age_get_by_idx(dev, flow->age); + + age_param = &act->age_params; + } else if (flow->counter) { + age_param = flow_dv_counter_idx_get_age(dev, flow->counter); if (!age_param || !age_param->timeout) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot read age data"); - resp->aged = __atomic_load_n(&age_param->state, - __ATOMIC_RELAXED) == - AGE_TMOUT ? 1 : 0; - resp->sec_since_last_hit_valid = !resp->aged; - if (resp->sec_since_last_hit_valid) - resp->sec_since_last_hit = - __atomic_load_n(&age_param->sec_since_last_hit, - __ATOMIC_RELAXED); - return 0; - } - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "age data not available"); + } else { + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "age data not available"); + } + resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) == + AGE_TMOUT ? 1 : 0; + resp->sec_since_last_hit_valid = !resp->aged; + if (resp->sec_since_last_hit_valid) + resp->sec_since_last_hit = __atomic_load_n + (&age_param->sec_since_last_hit, __ATOMIC_RELAXED); + return 0; } /** @@ -11563,9 +13301,9 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->egress.any_matcher)); if (mtd->egress.tbl) - flow_dv_tbl_resource_release(dev, mtd->egress.tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) - flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); if (mtd->ingress.color_matcher) claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->ingress.color_matcher)); @@ -11573,9 +13311,10 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) - flow_dv_tbl_resource_release(dev, mtd->ingress.tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) - flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), + mtd->ingress.sfx_tbl); if (mtd->transfer.color_matcher) claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->transfer.color_matcher)); @@ -11583,9 +13322,10 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, claim_zero(mlx5_flow_os_destroy_flow_matcher (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) - flow_dv_tbl_resource_release(dev, mtd->transfer.tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) - flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), + mtd->transfer.sfx_tbl); if (mtd->drop_actn) claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); @@ -11647,7 +13387,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, /* Create the meter table with METER level. */ dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, egress, transfer, false, NULL, 0, - &error); + 0, &error); if (!dtb->tbl) { DRV_LOG(ERR, "Failed to create meter policer table."); return -1; @@ -11656,7 +13396,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, dtb->sfx_tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_SUFFIX, egress, transfer, false, NULL, 0, - &error); + 0, &error); if (!dtb->sfx_tbl) { DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; @@ -11969,18 +13709,15 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) .match_mask = (void *)&mask, }; void *actions[2] = { 0 }; - struct mlx5_flow_tbl_resource *tbl = NULL, *dest_tbl = NULL; + struct mlx5_flow_tbl_resource *tbl = NULL; struct mlx5_devx_obj *dcs = NULL; void *matcher = NULL; void *flow = NULL; - int i, ret = -1; + int ret = -1; - tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, NULL); + tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, 0, NULL); if (!tbl) goto err; - dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, false, NULL, 0, NULL); - if (!dest_tbl) - goto err; dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); if (!dcs) goto err; @@ -11988,10 +13725,7 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) &actions[0]); if (ret) goto err; - ret = mlx5_flow_os_create_flow_action_dest_flow_tbl - (dest_tbl->obj, &actions[1]); - if (ret) - goto err; + actions[1] = priv->drop_queue.hrxq->action; dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf); ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj, &matcher); @@ -12020,17 +13754,12 @@ err: "support detection"); ret = 0; } - for (i = 0; i < 2; i++) { - if (actions[i]) - claim_zero(mlx5_flow_os_destroy_flow_action - (actions[i])); - } + if (actions[0]) + claim_zero(mlx5_flow_os_destroy_flow_action(actions[0])); if (matcher) claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher)); if (tbl) - flow_dv_tbl_resource_release(dev, tbl); - if (dest_tbl) - flow_dv_tbl_resource_release(dev, dest_tbl); + flow_dv_tbl_resource_release(MLX5_SH(dev), tbl); if (dcs) claim_zero(mlx5_devx_cmd_destroy(dcs)); return ret; @@ -12108,16 +13837,24 @@ flow_get_aged_flows(struct rte_eth_dev *dev, struct mlx5_age_info *age_info; struct mlx5_age_param *age_param; struct mlx5_flow_counter *counter; + struct mlx5_aso_age_action *act; int nb_flows = 0; if (nb_contexts && !context) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "Should assign at least one flow or" - " context to get if nb_contexts != 0"); + NULL, "empty context"); age_info = GET_PORT_AGE_INFO(priv); rte_spinlock_lock(&age_info->aged_sl); + LIST_FOREACH(act, &age_info->aged_aso, next) { + nb_flows++; + if (nb_contexts) { + context[nb_flows - 1] = + act->age_params.context; + if (!(--nb_contexts)) + break; + } + } TAILQ_FOREACH(counter, &age_info->aged_counters, next) { nb_flows++; if (nb_contexts) { @@ -12132,86 +13869,13 @@ flow_get_aged_flows(struct rte_eth_dev *dev, return nb_flows; } -/* - * Mutex-protected thunk to lock-free __flow_dv_translate(). - */ -static int -flow_dv_translate(struct rte_eth_dev *dev, - struct mlx5_flow *dev_flow, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - struct rte_flow_error *error) -{ - int ret; - - flow_dv_shared_lock(dev); - ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error); - flow_dv_shared_unlock(dev); - return ret; -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_apply(). - */ -static int -flow_dv_apply(struct rte_eth_dev *dev, - struct rte_flow *flow, - struct rte_flow_error *error) -{ - int ret; - - flow_dv_shared_lock(dev); - ret = __flow_dv_apply(dev, flow, error); - flow_dv_shared_unlock(dev); - return ret; -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_remove(). - */ -static void -flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) -{ - flow_dv_shared_lock(dev); - __flow_dv_remove(dev, flow); - flow_dv_shared_unlock(dev); -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_destroy(). - */ -static void -flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) -{ - flow_dv_shared_lock(dev); - __flow_dv_destroy(dev, flow); - flow_dv_shared_unlock(dev); -} - /* * Mutex-protected thunk to lock-free flow_dv_counter_alloc(). */ static uint32_t flow_dv_counter_allocate(struct rte_eth_dev *dev) { - uint32_t cnt; - - flow_dv_shared_lock(dev); - cnt = flow_dv_counter_alloc(dev, 0); - flow_dv_shared_unlock(dev); - return cnt; -} - -/* - * Mutex-protected thunk to lock-free flow_dv_counter_release(). - */ -static void -flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt) -{ - flow_dv_shared_lock(dev); - flow_dv_counter_release(dev, cnt); - flow_dv_shared_unlock(dev); + return flow_dv_counter_alloc(dev, 0); } /** @@ -12235,69 +13899,66 @@ static int flow_dv_action_validate(struct rte_eth_dev *dev, const struct rte_flow_shared_action_conf *conf, const struct rte_flow_action *action, - struct rte_flow_error *error) + struct rte_flow_error *err) { + struct mlx5_priv *priv = dev->data->dev_private; + RTE_SET_USED(conf); switch (action->type) { case RTE_FLOW_ACTION_TYPE_RSS: - return mlx5_validate_action_rss(dev, action, error); + /* + * priv->obj_ops is set according to driver capabilities. + * When DevX capabilities are + * sufficient, it is set to devx_obj_ops. + * Otherwise, it is set to ibv_obj_ops. + * ibv_obj_ops doesn't support ind_table_modify operation. + * In this case the shared RSS action can't be used. + */ + if (priv->obj_ops.ind_table_modify == NULL) + return rte_flow_error_set + (err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "shared RSS action not supported"); + return mlx5_validate_action_rss(dev, action, err); + case RTE_FLOW_ACTION_TYPE_AGE: + if (!priv->sh->aso_age_mng) + return rte_flow_error_set(err, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "shared age action not supported"); + return flow_dv_validate_action_age(0, action, dev, err); default: - return rte_flow_error_set(error, ENOTSUP, + return rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "action type not supported"); } } -/* - * Mutex-protected thunk to lock-free __flow_dv_action_create(). - */ -static struct rte_flow_shared_action * -flow_dv_action_create(struct rte_eth_dev *dev, - const struct rte_flow_shared_action_conf *conf, - const struct rte_flow_action *action, - struct rte_flow_error *error) -{ - struct rte_flow_shared_action *shared_action = NULL; - - flow_dv_shared_lock(dev); - shared_action = __flow_dv_action_create(dev, conf, action, error); - flow_dv_shared_unlock(dev); - return shared_action; -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_action_destroy(). - */ -static int -flow_dv_action_destroy(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - struct rte_flow_error *error) -{ - int ret; - - flow_dv_shared_lock(dev); - ret = __flow_dv_action_destroy(dev, action, error); - flow_dv_shared_unlock(dev); - return ret; -} - -/* - * Mutex-protected thunk to lock-free __flow_dv_action_update(). - */ static int -flow_dv_action_update(struct rte_eth_dev *dev, - struct rte_flow_shared_action *action, - const void *action_conf, - struct rte_flow_error *error) +flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags) { - int ret; + struct mlx5_priv *priv = dev->data->dev_private; + int ret = 0; - flow_dv_shared_lock(dev); - ret = __flow_dv_action_update(dev, action, action_conf, - error); - flow_dv_shared_unlock(dev); - return ret; + if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) { + ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain, + flags); + if (ret != 0) + return ret; + } + if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) { + ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags); + if (ret != 0) + return ret; + } + if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) { + ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags); + if (ret != 0) + return ret; + } + return 0; } const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { @@ -12320,6 +13981,8 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .action_create = flow_dv_action_create, .action_destroy = flow_dv_action_destroy, .action_update = flow_dv_action_update, + .action_query = flow_dv_action_query, + .sync_domain = flow_dv_sync_domain, }; #endif /* HAVE_IBV_FLOW_DV_SUPPORT */