X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=1b3160280dfd167f603f706e800f399dd182897f;hb=7b4f1e6bd367855716d84e95a1dd85ac3647a4c8;hp=204e15e74b465c9e0eadb1388370955a34dc0d9a;hpb=fcc8d2f716fd2979dc180b9910ba3fc4b0822bc6;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 204e15e74b..1b3160280d 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -27,12 +27,15 @@ #include #include #include +#include + +#include +#include +#include -#include "mlx5.h" #include "mlx5_defs.h" -#include "mlx5_glue.h" +#include "mlx5.h" #include "mlx5_flow.h" -#include "mlx5_prm.h" #include "mlx5_rxtx.h" #ifdef HAVE_IBV_FLOW_DV_SUPPORT @@ -51,6 +54,8 @@ #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1 #endif +#define MLX5_ENCAPSULATION_DECISION_SIZE (sizeof(struct rte_flow_item_eth) + \ + sizeof(struct rte_flow_item_ipv4)) /* VLAN header definitions */ #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT) @@ -73,6 +78,9 @@ union flow_dv_attr { /** * Initialize flow attributes structure according to flow items' types. * + * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel + * mode. For tunnel mode, the items to be modified are the outermost ones. + * * @param[in] item * Pointer to item specification. * @param[out] attr @@ -84,16 +92,20 @@ flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr) for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { switch (item->type) { case RTE_FLOW_ITEM_TYPE_IPV4: - attr->ipv4 = 1; + if (!attr->ipv6) + attr->ipv4 = 1; break; case RTE_FLOW_ITEM_TYPE_IPV6: - attr->ipv6 = 1; + if (!attr->ipv4) + attr->ipv6 = 1; break; case RTE_FLOW_ITEM_TYPE_UDP: - attr->udp = 1; + if (!attr->tcp) + attr->udp = 1; break; case RTE_FLOW_ITEM_TYPE_TCP: - attr->tcp = 1; + if (!attr->udp) + attr->tcp = 1; break; default: break; @@ -102,6 +114,31 @@ flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr) attr->valid = 1; } +/** + * Convert rte_mtr_color to mlx5 color. + * + * @param[in] rcol + * rte_mtr_color. + * + * @return + * mlx5 color. + */ +static int +rte_col_2_mlx5_col(enum rte_color rcol) +{ + switch (rcol) { + case RTE_COLOR_GREEN: + return MLX5_FLOW_COLOR_GREEN; + case RTE_COLOR_YELLOW: + return MLX5_FLOW_COLOR_YELLOW; + case RTE_COLOR_RED: + return MLX5_FLOW_COLOR_RED; + default: + break; + } + return MLX5_FLOW_COLOR_UNDEFINED; +} + struct field_modify_info { uint32_t size; /* Size of field in protocol header, in bytes. */ uint32_t offset; /* Offset of field in protocol header, in bytes. */ @@ -123,6 +160,7 @@ struct field_modify_info modify_vlan_out_first_vid[] = { }; struct field_modify_info modify_ipv4[] = { + {1, 1, MLX5_MODI_OUT_IP_DSCP}, {1, 8, MLX5_MODI_OUT_IPV4_TTL}, {4, 12, MLX5_MODI_OUT_SIPV4}, {4, 16, MLX5_MODI_OUT_DIPV4}, @@ -130,6 +168,7 @@ struct field_modify_info modify_ipv4[] = { }; struct field_modify_info modify_ipv6[] = { + {1, 0, MLX5_MODI_OUT_IP_DSCP}, {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT}, {4, 8, MLX5_MODI_OUT_SIPV6_127_96}, {4, 12, MLX5_MODI_OUT_SIPV6_95_64}, @@ -329,7 +368,7 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, uint32_t mask; uint32_t data; - if (i >= MLX5_MODIFY_NUM) + if (i >= MLX5_MAX_MODIFY_NUM) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many items to modify"); @@ -370,11 +409,11 @@ flow_dv_convert_modify_action(struct rte_flow_item *item, ++i; ++field; } while (field->size); - resource->actions_num = i; - if (!resource->actions_num) + if (resource->actions_num == i) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "invalid modification flow item"); + resource->actions_num = i; return 0; } @@ -535,7 +574,7 @@ flow_dv_convert_action_modify_vlan_vid struct mlx5_modification_cmd *actions = &resource->actions[i]; struct field_modify_info *field = modify_vlan_out_first_vid; - if (i >= MLX5_MODIFY_NUM) + if (i >= MLX5_MAX_MODIFY_NUM) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many items to modify"); @@ -868,7 +907,7 @@ flow_dv_convert_action_set_reg struct mlx5_modification_cmd *actions = resource->actions; uint32_t i = resource->actions_num; - if (i >= MLX5_MODIFY_NUM) + if (i >= MLX5_MAX_MODIFY_NUM) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many items to modify"); @@ -880,10 +919,6 @@ flow_dv_convert_action_set_reg actions[i].data1 = rte_cpu_to_be_32(conf->data); ++i; resource->actions_num = i; - if (!resource->actions_num) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "invalid modification flow item"); return 0; } @@ -1044,7 +1079,7 @@ flow_dv_convert_action_mark(struct rte_eth_dev *dev, {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */ {0, 0, 0}, }; - enum modify_reg reg; + int reg; if (!mask) return rte_flow_error_set(error, EINVAL, @@ -1054,6 +1089,14 @@ flow_dv_convert_action_mark(struct rte_eth_dev *dev, if (reg < 0) return reg; assert(reg > 0); + if (reg == REG_C_0) { + uint32_t msk_c0 = priv->sh->dv_regc0_mask; + uint32_t shl_c0 = rte_bsf32(msk_c0); + + data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0); + mask = rte_cpu_to_be_32(mask) & msk_c0; + mask = rte_cpu_to_be_32(mask << shl_c0); + } reg_c_x[0].id = reg_to_field[reg]; return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, MLX5_MODIFICATION_TYPE_SET, error); @@ -1078,7 +1121,7 @@ flow_dv_get_metadata_reg(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - enum modify_reg reg = + int reg = mlx5_flow_get_reg_id(dev, attr->transfer ? MLX5_METADATA_FDB : attr->egress ? @@ -1126,7 +1169,7 @@ flow_dv_convert_action_set_meta struct field_modify_info reg_c_x[] = { [1] = {0, 0, 0}, }; - enum modify_reg reg = flow_dv_get_metadata_reg(dev, attr, error); + int reg = flow_dv_get_metadata_reg(dev, attr, error); if (reg < 0) return reg; @@ -1156,6 +1199,82 @@ flow_dv_convert_action_set_meta MLX5_MODIFICATION_TYPE_SET, error); } +/** + * Convert modify-header set IPv4 DSCP action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_ipv4_dscp + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_dscp *conf = + (const struct rte_flow_action_set_dscp *)(action->conf); + struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; + struct rte_flow_item_ipv4 ipv4; + struct rte_flow_item_ipv4 ipv4_mask; + + memset(&ipv4, 0, sizeof(ipv4)); + memset(&ipv4_mask, 0, sizeof(ipv4_mask)); + ipv4.hdr.type_of_service = conf->dscp; + ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2; + item.spec = &ipv4; + item.mask = &ipv4_mask; + return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Convert modify-header set IPv6 DSCP action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_ipv6_dscp + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_dscp *conf = + (const struct rte_flow_action_set_dscp *)(action->conf); + struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; + struct rte_flow_item_ipv6 ipv6; + struct rte_flow_item_ipv6 ipv6_mask; + + memset(&ipv6, 0, sizeof(ipv6)); + memset(&ipv6_mask, 0, sizeof(ipv6_mask)); + /* + * Even though the DSCP bits offset of IPv6 is not byte aligned, + * rdma-core only accept the DSCP bits byte aligned start from + * bit 0 to 5 as to be compatible with IPv4. No need to shift the + * bits in IPv6 case as rdma-core requires byte aligned value. + */ + ipv6.hdr.vtc_flow = conf->dscp; + ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22; + item.spec = &ipv6; + item.mask = &ipv6_mask; + return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + /** * Validate MARK item. * @@ -1253,7 +1372,7 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, struct rte_flow_item_meta nic_mask = { .data = UINT32_MAX }; - enum modify_reg reg; + int reg; int ret; if (!spec) @@ -1431,6 +1550,56 @@ flow_dv_validate_item_port_id(struct rte_eth_dev *dev, return 0; } +/** + * Validate GTP item. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] item + * Item specification. + * @param[in] item_flags + * Bit-fields that holds the items detected until now. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_gtp(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + uint64_t item_flags, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_item_gtp *mask = item->mask; + const struct rte_flow_item_gtp nic_mask = { + .msg_type = 0xff, + .teid = RTE_BE32(0xffffffff), + }; + + if (!priv->config.hca_attr.tunnel_stateless_gtp) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "GTP support is not enabled"); + if (item_flags & MLX5_FLOW_LAYER_TUNNEL) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "multiple tunnel layers not" + " supported"); + if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "no outer UDP layer found"); + if (!mask) + mask = &rte_flow_item_gtp_mask; + return mlx5_flow_item_acceptable + (item, (const uint8_t *)mask, + (const uint8_t *)&nic_mask, + sizeof(struct rte_flow_item_gtp), + error); +} + /** * Validate the pop VLAN action. * @@ -1467,17 +1636,16 @@ flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "pop vlan action is not supported"); - /* - * Check for inconsistencies: - * fail strip_vlan in a flow that matches packets without VLAN tags. - * fail strip_vlan in a flow that matches packets without explicitly a - * matching on VLAN tag ? - */ - if (action_flags & MLX5_FLOW_ACTION_OF_POP_VLAN) + if (attr->egress) return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, - "no support for multiple vlan pop " + "pop vlan action not supported for " + "egress"); + if (action_flags & MLX5_FLOW_VLAN_ACTIONS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "no support for multiple VLAN " "actions"); if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) return rte_flow_error_set(error, ENOTSUP, @@ -1496,14 +1664,10 @@ flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, /** * Get VLAN default info from vlan match info. * - * @param[in] dev - * Pointer to the rte_eth_dev structure. - * @param[in] item + * @param[in] items * the list of item specifications. * @param[out] vlan * pointer VLAN info to fill to. - * @param[out] error - * Pointer to error structure. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. @@ -1555,8 +1719,10 @@ flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, * * @param[in] action_flags * Holds the actions detected until now. + * @param[in] item_flags + * The items found in this flow rule. * @param[in] action - * Pointer to the encap action. + * Pointer to the action structure. * @param[in] attr * Pointer to flow attributes * @param[out] error @@ -1567,32 +1733,29 @@ flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, */ static int flow_dv_validate_action_push_vlan(uint64_t action_flags, - uint64_t item_flags, + uint64_t item_flags __rte_unused, const struct rte_flow_action *action, const struct rte_flow_attr *attr, struct rte_flow_error *error) { const struct rte_flow_action_of_push_vlan *push_vlan = action->conf; + if (attr->ingress) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + NULL, + "push VLAN action not supported for " + "ingress"); if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) && push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "invalid vlan ethertype"); - if (action_flags & - (MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN)) + if (action_flags & MLX5_FLOW_VLAN_ACTIONS) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, action, "no support for multiple VLAN " "actions"); - if (!mlx5_flow_find_action - (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) && - !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "push VLAN needs to match on VLAN in order to " - "get VLAN VID information because there is " - "no followed set VLAN VID action"); if (action_flags & MLX5_FLOW_ACTION_PORT_ID) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, @@ -1609,8 +1772,6 @@ flow_dv_validate_action_push_vlan(uint64_t action_flags, * Holds the actions detected until now. * @param[in] actions * Pointer to the list of actions remaining in the flow rule. - * @param[in] attr - * Pointer to flow attributes * @param[out] error * Pointer to error structure. * @@ -1652,10 +1813,10 @@ flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags, * * @param[in] item_flags * Holds the items detected in this rule. + * @param[in] action_flags + * Holds the actions detected until now. * @param[in] actions * Pointer to the list of actions remaining in the flow rule. - * @param[in] attr - * Pointer to flow attributes * @param[out] error * Pointer to error structure. * @@ -1675,33 +1836,17 @@ flow_dv_validate_action_set_vlan_vid(uint64_t item_flags, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "VLAN VID value is too big"); - /* there is an of_push_vlan action before us */ - if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) { - if (mlx5_flow_find_action(actions + 1, - RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "Multiple VLAN VID modifications are " - "not supported"); - else - return 0; - } - - /* - * Action is on an existing VLAN header: - * Need to verify this is a single modify CID action. - * Rule mast include a match on outer VLAN. - */ + if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) && + !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, action, + "set VLAN VID action must follow push" + " VLAN action or match on VLAN item"); if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, action, "Multiple VLAN VID modifications are " "not supported"); - if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, action, - "match on VLAN is required in order " - "to set VLAN VID"); if (action_flags & MLX5_FLOW_ACTION_PORT_ID) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, @@ -1754,10 +1899,6 @@ flow_dv_validate_action_flag(struct rte_eth_dev *dev, if (ret < 0) return ret; assert(ret > 0); - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and flag in same flow"); if (action_flags & MLX5_FLOW_ACTION_MARK) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -1827,10 +1968,6 @@ flow_dv_validate_action_mark(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_ACTION_CONF, &mark->id, "mark id exceeds the limit"); - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and mark in same flow"); if (action_flags & MLX5_FLOW_ACTION_FLAG) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -1849,7 +1986,7 @@ flow_dv_validate_action_mark(struct rte_eth_dev *dev, * @param[in] dev * Pointer to the rte_eth_dev structure. * @param[in] action - * Pointer to the encap action. + * Pointer to the action structure. * @param[in] action_flags * Holds the actions detected until now. * @param[in] attr @@ -1869,7 +2006,7 @@ flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, { const struct rte_flow_action_set_meta *conf; uint32_t nic_mask = UINT32_MAX; - enum modify_reg reg; + int reg; if (!mlx5_flow_ext_mreg_supported(dev)) return rte_flow_error_set(error, ENOTSUP, @@ -1910,7 +2047,7 @@ flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, * @param[in] dev * Pointer to the rte_eth_dev structure. * @param[in] action - * Pointer to the encap action. + * Pointer to the action structure. * @param[in] action_flags * Holds the actions detected until now. * @param[in] attr @@ -1964,7 +2101,7 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, * Validate count action. * * @param[in] dev - * device otr. + * Pointer to rte_eth_dev structure. * @param[out] error * Pointer to error structure. * @@ -1996,7 +2133,7 @@ notsup_err: * @param[in] action_flags * Holds the actions detected until now. * @param[in] action - * Pointer to the encap action. + * Pointer to the action structure. * @param[in] attr * Pointer to flow attributes * @param[out] error @@ -2015,10 +2152,6 @@ flow_dv_validate_action_l2_encap(uint64_t action_flags, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "configuration cannot be null"); - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and encap in same flow"); if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -2051,10 +2184,6 @@ flow_dv_validate_action_l2_decap(uint64_t action_flags, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and decap in same flow"); if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -2101,10 +2230,6 @@ flow_dv_validate_action_raw_encap(uint64_t action_flags, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, action, "configuration cannot be null"); - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and encap in same flow"); if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -2146,10 +2271,8 @@ flow_dv_validate_action_raw_decap(uint64_t action_flags, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - if (action_flags & MLX5_FLOW_ACTION_DROP) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't drop and decap in same flow"); + const struct rte_flow_action_raw_decap *decap = action->conf; + if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -2160,23 +2283,20 @@ flow_dv_validate_action_raw_decap(uint64_t action_flags, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can only have a single decap" " action in a flow"); - if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "can't have decap action after" - " modify action"); /* decap action is valid on egress only if it is followed by encap */ - if (attr->egress) { - for (; action->type != RTE_FLOW_ACTION_TYPE_END && - action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP; - action++) { - } - if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) - return rte_flow_error_set - (error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, - NULL, "decap action not supported" - " for egress"); + if (attr->egress && decap && + decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, "decap action not supported" + " for egress"); + } else if (decap && decap->size > MLX5_ENCAPSULATION_DECISION_SIZE && + (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) { + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "can't have decap action " + "after modify action"); } return 0; } @@ -2184,7 +2304,7 @@ flow_dv_validate_action_raw_decap(uint64_t action_flags, /** * Find existing encap/decap resource or create and register a new one. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in, out] resource * Pointer to encap/decap resource. @@ -2215,7 +2335,6 @@ flow_dv_encap_decap_resource_register domain = sh->rx_domain; else domain = sh->tx_domain; - /* Lookup a matching resource from cache. */ LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) { if (resource->reformat_type == cache_resource->reformat_type && @@ -2265,10 +2384,10 @@ flow_dv_encap_decap_resource_register /** * Find existing table jump resource or create and register a new one. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. - * @param[in, out] resource - * Pointer to jump table resource. + * @param[in, out] tbl + * Pointer to flow table resource. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -2279,56 +2398,41 @@ flow_dv_encap_decap_resource_register */ static int flow_dv_jump_tbl_resource_register - (struct rte_eth_dev *dev, - struct mlx5_flow_dv_jump_tbl_resource *resource, + (struct rte_eth_dev *dev __rte_unused, + struct mlx5_flow_tbl_resource *tbl, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_ibv_shared *sh = priv->sh; - struct mlx5_flow_dv_jump_tbl_resource *cache_resource; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); + int cnt; - /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &sh->jump_tbl, next) { - if (resource->tbl == cache_resource->tbl) { - DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->dv.jump = cache_resource; - return 0; - } - } - /* Register new jump table resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); - if (!cache_resource) - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate resource memory"); - *cache_resource = *resource; - cache_resource->action = - mlx5_glue->dr_create_flow_action_dest_flow_tbl - (resource->tbl->obj); - if (!cache_resource->action) { - rte_free(cache_resource); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create action"); + assert(tbl); + cnt = rte_atomic32_read(&tbl_data->jump.refcnt); + if (!cnt) { + tbl_data->jump.action = + mlx5_glue->dr_create_flow_action_dest_flow_tbl + (tbl->obj); + if (!tbl_data->jump.action) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create jump action"); + DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", + (void *)&tbl_data->jump, cnt); + } else { + assert(tbl_data->jump.action); + DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++", + (void *)&tbl_data->jump, cnt); } - rte_atomic32_init(&cache_resource->refcnt); - rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next); - dev_flow->dv.jump = cache_resource; - DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); + rte_atomic32_inc(&tbl_data->jump.refcnt); + dev_flow->dv.jump = &tbl_data->jump; return 0; } /** * Find existing table port ID resource or create and register a new one. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in, out] resource * Pointer to port ID action resource. @@ -2370,8 +2474,13 @@ flow_dv_port_id_action_resource_register RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); *cache_resource = *resource; + /* + * Depending on rdma_core version the glue routine calls + * either mlx5dv_dr_action_create_dest_ib_port(domain, ibv_port) + * or mlx5dv_dr_action_create_dest_vport(domain, vport_id). + */ cache_resource->action = - mlx5_glue->dr_create_flow_action_dest_vport + mlx5_glue->dr_create_flow_action_dest_port (priv->sh->fdb_domain, resource->port_id); if (!cache_resource->action) { rte_free(cache_resource); @@ -2392,7 +2501,7 @@ flow_dv_port_id_action_resource_register /** * Find existing push vlan resource or create and register a new one. * - * @param dev[in, out] + * @param [in, out] dev * Pointer to rte_eth_dev structure. * @param[in, out] resource * Pointer to port ID action resource. @@ -2776,8 +2885,6 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, (const struct rte_flow_action_raw_encap *)action->conf; res.size = raw_encap_data->size; memcpy(res.buf, raw_encap_data->data, res.size); - if (flow_dv_zero_encap_udp_csum(res.buf, error)) - return -rte_errno; } else { if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) encap_data = @@ -2791,6 +2898,8 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, &res.size, error)) return -rte_errno; } + if (flow_dv_zero_encap_udp_csum(res.buf, error)) + return -rte_errno; if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -2864,9 +2973,9 @@ flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, encap_data = (const struct rte_flow_action_raw_encap *)action->conf; res.size = encap_data->size; memcpy(res.buf, encap_data->data, res.size); - res.reformat_type = attr->egress ? - MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL : - MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2; + res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ? + MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 : + MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL; if (attr->transfer) res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; else @@ -2884,12 +2993,12 @@ flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to rte_eth_dev structure. - * @param[in] vlan_tag - * the vlan tag to push to the Ethernet header. - * @param[in, out] dev_flow - * Pointer to the mlx5_flow. * @param[in] attr * Pointer to the flow attributes. + * @param[in] vlan + * Pointer to the vlan to push to the Ethernet header. + * @param[in, out] dev_flow + * Pointer to the mlx5_flow. * @param[out] error * Pointer to the error structure. * @@ -3230,8 +3339,6 @@ flow_dv_validate_action_jump(const struct rte_flow_action *action, const struct rte_flow_attr *attributes, bool external, struct rte_flow_error *error) { - uint32_t max_group = attributes->transfer ? MLX5_MAX_TABLES_FDB : - MLX5_MAX_TABLES; uint32_t target_group, table; int ret = 0; @@ -3241,6 +3348,10 @@ flow_dv_validate_action_jump(const struct rte_flow_action *action, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can't have 2 fate actions in" " same flow"); + if (action_flags & MLX5_FLOW_ACTION_METER) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "jump with meter not support"); if (!action->conf) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF, @@ -3251,14 +3362,10 @@ flow_dv_validate_action_jump(const struct rte_flow_action *action, &table, error); if (ret) return ret; - if (table >= max_group) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, - "target group index out of range"); - if (attributes->group >= target_group) + if (attributes->group == target_group) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "target group must be higher than" + "target group must be other than" " the current flow group"); return 0; } @@ -3338,39 +3445,176 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev, * * @param dev * Pointer to rte_eth_dev structure. + * @param flags + * Flags bits to check if root level. * * @return * Max number of modify header actions device can support. */ static unsigned int -flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev) +flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev, uint64_t flags) { /* * There's no way to directly query the max cap. Although it has to be * acquried by iterative trial, it is a safe assumption that more * actions are supported by FW if extensive metadata register is - * supported. + * supported. (Only in the root table) */ - return mlx5_flow_ext_mreg_supported(dev) ? MLX5_MODIFY_NUM : - MLX5_MODIFY_NUM_NO_MREG; + if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL)) + return MLX5_MAX_MODIFY_NUM; + else + return mlx5_flow_ext_mreg_supported(dev) ? + MLX5_ROOT_TBL_MODIFY_NUM : + MLX5_ROOT_TBL_MODIFY_NUM_NO_MREG; } + /** - * Find existing modify-header resource or create and register a new one. + * Validate the meter action. * - * @param dev[in, out] + * @param[in] dev * Pointer to rte_eth_dev structure. - * @param[in, out] resource - * Pointer to modify-header resource. - * @parm[in, out] dev_flow - * Pointer to the dev_flow. + * @param[in] action_flags + * Bit-fields that holds the actions detected until now. + * @param[in] action + * Pointer to the meter action. + * @param[in] attr + * Attributes of flow that includes this action. * @param[out] error - * pointer to error structure. + * Pointer to error structure. * * @return - * 0 on success otherwise -errno and errno is set. + * 0 on success, a negative errno value otherwise and rte_ernno is set. */ static int -flow_dv_modify_hdr_resource_register +mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, + uint64_t action_flags, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const struct rte_flow_action_meter *am = action->conf; + struct mlx5_flow_meter *fm; + + if (!am) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "meter action conf is NULL"); + + if (action_flags & MLX5_FLOW_ACTION_METER) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "meter chaining not support"); + if (action_flags & MLX5_FLOW_ACTION_JUMP) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "meter with jump not support"); + if (!priv->mtr_en) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "meter action not supported"); + fm = mlx5_flow_meter_find(priv, am->mtr_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Meter not found"); + if (fm->ref_cnt && (!(fm->attr.transfer == attr->transfer || + (!fm->attr.ingress && !attr->ingress && attr->egress) || + (!fm->attr.egress && !attr->egress && attr->ingress)))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Flow attributes are either invalid " + "or have a conflict with current " + "meter attributes"); + return 0; +} + +/** + * Validate the modify-header IPv4 DSCP actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no ipv4 item in pattern"); + } + return ret; +} + +/** + * Validate the modify-header IPv6 DSCP actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no ipv6 item in pattern"); + } + return ret; +} + +/** + * Find existing modify-header resource or create and register a new one. + * + * @param dev[in, out] + * Pointer to rte_eth_dev structure. + * @param[in, out] resource + * Pointer to modify-header resource. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ +static int +flow_dv_modify_hdr_resource_register (struct rte_eth_dev *dev, struct mlx5_flow_dv_modify_hdr_resource *resource, struct mlx5_flow *dev_flow, @@ -3380,8 +3624,12 @@ flow_dv_modify_hdr_resource_register struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_modify_hdr_resource *cache_resource; struct mlx5dv_dr_domain *ns; + uint32_t actions_len; - if (resource->actions_num > flow_dv_modify_hdr_action_max(dev)) + resource->flags = + dev_flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; + if (resource->actions_num > flow_dv_modify_hdr_action_max(dev, + resource->flags)) return rte_flow_error_set(error, EOVERFLOW, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "too many modify header items"); @@ -3391,17 +3639,15 @@ flow_dv_modify_hdr_resource_register ns = sh->tx_domain; else ns = sh->rx_domain; - resource->flags = - dev_flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; /* Lookup a matching resource from cache. */ + actions_len = resource->actions_num * sizeof(resource->actions[0]); LIST_FOREACH(cache_resource, &sh->modify_cmds, next) { if (resource->ft_type == cache_resource->ft_type && resource->actions_num == cache_resource->actions_num && resource->flags == cache_resource->flags && !memcmp((const void *)resource->actions, (const void *)cache_resource->actions, - (resource->actions_num * - sizeof(resource->actions[0])))) { + actions_len)) { DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); @@ -3411,18 +3657,18 @@ flow_dv_modify_hdr_resource_register } } /* Register new modify-header resource. */ - cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + cache_resource = rte_calloc(__func__, 1, + sizeof(*cache_resource) + actions_len, 0); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); *cache_resource = *resource; + rte_memcpy(cache_resource->actions, resource->actions, actions_len); cache_resource->verbs_action = mlx5_glue->dv_create_flow_action_modify_header - (sh->ctx, cache_resource->ft_type, - ns, cache_resource->flags, - cache_resource->actions_num * - sizeof(cache_resource->actions[0]), + (sh->ctx, cache_resource->ft_type, ns, + cache_resource->flags, actions_len, (uint64_t *)cache_resource->actions); if (!cache_resource->verbs_action) { rte_free(cache_resource); @@ -4086,8 +4332,6 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, NULL, "groups are not supported"); #else - uint32_t max_group = attributes->transfer ? MLX5_MAX_TABLES_FDB : - MLX5_MAX_TABLES; uint32_t table; int ret; @@ -4096,10 +4340,6 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, &table, error); if (ret) return ret; - if (table >= max_group) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, - "group index out of range"); #endif if (attributes->priority != MLX5_FLOW_PRIO_RSVD && attributes->priority >= priority_max) @@ -4164,6 +4404,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, uint8_t next_protocol = 0xff; uint16_t ether_type = 0; int actions_n = 0; + uint8_t item_ipv6_proto = 0; const struct rte_flow_item *gre_item = NULL; struct rte_flow_item_tcp nic_tcp_mask = { .hdr = { @@ -4271,6 +4512,9 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (items->mask != NULL && ((const struct rte_flow_item_ipv6 *) items->mask)->hdr.proto) { + item_ipv6_proto = + ((const struct rte_flow_item_ipv6 *) + items->spec)->hdr.proto; next_protocol = ((const struct rte_flow_item_ipv6 *) items->spec)->hdr.proto; @@ -4346,7 +4590,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; - last_item = MLX5_FLOW_LAYER_VXLAN_GPE; + last_item = MLX5_FLOW_LAYER_GENEVE; break; case RTE_FLOW_ITEM_TYPE_MPLS: ret = mlx5_flow_validate_item_mpls(dev, items, @@ -4397,6 +4641,13 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case MLX5_RTE_FLOW_ITEM_TYPE_TAG: case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: break; + case RTE_FLOW_ITEM_TYPE_GTP: + ret = flow_dv_validate_item_gtp(dev, items, item_flags, + error); + if (ret < 0) + return ret; + last_item = MLX5_FLOW_LAYER_GTP; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, @@ -4636,6 +4887,12 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; + if (item_ipv6_proto == IPPROTO_ICMPV6) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "Can't change header " + "with ICMPv6 proto"); /* Count all modify-header actions as one action. */ if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) ++actions_n; @@ -4721,8 +4978,45 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, MLX5_FLOW_ACTION_DEC_TCP_ACK; break; case MLX5_RTE_FLOW_ACTION_TYPE_TAG: + case MLX5_RTE_FLOW_ACTION_TYPE_MARK: case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: break; + case RTE_FLOW_ACTION_TYPE_METER: + ret = mlx5_flow_validate_action_meter(dev, + action_flags, + actions, attr, + error); + if (ret < 0) + return ret; + action_flags |= MLX5_FLOW_ACTION_METER; + ++actions_n; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: + ret = flow_dv_validate_action_modify_ipv4_dscp + (action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: + ret = flow_dv_validate_action_modify_ipv6_dscp + (action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -4730,13 +5024,18 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, "action not supported"); } } - if ((action_flags & MLX5_FLOW_LAYER_TUNNEL) && - (action_flags & MLX5_FLOW_VLAN_ACTIONS)) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ACTION, - actions, - "can't have vxlan and vlan" - " actions in the same rule"); + /* + * Validate the drop action mutual exclusion with other actions. + * Drop action is mutually-exclusive with any other action, except for + * Count action. + */ + if ((action_flags & MLX5_FLOW_ACTION_DROP) && + (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "Drop action is mutually-exclusive " + "with any other action, except for " + "Count action"); /* Eswitch has few restrictions on using items and actions */ if (attr->transfer) { if (!mlx5_flow_ext_mreg_supported(dev) && @@ -5659,6 +5958,7 @@ flow_dv_match_meta_reg(void *matcher, void *key, MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); + uint32_t temp; data &= mask; switch (reg_type) { @@ -5671,8 +5971,18 @@ flow_dv_match_meta_reg(void *matcher, void *key, MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data); break; case REG_C_0: - MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, mask); - MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, data); + /* + * The metadata register C0 field might be divided into + * source vport index and META item value, we should set + * this field according to specified mask, not as whole one. + */ + temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0); + temp |= mask; + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp); + temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0); + temp &= ~mask; + temp |= data; + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp); break; case REG_C_1: MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask); @@ -5742,6 +6052,15 @@ flow_dv_translate_item_mark(struct rte_eth_dev *dev, /* Get the metadata register index for the mark. */ reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL); assert(reg > 0); + if (reg == REG_C_0) { + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t msk_c0 = priv->sh->dv_regc0_mask; + uint32_t shl_c0 = rte_bsf32(msk_c0); + + mask &= msk_c0; + mask <<= shl_c0; + value <<= shl_c0; + } flow_dv_match_meta_reg(matcher, key, reg, value, mask); } } @@ -5774,7 +6093,7 @@ flow_dv_translate_item_meta(struct rte_eth_dev *dev, meta_m = &rte_flow_item_meta_mask; meta_v = (const void *)item->spec; if (meta_v) { - enum modify_reg reg; + int reg; uint32_t value = meta_v->data; uint32_t mask = meta_m->data; @@ -5792,8 +6111,12 @@ flow_dv_translate_item_meta(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; uint32_t msk_c0 = priv->sh->dv_regc0_mask; uint32_t shl_c0 = rte_bsf32(msk_c0); +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask); - msk_c0 = rte_cpu_to_be_32(msk_c0); + value >>= shr_c0; + mask >>= shr_c0; +#endif value <<= shl_c0; mask <<= shl_c0; assert(msk_c0); @@ -5823,6 +6146,8 @@ flow_dv_translate_item_meta_vport(void *matcher, void *key, /** * Add tag item to matcher * + * @param[in] dev + * The devich to configure through. * @param[in, out] matcher * Flow matcher. * @param[in, out] key @@ -5831,15 +6156,27 @@ flow_dv_translate_item_meta_vport(void *matcher, void *key, * Flow pattern to translate. */ static void -flow_dv_translate_mlx5_item_tag(void *matcher, void *key, +flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev, + void *matcher, void *key, const struct rte_flow_item *item) { const struct mlx5_rte_flow_item_tag *tag_v = item->spec; const struct mlx5_rte_flow_item_tag *tag_m = item->mask; + uint32_t mask, value; assert(tag_v); - flow_dv_match_meta_reg(matcher, key, tag_v->id, tag_v->data, - tag_m ? tag_m->data : UINT32_MAX); + value = tag_v->data; + mask = tag_m ? tag_m->data : UINT32_MAX; + if (tag_v->id == REG_C_0) { + struct mlx5_priv *priv = dev->data->dev_private; + uint32_t msk_c0 = priv->sh->dv_regc0_mask; + uint32_t shl_c0 = rte_bsf32(msk_c0); + + mask &= msk_c0; + mask <<= shl_c0; + value <<= shl_c0; + } + flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask); } /** @@ -6030,6 +6367,57 @@ flow_dv_translate_item_icmp(void *matcher, void *key, icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code); } +/** + * Add GTP item to matcher and to the value. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[in] inner + * Item is inner pattern. + */ +static void +flow_dv_translate_item_gtp(void *matcher, void *key, + const struct rte_flow_item *item, int inner) +{ + const struct rte_flow_item_gtp *gtp_m = item->mask; + const struct rte_flow_item_gtp *gtp_v = item->spec; + void *headers_m; + void *headers_v; + void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, + misc_parameters_3); + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + uint16_t dport = RTE_GTPU_UDP_PORT; + + if (inner) { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + inner_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); + } else { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + outer_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); + } + if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { + MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); + } + if (!gtp_v) + return; + if (!gtp_m) + gtp_m = &rte_flow_item_gtp_mask; + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type); + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type, + gtp_v->msg_type & gtp_m->msg_type); + MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid, + rte_be_to_cpu_32(gtp_m->teid)); + MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid, + rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid)); +} + static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; #define HEADER_IS_ZERO(match_criteria, headers) \ @@ -6072,7 +6460,7 @@ flow_dv_matcher_enable(uint32_t *match_criteria) /** * Get a flow table. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in] table_id * Table id to use. @@ -6095,47 +6483,76 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_tbl_resource *tbl; - -#ifdef HAVE_MLX5DV_DR - if (transfer) { - tbl = &sh->fdb_tbl[table_id]; - if (!tbl->obj) - tbl->obj = mlx5_glue->dr_create_flow_tbl - (sh->fdb_domain, table_id); - } else if (egress) { - tbl = &sh->tx_tbl[table_id]; - if (!tbl->obj) - tbl->obj = mlx5_glue->dr_create_flow_tbl - (sh->tx_domain, table_id); - } else { - tbl = &sh->rx_tbl[table_id]; - if (!tbl->obj) - tbl->obj = mlx5_glue->dr_create_flow_tbl - (sh->rx_domain, table_id); + union mlx5_flow_tbl_key table_key = { + { + .table_id = table_id, + .reserved = 0, + .domain = !!transfer, + .direction = !!egress, + } + }; + struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, + table_key.v64); + struct mlx5_flow_tbl_data_entry *tbl_data; + int ret; + void *domain; + + if (pos) { + tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, + entry); + tbl = &tbl_data->tbl; + rte_atomic32_inc(&tbl->refcnt); + return tbl; } - if (!tbl->obj) { + tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0); + if (!tbl_data) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create table"); + NULL, + "cannot allocate flow table data entry"); return NULL; } - rte_atomic32_inc(&tbl->refcnt); - return tbl; -#else - (void)error; - (void)tbl; + tbl = &tbl_data->tbl; + pos = &tbl_data->entry; if (transfer) - return &sh->fdb_tbl[table_id]; + domain = sh->fdb_domain; else if (egress) - return &sh->tx_tbl[table_id]; + domain = sh->tx_domain; else - return &sh->rx_tbl[table_id]; -#endif + domain = sh->rx_domain; + tbl->obj = mlx5_glue->dr_create_flow_tbl(domain, table_id); + if (!tbl->obj) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create flow table object"); + rte_free(tbl_data); + return NULL; + } + /* + * No multi-threads now, but still better to initialize the reference + * count before insert it into the hash list. + */ + rte_atomic32_init(&tbl->refcnt); + /* Jump action reference count is initialized here. */ + rte_atomic32_init(&tbl_data->jump.refcnt); + pos->key = table_key.v64; + ret = mlx5_hlist_insert(sh->flow_tbls, pos); + if (ret < 0) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot insert flow table data entry"); + mlx5_glue->dr_destroy_flow_tbl(tbl->obj); + rte_free(tbl_data); + } + rte_atomic32_inc(&tbl->refcnt); + return tbl; } /** * Release a flow table. * + * @param[in] dev + * Pointer to rte_eth_dev structure. * @param[in] tbl * Table resource to be released. * @@ -6143,13 +6560,24 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, * Returns 0 if table was released, else return 1; */ static int -flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl) +flow_dv_tbl_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow_tbl_resource *tbl) { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!tbl) return 0; if (rte_atomic32_dec_and_test(&tbl->refcnt)) { + struct mlx5_hlist_entry *pos = &tbl_data->entry; + mlx5_glue->dr_destroy_flow_tbl(tbl->obj); tbl->obj = NULL; + /* remove the entry from the hash list and free memory. */ + mlx5_hlist_remove(sh->flow_tbls, pos); + rte_free(tbl_data); return 0; } return 1; @@ -6158,10 +6586,12 @@ flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl) /** * Register the flow matcher. * - * @param dev[in, out] + * @param[in, out] dev * Pointer to rte_eth_dev structure. * @param[in, out] matcher * Pointer to flow matcher. + * @param[in, out] key + * Pointer to flow table key. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -6173,6 +6603,7 @@ flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl) static int flow_dv_matcher_register(struct rte_eth_dev *dev, struct mlx5_flow_dv_matcher *matcher, + union mlx5_flow_tbl_key *key, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { @@ -6183,69 +6614,73 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, .type = IBV_FLOW_ATTR_NORMAL, .match_mask = (void *)&matcher->mask, }; - struct mlx5_flow_tbl_resource *tbl = NULL; + struct mlx5_flow_tbl_resource *tbl; + struct mlx5_flow_tbl_data_entry *tbl_data; + tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, + key->domain, error); + if (!tbl) + return -rte_errno; /* No need to refill the error info */ + tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); /* Lookup from cache. */ - LIST_FOREACH(cache_matcher, &sh->matchers, next) { + LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) { if (matcher->crc == cache_matcher->crc && matcher->priority == cache_matcher->priority && - matcher->egress == cache_matcher->egress && - matcher->group == cache_matcher->group && - matcher->transfer == cache_matcher->transfer && !memcmp((const void *)matcher->mask.buf, (const void *)cache_matcher->mask.buf, cache_matcher->mask.size)) { DRV_LOG(DEBUG, - "priority %hd use %s matcher %p: refcnt %d++", + "%s group %u priority %hd use %s " + "matcher %p: refcnt %d++", + key->domain ? "FDB" : "NIC", key->table_id, cache_matcher->priority, - cache_matcher->egress ? "tx" : "rx", + key->direction ? "tx" : "rx", (void *)cache_matcher, rte_atomic32_read(&cache_matcher->refcnt)); rte_atomic32_inc(&cache_matcher->refcnt); dev_flow->dv.matcher = cache_matcher; + /* old matcher should not make the table ref++. */ + flow_dv_tbl_resource_release(dev, tbl); return 0; } } /* Register new matcher. */ cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0); - if (!cache_matcher) + if (!cache_matcher) { + flow_dv_tbl_resource_release(dev, tbl); return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate matcher memory"); - tbl = flow_dv_tbl_resource_get(dev, matcher->group, - matcher->egress, matcher->transfer, - error); - if (!tbl) { - rte_free(cache_matcher); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create table"); } *cache_matcher = *matcher; dv_attr.match_criteria_enable = flow_dv_matcher_enable(cache_matcher->mask.buf); dv_attr.priority = matcher->priority; - if (matcher->egress) + if (key->direction) dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; cache_matcher->matcher_object = mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj); if (!cache_matcher->matcher_object) { rte_free(cache_matcher); #ifdef HAVE_MLX5DV_DR - flow_dv_tbl_resource_release(tbl); + flow_dv_tbl_resource_release(dev, tbl); #endif return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create matcher"); } + /* Save the table information */ + cache_matcher->tbl = tbl; + rte_atomic32_init(&cache_matcher->refcnt); + /* only matcher ref++, table ref++ already done above in get API. */ rte_atomic32_inc(&cache_matcher->refcnt); - LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next); + LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); dev_flow->dv.matcher = cache_matcher; - DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d", + DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", + key->domain ? "FDB" : "NIC", key->table_id, cache_matcher->priority, - cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher, + key->direction ? "tx" : "rx", (void *)cache_matcher, rte_atomic32_read(&cache_matcher->refcnt)); - rte_atomic32_inc(&tbl->refcnt); return 0; } @@ -6254,8 +6689,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, * * @param dev[in, out] * Pointer to rte_eth_dev structure. - * @param[in, out] resource - * Pointer to tag resource. + * @param[in, out] tag_be24 + * Tag value in big endian then R-shift 8. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -6267,34 +6702,35 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, static int flow_dv_tag_resource_register (struct rte_eth_dev *dev, - struct mlx5_flow_dv_tag_resource *resource, + uint32_t tag_be24, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_tag_resource *cache_resource; + struct mlx5_hlist_entry *entry; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &sh->tags, next) { - if (resource->tag == cache_resource->tag) { - DRV_LOG(DEBUG, "tag resource %p: refcnt %d++", - (void *)cache_resource, - rte_atomic32_read(&cache_resource->refcnt)); - rte_atomic32_inc(&cache_resource->refcnt); - dev_flow->dv.tag_resource = cache_resource; - return 0; - } + entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24); + if (entry) { + cache_resource = container_of + (entry, struct mlx5_flow_dv_tag_resource, entry); + rte_atomic32_inc(&cache_resource->refcnt); + dev_flow->dv.tag_resource = cache_resource; + DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + return 0; } - /* Register new resource. */ + /* Register new resource. */ cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); if (!cache_resource) return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate resource memory"); - *cache_resource = *resource; - cache_resource->action = mlx5_glue->dv_create_flow_action_tag - (resource->tag); + cache_resource->entry.key = (uint64_t)tag_be24; + cache_resource->action = mlx5_glue->dv_create_flow_action_tag(tag_be24); if (!cache_resource->action) { rte_free(cache_resource); return rte_flow_error_set(error, ENOMEM, @@ -6303,9 +6739,15 @@ flow_dv_tag_resource_register } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&sh->tags, cache_resource, next); + if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) { + mlx5_glue->destroy_flow_action(cache_resource->action); + rte_free(cache_resource); + return rte_flow_error_set(error, EEXIST, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot insert tag"); + } dev_flow->dv.tag_resource = cache_resource; - DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++", + DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", (void *)cache_resource, rte_atomic32_read(&cache_resource->refcnt)); return 0; @@ -6326,13 +6768,16 @@ static int flow_dv_tag_release(struct rte_eth_dev *dev, struct mlx5_flow_dv_tag_resource *tag) { + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + assert(tag); DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", dev->data->port_id, (void *)tag, rte_atomic32_read(&tag->refcnt)); if (rte_atomic32_dec_and_test(&tag->refcnt)) { claim_zero(mlx5_glue->destroy_flow_action(tag->action)); - LIST_REMOVE(tag, next); + mlx5_hlist_remove(sh->tag_table, &tag->entry); DRV_LOG(DEBUG, "port %u tag %p: removed", dev->data->port_id, (void *)tag); rte_free(tag); @@ -6374,10 +6819,20 @@ flow_dv_translate_action_port_id(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "No eswitch info was found for port"); - if (priv->vport_meta_mask) - *dst_port_id = priv->vport_meta_tag; - else - *dst_port_id = priv->vport_id; +#ifdef HAVE_MLX5DV_DR_DEVX_PORT + /* + * This parameter is transferred to + * mlx5dv_dr_action_create_dest_ib_port(). + */ + *dst_port_id = priv->ibv_port; +#else + /* + * Legacy mode, no LAG configurations is supported. + * This parameter is transferred to + * mlx5dv_dr_action_create_dest_vport(). + */ + *dst_port_id = priv->vport_id; +#endif return 0; } @@ -6426,6 +6881,75 @@ flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev, mlx5_txq_release(dev, queue_v->queue); } +/** + * Set the hash fields according to the @p flow information. + * + * @param[in] dev_flow + * Pointer to the mlx5_flow. + */ +static void +flow_dv_hashfields_set(struct mlx5_flow *dev_flow) +{ + struct rte_flow *flow = dev_flow->flow; + uint64_t items = dev_flow->layers; + int rss_inner = 0; + uint64_t rss_types = rte_eth_rss_hf_refine(flow->rss.types); + + dev_flow->hash_fields = 0; +#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT + if (flow->rss.level >= 2) { + dev_flow->hash_fields |= IBV_RX_HASH_INNER; + rss_inner = 1; + } +#endif + if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) || + (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) { + if (rss_types & MLX5_IPV4_LAYER_TYPES) { + if (rss_types & ETH_RSS_L3_SRC_ONLY) + dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4; + else if (rss_types & ETH_RSS_L3_DST_ONLY) + dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4; + else + dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH; + } + } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) || + (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) { + if (rss_types & MLX5_IPV6_LAYER_TYPES) { + if (rss_types & ETH_RSS_L3_SRC_ONLY) + dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6; + else if (rss_types & ETH_RSS_L3_DST_ONLY) + dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6; + else + dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH; + } + } + if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) || + (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) { + if (rss_types & ETH_RSS_UDP) { + if (rss_types & ETH_RSS_L4_SRC_ONLY) + dev_flow->hash_fields |= + IBV_RX_HASH_SRC_PORT_UDP; + else if (rss_types & ETH_RSS_L4_DST_ONLY) + dev_flow->hash_fields |= + IBV_RX_HASH_DST_PORT_UDP; + else + dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH; + } + } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) || + (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) { + if (rss_types & ETH_RSS_TCP) { + if (rss_types & ETH_RSS_L4_SRC_ONLY) + dev_flow->hash_fields |= + IBV_RX_HASH_SRC_PORT_TCP; + else if (rss_types & ETH_RSS_L4_DST_ONLY) + dev_flow->hash_fields |= + IBV_RX_HASH_DST_PORT_TCP; + else + dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH; + } + } +} + /** * Fill the flow with DV spec, lock free * (mutex should be acquired by caller). @@ -6468,12 +6992,16 @@ __flow_dv_translate(struct rte_eth_dev *dev, }; int actions_n = 0; bool actions_end = false; - struct mlx5_flow_dv_modify_hdr_resource mhdr_res = { - .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : - MLX5DV_FLOW_TABLE_TYPE_NIC_RX - }; + union { + struct mlx5_flow_dv_modify_hdr_resource res; + uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) + + sizeof(struct mlx5_modification_cmd) * + (MLX5_MAX_MODIFY_NUM + 1)]; + } mhdr_dummy; + struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; union flow_dv_attr flow_attr = { .attr = 0 }; - struct mlx5_flow_dv_tag_resource tag_resource; + uint32_t tag_be; + union mlx5_flow_tbl_key tbl_key; uint32_t modify_action_position = UINT32_MAX; void *match_mask = matcher.mask.buf; void *match_value = dev_flow->dv.value.buf; @@ -6482,15 +7010,19 @@ __flow_dv_translate(struct rte_eth_dev *dev, uint32_t table; int ret = 0; + mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : + MLX5DV_FLOW_TABLE_TYPE_NIC_RX; ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group, &table, error); if (ret) return ret; dev_flow->group = table; if (attr->transfer) - mhdr_res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; + mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; if (priority == MLX5_FLOW_PRIO_RSVD) priority = dev_conf->flow_prio - 1; + /* number of actions must be set to 0 in case of dirty stack. */ + mhdr_res->actions_num = 0; for (; !actions_end ; actions++) { const struct rte_flow_action_queue *queue; const struct rte_flow_action_rss *rss; @@ -6498,7 +7030,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_count *count = action->conf; const uint8_t *rss_key; const struct rte_flow_action_jump *jump_data; - struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource; + const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; @@ -6528,18 +7060,17 @@ __flow_dv_translate(struct rte_eth_dev *dev, }; if (flow_dv_convert_action_mark(dev, &mark, - &mhdr_res, + mhdr_res, error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_MARK_EXT; break; } - tag_resource.tag = - mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); + tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); if (!dev_flow->dv.tag_resource) if (flow_dv_tag_resource_register - (dev, &tag_resource, dev_flow, error)) - return errno; + (dev, tag_be, dev_flow, error)) + return -rte_errno; dev_flow->dv.actions[actions_n++] = dev_flow->dv.tag_resource->action; break; @@ -6551,26 +7082,28 @@ __flow_dv_translate(struct rte_eth_dev *dev, actions->conf; if (flow_dv_convert_action_mark(dev, mark, - &mhdr_res, + mhdr_res, error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_MARK_EXT; break; } + /* Fall-through */ + case MLX5_RTE_FLOW_ACTION_TYPE_MARK: /* Legacy (non-extensive) MARK action. */ - tag_resource.tag = mlx5_flow_mark_set + tag_be = mlx5_flow_mark_set (((const struct rte_flow_action_mark *) (actions->conf))->id); if (!dev_flow->dv.tag_resource) if (flow_dv_tag_resource_register - (dev, &tag_resource, dev_flow, error)) - return errno; + (dev, tag_be, dev_flow, error)) + return -rte_errno; dev_flow->dv.actions[actions_n++] = dev_flow->dv.tag_resource->action; break; case RTE_FLOW_ACTION_TYPE_SET_META: if (flow_dv_convert_action_set_meta - (dev, &mhdr_res, attr, + (dev, mhdr_res, attr, (const struct rte_flow_action_set_meta *) actions->conf, error)) return -rte_errno; @@ -6578,7 +7111,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, break; case RTE_FLOW_ACTION_TYPE_SET_TAG: if (flow_dv_convert_action_set_tag - (dev, &mhdr_res, + (dev, mhdr_res, (const struct rte_flow_action_set_tag *) actions->conf, error)) return -rte_errno; @@ -6678,7 +7211,7 @@ cnt_err: mlx5_update_vlan_vid_pcp(actions, &vlan); /* If no VLAN push - this is a modify header action */ if (flow_dv_convert_action_modify_vlan_vid - (&mhdr_res, actions, error)) + (mhdr_res, actions, error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; break; @@ -6761,10 +7294,9 @@ cnt_err: RTE_FLOW_ERROR_TYPE_ACTION, NULL, "cannot create jump action."); - jump_tbl_resource.tbl = tbl; if (flow_dv_jump_tbl_resource_register - (dev, &jump_tbl_resource, dev_flow, error)) { - flow_dv_tbl_resource_release(tbl); + (dev, tbl, dev_flow, error)) { + flow_dv_tbl_resource_release(dev, tbl); return rte_flow_error_set (error, errno, RTE_FLOW_ERROR_TYPE_ACTION, @@ -6778,7 +7310,7 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: if (flow_dv_convert_action_modify_mac - (&mhdr_res, actions, error)) + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? @@ -6788,7 +7320,7 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: if (flow_dv_convert_action_modify_ipv4 - (&mhdr_res, actions, error)) + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? @@ -6798,7 +7330,7 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: if (flow_dv_convert_action_modify_ipv6 - (&mhdr_res, actions, error)) + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? @@ -6808,7 +7340,7 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: case RTE_FLOW_ACTION_TYPE_SET_TP_DST: if (flow_dv_convert_action_modify_tp - (&mhdr_res, actions, items, + (mhdr_res, actions, items, &flow_attr, error)) return -rte_errno; action_flags |= actions->type == @@ -6818,13 +7350,13 @@ cnt_err: break; case RTE_FLOW_ACTION_TYPE_DEC_TTL: if (flow_dv_convert_action_modify_dec_ttl - (&mhdr_res, items, &flow_attr, error)) + (mhdr_res, items, &flow_attr, error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_DEC_TTL; break; case RTE_FLOW_ACTION_TYPE_SET_TTL: if (flow_dv_convert_action_modify_ttl - (&mhdr_res, actions, items, + (mhdr_res, actions, items, &flow_attr, error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_SET_TTL; @@ -6832,7 +7364,7 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: if (flow_dv_convert_action_modify_tcp_seq - (&mhdr_res, actions, error)) + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? @@ -6843,7 +7375,7 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: if (flow_dv_convert_action_modify_tcp_ack - (&mhdr_res, actions, error)) + (mhdr_res, actions, error)) return -rte_errno; action_flags |= actions->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? @@ -6852,22 +7384,53 @@ cnt_err: break; case MLX5_RTE_FLOW_ACTION_TYPE_TAG: if (flow_dv_convert_action_set_reg - (&mhdr_res, actions, error)) + (mhdr_res, actions, error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_SET_TAG; break; case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: if (flow_dv_convert_action_copy_mreg - (dev, &mhdr_res, actions, error)) + (dev, mhdr_res, actions, error)) return -rte_errno; action_flags |= MLX5_FLOW_ACTION_SET_TAG; break; + case RTE_FLOW_ACTION_TYPE_METER: + mtr = actions->conf; + if (!flow->meter) { + flow->meter = mlx5_flow_meter_attach(priv, + mtr->mtr_id, attr, + error); + if (!flow->meter) + return rte_flow_error_set(error, + rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "meter not found " + "or invalid parameters"); + } + /* Set the meter action. */ + dev_flow->dv.actions[actions_n++] = + flow->meter->mfts->meter_action; + action_flags |= MLX5_FLOW_ACTION_METER; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: + if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res, + actions, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: + if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res, + actions, error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; + break; case RTE_FLOW_ACTION_TYPE_END: actions_end = true; - if (mhdr_res.actions_num) { + if (mhdr_res->actions_num) { /* create modify action if needed. */ if (flow_dv_modify_hdr_resource_register - (dev, &mhdr_res, dev_flow, error)) + (dev, mhdr_res, dev_flow, error)) return -rte_errno; dev_flow->dv.actions[modify_action_position] = dev_flow->dv.modify_hdr->verbs_action; @@ -6876,7 +7439,7 @@ cnt_err: default: break; } - if (mhdr_res.actions_num && + if (mhdr_res->actions_num && modify_action_position == UINT32_MAX) modify_action_position = actions_n++; } @@ -6916,11 +7479,6 @@ cnt_err: items, tunnel, dev_flow->group); matcher.priority = MLX5_PRIORITY_MAP_L3; - dev_flow->hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, - MLX5_IPV4_LAYER_TYPES, - MLX5_IPV4_IBV_RX_HASH); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : MLX5_FLOW_LAYER_OUTER_L3_IPV4; if (items->mask != NULL && @@ -6944,11 +7502,6 @@ cnt_err: items, tunnel, dev_flow->group); matcher.priority = MLX5_PRIORITY_MAP_L3; - dev_flow->hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, - MLX5_IPV6_LAYER_TYPES, - MLX5_IPV6_IBV_RX_HASH); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : MLX5_FLOW_LAYER_OUTER_L3_IPV6; if (items->mask != NULL && @@ -6969,11 +7522,6 @@ cnt_err: flow_dv_translate_item_tcp(match_mask, match_value, items, tunnel); matcher.priority = MLX5_PRIORITY_MAP_L4; - dev_flow->hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, ETH_RSS_TCP, - IBV_RX_HASH_SRC_PORT_TCP | - IBV_RX_HASH_DST_PORT_TCP); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : MLX5_FLOW_LAYER_OUTER_L4_TCP; break; @@ -6981,11 +7529,6 @@ cnt_err: flow_dv_translate_item_udp(match_mask, match_value, items, tunnel); matcher.priority = MLX5_PRIORITY_MAP_L4; - dev_flow->hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, ETH_RSS_UDP, - IBV_RX_HASH_SRC_PORT_UDP | - IBV_RX_HASH_DST_PORT_UDP); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : MLX5_FLOW_LAYER_OUTER_L4_UDP; break; @@ -7050,7 +7593,7 @@ cnt_err: last_item = MLX5_FLOW_ITEM_TAG; break; case MLX5_RTE_FLOW_ITEM_TYPE_TAG: - flow_dv_translate_mlx5_item_tag(match_mask, + flow_dv_translate_mlx5_item_tag(dev, match_mask, match_value, items); last_item = MLX5_FLOW_ITEM_TAG; break; @@ -7060,19 +7603,24 @@ cnt_err: items); last_item = MLX5_FLOW_ITEM_TX_QUEUE; break; + case RTE_FLOW_ITEM_TYPE_GTP: + flow_dv_translate_item_gtp(match_mask, match_value, + items, tunnel); + last_item = MLX5_FLOW_LAYER_GTP; + break; default: break; } item_flags |= last_item; } /* - * In case of ingress traffic when E-Switch mode is enabled, - * we have two cases where we need to set the source port manually. + * When E-Switch mode is enabled, we have two cases where we need to + * set the source port manually. * The first one, is in case of Nic steering rule, and the second is * E-Switch rule where no port_id item was found. In both cases * the source port is set according the current port in use. */ - if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) && + if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && (priv->representor || priv->master)) { if (flow_dv_translate_item_port_id(dev, match_mask, match_value, NULL)) @@ -7081,15 +7629,18 @@ cnt_err: assert(!flow_dv_check_valid_spec(matcher.mask.buf, dev_flow->dv.value.buf)); dev_flow->layers = item_flags; + if (action_flags & MLX5_FLOW_ACTION_RSS) + flow_dv_hashfields_set(dev_flow); /* Register matcher. */ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, matcher.mask.size); matcher.priority = mlx5_flow_adjust_priority(dev, priority, matcher.priority); - matcher.egress = attr->egress; - matcher.group = dev_flow->group; - matcher.transfer = attr->transfer; - if (flow_dv_matcher_register(dev, &matcher, dev_flow, error)) + /* reserved field no needs to be set to 0 here. */ + tbl_key.domain = attr->transfer; + tbl_key.direction = attr->egress; + tbl_key.table_id = dev_flow->group; + if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error)) return -rte_errno; return 0; } @@ -7225,9 +7776,6 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, struct mlx5_flow *flow) { struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher; - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_ibv_shared *sh = priv->sh; - struct mlx5_flow_tbl_resource *tbl; assert(matcher->matcher_object); DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", @@ -7237,11 +7785,8 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, claim_zero(mlx5_glue->dv_destroy_flow_matcher (matcher->matcher_object)); LIST_REMOVE(matcher, next); - if (matcher->egress) - tbl = &sh->tx_tbl[matcher->group]; - else - tbl = &sh->rx_tbl[matcher->group]; - flow_dv_tbl_resource_release(tbl); + /* table ref-- in release interface. */ + flow_dv_tbl_resource_release(dev, matcher->tbl); rte_free(matcher); DRV_LOG(DEBUG, "port %u matcher %p: removed", dev->data->port_id, (void *)matcher); @@ -7284,6 +7829,8 @@ flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) /** * Release an jump to table action resource. * + * @param dev + * Pointer to Ethernet device. * @param flow * Pointer to mlx5_flow. * @@ -7291,10 +7838,13 @@ flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) * 1 while a reference on it exists, 0 when freed. */ static int -flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow) +flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, + struct mlx5_flow *flow) { - struct mlx5_flow_dv_jump_tbl_resource *cache_resource = - flow->dv.jump; + struct mlx5_flow_dv_jump_tbl_resource *cache_resource = flow->dv.jump; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(cache_resource, + struct mlx5_flow_tbl_data_entry, jump); assert(cache_resource->action); DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", @@ -7303,9 +7853,8 @@ flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow) if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { claim_zero(mlx5_glue->destroy_flow_action (cache_resource->action)); - LIST_REMOVE(cache_resource, next); - flow_dv_tbl_resource_release(cache_resource->tbl); - rte_free(cache_resource); + /* jump action memory free is inside the table release. */ + flow_dv_tbl_resource_release(dev, &tbl_data->tbl); DRV_LOG(DEBUG, "jump table resource %p: removed", (void *)cache_resource); return 0; @@ -7463,6 +8012,10 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_counter_release(dev, flow->counter); flow->counter = NULL; } + if (flow->meter) { + mlx5_flow_meter_detach(flow->meter); + flow->meter = NULL; + } while (!LIST_EMPTY(&flow->dev_flows)) { dev_flow = LIST_FIRST(&flow->dev_flows); LIST_REMOVE(dev_flow, next); @@ -7473,7 +8026,7 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) if (dev_flow->dv.modify_hdr) flow_dv_modify_hdr_resource_release(dev_flow); if (dev_flow->dv.jump) - flow_dv_jump_tbl_resource_release(dev_flow); + flow_dv_jump_tbl_resource_release(dev, dev_flow); if (dev_flow->dv.port_id_action) flow_dv_port_id_action_resource_release(dev_flow); if (dev_flow->dv.push_vlan_res) @@ -7568,6 +8121,469 @@ flow_dv_query(struct rte_eth_dev *dev, return ret; } +/** + * Destroy the meter table set. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] tbl + * Pointer to the meter table set. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, + struct mlx5_meter_domains_infos *tbl) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_meter_domains_infos *mtd = + (struct mlx5_meter_domains_infos *)tbl; + + if (!mtd || !priv->config.dv_flow_en) + return 0; + if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) + claim_zero(mlx5_glue->dv_destroy_flow + (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); + if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) + claim_zero(mlx5_glue->dv_destroy_flow + (mtd->egress.policer_rules[RTE_MTR_DROPPED])); + if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) + claim_zero(mlx5_glue->dv_destroy_flow + (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); + if (mtd->egress.color_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->egress.color_matcher)); + if (mtd->egress.any_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->egress.any_matcher)); + if (mtd->egress.tbl) + claim_zero(flow_dv_tbl_resource_release(dev, + mtd->egress.tbl)); + if (mtd->ingress.color_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->ingress.color_matcher)); + if (mtd->ingress.any_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->ingress.any_matcher)); + if (mtd->ingress.tbl) + claim_zero(flow_dv_tbl_resource_release(dev, + mtd->ingress.tbl)); + if (mtd->transfer.color_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->transfer.color_matcher)); + if (mtd->transfer.any_matcher) + claim_zero(mlx5_glue->dv_destroy_flow_matcher + (mtd->transfer.any_matcher)); + if (mtd->transfer.tbl) + claim_zero(flow_dv_tbl_resource_release(dev, + mtd->transfer.tbl)); + if (mtd->drop_actn) + claim_zero(mlx5_glue->destroy_flow_action(mtd->drop_actn)); + rte_free(mtd); + return 0; +} + +/* Number of meter flow actions, count and jump or count and drop. */ +#define METER_ACTIONS 2 + +/** + * Create specify domain meter table and suffix table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] mtb + * Pointer to DV meter table set. + * @param[in] egress + * Table attribute. + * @param[in] transfer + * Table attribute. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * + * @return + * 0 on success, -1 otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, + struct mlx5_meter_domains_infos *mtb, + uint8_t egress, uint8_t transfer, + uint32_t color_reg_c_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_dv_match_params mask = { + .size = sizeof(mask.buf), + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf), + }; + struct mlx5dv_flow_matcher_attr dv_attr = { + .type = IBV_FLOW_ATTR_NORMAL, + .priority = 0, + .match_criteria_enable = 0, + .match_mask = (void *)&mask, + }; + void *actions[METER_ACTIONS]; + struct mlx5_flow_tbl_resource **sfx_tbl; + struct mlx5_meter_domain_info *dtb; + struct rte_flow_error error; + int i = 0; + + if (transfer) { + sfx_tbl = &sh->fdb_mtr_sfx_tbl; + dtb = &mtb->transfer; + } else if (egress) { + sfx_tbl = &sh->tx_mtr_sfx_tbl; + dtb = &mtb->egress; + } else { + sfx_tbl = &sh->rx_mtr_sfx_tbl; + dtb = &mtb->ingress; + } + /* If the suffix table in missing, create it. */ + if (!(*sfx_tbl)) { + *sfx_tbl = flow_dv_tbl_resource_get(dev, + MLX5_FLOW_TABLE_LEVEL_SUFFIX, + egress, transfer, &error); + if (!(*sfx_tbl)) { + DRV_LOG(ERR, "Failed to create meter suffix table."); + return -1; + } + } + /* Create the meter table with METER level. */ + dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, + egress, transfer, &error); + if (!dtb->tbl) { + DRV_LOG(ERR, "Failed to create meter policer table."); + return -1; + } + /* Create matchers, Any and Color. */ + dv_attr.priority = 3; + dv_attr.match_criteria_enable = 0; + dtb->any_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx, + &dv_attr, + dtb->tbl->obj); + if (!dtb->any_matcher) { + DRV_LOG(ERR, "Failed to create meter" + " policer default matcher."); + goto error_exit; + } + dv_attr.priority = 0; + dv_attr.match_criteria_enable = + 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; + flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, + rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); + dtb->color_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx, + &dv_attr, + dtb->tbl->obj); + if (!dtb->color_matcher) { + DRV_LOG(ERR, "Failed to create meter policer color matcher."); + goto error_exit; + } + if (mtb->count_actns[RTE_MTR_DROPPED]) + actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; + actions[i++] = mtb->drop_actn; + /* Default rule: lowest priority, match any, actions: drop. */ + dtb->policer_rules[RTE_MTR_DROPPED] = + mlx5_glue->dv_create_flow(dtb->any_matcher, + (void *)&value, i, actions); + if (!dtb->policer_rules[RTE_MTR_DROPPED]) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error_exit; + } + return 0; +error_exit: + return -1; +} + +/** + * Create the needed meter and suffix tables. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] fm + * Pointer to the flow meter. + * + * @return + * Pointer to table set on success, NULL otherwise and rte_errno is set. + */ +static struct mlx5_meter_domains_infos * +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, + const struct mlx5_flow_meter *fm) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_meter_domains_infos *mtb; + int ret; + int i; + + if (!priv->mtr_en) { + rte_errno = ENOTSUP; + return NULL; + } + mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0); + if (!mtb) { + DRV_LOG(ERR, "Failed to allocate memory for meter."); + return NULL; + } + /* Create meter count actions */ + for (i = 0; i <= RTE_MTR_DROPPED; i++) { + if (!fm->policer_stats.cnt[i]) + continue; + mtb->count_actns[i] = fm->policer_stats.cnt[i]->action; + } + /* Create drop action. */ + mtb->drop_actn = mlx5_glue->dr_create_flow_action_drop(); + if (!mtb->drop_actn) { + DRV_LOG(ERR, "Failed to create drop action."); + goto error_exit; + } + /* Egress meter table. */ + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to prepare egress meter table."); + goto error_exit; + } + /* Ingress meter table. */ + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to prepare ingress meter table."); + goto error_exit; + } + /* FDB meter table. */ + if (priv->config.dv_esw_en) { + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, + priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to prepare fdb meter table."); + goto error_exit; + } + } + return mtb; +error_exit: + flow_dv_destroy_mtr_tbl(dev, mtb); + return NULL; +} + +/** + * Destroy domain policer rule. + * + * @param[in] dt + * Pointer to domain table. + */ +static void +flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +{ + int i; + + for (i = 0; i < RTE_MTR_DROPPED; i++) { + if (dt->policer_rules[i]) { + claim_zero(mlx5_glue->dv_destroy_flow + (dt->policer_rules[i])); + dt->policer_rules[i] = NULL; + } + } + if (dt->jump_actn) { + claim_zero(mlx5_glue->destroy_flow_action(dt->jump_actn)); + dt->jump_actn = NULL; + } +} + +/** + * Destroy policer rules. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. + * @param[in] attr + * Pointer to flow attributes. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, + const struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) +{ + struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; + + if (!mtb) + return 0; + if (attr->egress) + flow_dv_destroy_domain_policer_rule(&mtb->egress); + if (attr->ingress) + flow_dv_destroy_domain_policer_rule(&mtb->ingress); + if (attr->transfer) + flow_dv_destroy_domain_policer_rule(&mtb->transfer); + return 0; +} + +/** + * Create specify domain meter policer rule. + * + * @param[in] fm + * Pointer to flow meter structure. + * @param[in] mtb + * Pointer to DV meter table set. + * @param[in] sfx_tb + * Pointer to suffix table. + * @param[in] mtr_reg_c + * Color match REG_C. + * + * @return + * 0 on success, -1 otherwise. + */ +static int +flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, + struct mlx5_meter_domain_info *dtb, + struct mlx5_flow_tbl_resource *sfx_tb, + uint8_t mtr_reg_c) +{ + struct mlx5_flow_dv_match_params matcher = { + .size = sizeof(matcher.buf), + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf), + }; + struct mlx5_meter_domains_infos *mtb = fm->mfts; + void *actions[METER_ACTIONS]; + int i; + + /* Create jump action. */ + if (!sfx_tb) + return -1; + if (!dtb->jump_actn) + dtb->jump_actn = + mlx5_glue->dr_create_flow_action_dest_flow_tbl + (sfx_tb->obj); + if (!dtb->jump_actn) { + DRV_LOG(ERR, "Failed to create policer jump action."); + goto error; + } + for (i = 0; i < RTE_MTR_DROPPED; i++) { + int j = 0; + + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, + rte_col_2_mlx5_col(i), UINT8_MAX); + if (mtb->count_actns[i]) + actions[j++] = mtb->count_actns[i]; + if (fm->params.action[i] == MTR_POLICER_ACTION_DROP) + actions[j++] = mtb->drop_actn; + else + actions[j++] = dtb->jump_actn; + dtb->policer_rules[i] = + mlx5_glue->dv_create_flow(dtb->color_matcher, + (void *)&value, + j, actions); + if (!dtb->policer_rules[i]) { + DRV_LOG(ERR, "Failed to create policer rule."); + goto error; + } + } + return 0; +error: + rte_errno = errno; + return -1; +} + +/** + * Create policer rules. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. + * @param[in] attr + * Pointer to flow attributes. + * + * @return + * 0 on success, -1 otherwise. + */ +static int +flow_dv_create_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_meter_domains_infos *mtb = fm->mfts; + int ret; + + if (attr->egress) { + ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, + priv->sh->tx_mtr_sfx_tbl, + priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to create egress policer."); + goto error; + } + } + if (attr->ingress) { + ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, + priv->sh->rx_mtr_sfx_tbl, + priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to create ingress policer."); + goto error; + } + } + if (attr->transfer) { + ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, + priv->sh->fdb_mtr_sfx_tbl, + priv->mtr_color_reg); + if (ret) { + DRV_LOG(ERR, "Failed to create transfer policer."); + goto error; + } + } + return 0; +error: + flow_dv_destroy_policer_rules(dev, fm, attr); + return -1; +} + +/** + * Query a devx counter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] cnt + * Pointer to the flow counter. + * @param[in] clear + * Set to clear the counter statistics. + * @param[out] pkts + * The statistics value of packets. + * @param[out] bytes + * The statistics value of bytes. + * + * @return + * 0 on success, otherwise return -1. + */ +static int +flow_dv_counter_query(struct rte_eth_dev *dev, + struct mlx5_flow_counter *cnt, bool clear, + uint64_t *pkts, uint64_t *bytes) +{ + struct mlx5_priv *priv = dev->data->dev_private; + uint64_t inn_pkts, inn_bytes; + int ret; + + if (!priv->config.devx) + return -1; + ret = _flow_dv_query_count(dev, cnt, &inn_pkts, &inn_bytes); + if (ret) + return -1; + *pkts = inn_pkts - cnt->hits; + *bytes = inn_bytes - cnt->bytes; + if (clear) { + cnt->hits = inn_pkts; + cnt->bytes = inn_bytes; + } + return 0; +} + /* * Mutex-protected thunk to lock-free __flow_dv_translate(). */ @@ -7625,6 +8641,31 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_shared_unlock(dev); } +/* + * Mutex-protected thunk to lock-free flow_dv_counter_alloc(). + */ +static struct mlx5_flow_counter * +flow_dv_counter_allocate(struct rte_eth_dev *dev) +{ + struct mlx5_flow_counter *cnt; + + flow_dv_shared_lock(dev); + cnt = flow_dv_counter_alloc(dev, 0, 0, 1); + flow_dv_shared_unlock(dev); + return cnt; +} + +/* + * Mutex-protected thunk to lock-free flow_dv_counter_release(). + */ +static void +flow_dv_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt) +{ + flow_dv_shared_lock(dev); + flow_dv_counter_release(dev, cnt); + flow_dv_shared_unlock(dev); +} + const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .validate = flow_dv_validate, .prepare = flow_dv_prepare, @@ -7633,6 +8674,13 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .remove = flow_dv_remove, .destroy = flow_dv_destroy, .query = flow_dv_query, + .create_mtr_tbls = flow_dv_create_mtr_tbl, + .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, + .create_policer_rules = flow_dv_create_policer_rules, + .destroy_policer_rules = flow_dv_destroy_policer_rules, + .counter_alloc = flow_dv_counter_allocate, + .counter_free = flow_dv_counter_free, + .counter_query = flow_dv_counter_query, }; #endif /* HAVE_IBV_FLOW_DV_SUPPORT */