From 4bb14c83df952bdb54a735873d4a5347b38e1117 Mon Sep 17 00:00:00 2001 From: Dekel Peled Date: Thu, 27 Dec 2018 13:09:38 +0200 Subject: [PATCH] net/mlx5: support modify header using Direct Verbs This patch implements the set of actions to support offload of packet header modifications to MLX5 NIC. Implementation is based on RFC [1]. [1] http://mails.dpdk.org/archives/dev/2018-November/119971.html Signed-off-by: Dekel Peled Acked-by: Shahaf Shuler --- drivers/net/mlx5/Makefile | 2 +- drivers/net/mlx5/meson.build | 2 +- drivers/net/mlx5/mlx5.h | 1 + drivers/net/mlx5/mlx5_flow.h | 56 +- drivers/net/mlx5/mlx5_flow_dv.c | 947 +++++++++++++++++++++++++++++++- drivers/net/mlx5/mlx5_glue.c | 22 + drivers/net/mlx5/mlx5_glue.h | 5 + drivers/net/mlx5/mlx5_prm.h | 22 +- 8 files changed, 1036 insertions(+), 21 deletions(-) diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index 895cdfeeda..1353c18278 100644 --- a/drivers/net/mlx5/Makefile +++ b/drivers/net/mlx5/Makefile @@ -8,7 +8,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_pmd_mlx5.a LIB_GLUE = $(LIB_GLUE_BASE).$(LIB_GLUE_VERSION) LIB_GLUE_BASE = librte_pmd_mlx5_glue.so -LIB_GLUE_VERSION = 18.11.0 +LIB_GLUE_VERSION = 19.02.0 # Sources. SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build index 28938db0fd..8ba19e8187 100644 --- a/drivers/net/mlx5/meson.build +++ b/drivers/net/mlx5/meson.build @@ -4,7 +4,7 @@ pmd_dlopen = get_option('enable_driver_mlx_glue') LIB_GLUE_BASE = 'librte_pmd_mlx5_glue.so' -LIB_GLUE_VERSION = '18.11.0' +LIB_GLUE_VERSION = '19.02.0' LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION if pmd_dlopen dpdk_conf.set('RTE_LIBRTE_MLX5_DLOPEN_DEPS', 1) diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 75aeeb2b61..b2fe5cb34a 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -227,6 +227,7 @@ struct priv { LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls; LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers; LIST_HEAD(encap_decap, mlx5_flow_dv_encap_decap_resource) encaps_decaps; + LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds; uint32_t link_speed_capa; /* Link speed capabilities. */ struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */ struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 4a7c052936..cb1e6fd8b9 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -69,6 +69,18 @@ (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \ MLX5_FLOW_LAYER_INNER_L4) +/* Layer Masks. */ +#define MLX5_FLOW_LAYER_L2 \ + (MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_INNER_L2) +#define MLX5_FLOW_LAYER_L3_IPV4 \ + (MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV4) +#define MLX5_FLOW_LAYER_L3_IPV6 \ + (MLX5_FLOW_LAYER_OUTER_L3_IPV6 | MLX5_FLOW_LAYER_INNER_L3_IPV6) +#define MLX5_FLOW_LAYER_L3 \ + (MLX5_FLOW_LAYER_L3_IPV4 | MLX5_FLOW_LAYER_L3_IPV6) +#define MLX5_FLOW_LAYER_L4 \ + (MLX5_FLOW_LAYER_OUTER_L4 | MLX5_FLOW_LAYER_INNER_L4) + /* Actions */ #define MLX5_FLOW_ACTION_DROP (1u << 0) #define MLX5_FLOW_ACTION_QUEUE (1u << 1) @@ -110,6 +122,17 @@ MLX5_FLOW_ACTION_NVGRE_DECAP | \ MLX5_FLOW_ACTION_RAW_DECAP) +#define MLX5_FLOW_MODIFY_HDR_ACTIONS (MLX5_FLOW_ACTION_SET_IPV4_SRC | \ + MLX5_FLOW_ACTION_SET_IPV4_DST | \ + MLX5_FLOW_ACTION_SET_IPV6_SRC | \ + MLX5_FLOW_ACTION_SET_IPV6_DST | \ + MLX5_FLOW_ACTION_SET_TP_SRC | \ + MLX5_FLOW_ACTION_SET_TP_DST | \ + MLX5_FLOW_ACTION_SET_TTL | \ + MLX5_FLOW_ACTION_DEC_TTL | \ + MLX5_FLOW_ACTION_SET_MAC_SRC | \ + MLX5_FLOW_ACTION_SET_MAC_DST) + #ifndef IPPROTO_MPLS #define IPPROTO_MPLS 137 #endif @@ -153,9 +176,6 @@ /* IBV hash source bits for IPV6. */ #define MLX5_IPV6_IBV_RX_HASH (IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6) -/* Max number of actions per DV flow. */ -#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8 - enum mlx5_flow_drv_type { MLX5_FLOW_TYPE_MIN, MLX5_FLOW_TYPE_DV, @@ -172,9 +192,6 @@ struct mlx5_flow_dv_match_params { /**< Matcher value. This value is used as the mask or as a key. */ }; -#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8 -#define MLX5_ENCAP_MAX_LEN 132 - /* Matcher structure. */ struct mlx5_flow_dv_matcher { LIST_ENTRY(mlx5_flow_dv_matcher) next; @@ -187,6 +204,8 @@ struct mlx5_flow_dv_matcher { struct mlx5_flow_dv_match_params mask; /**< Matcher mask. */ }; +#define MLX5_ENCAP_MAX_LEN 132 + /* Encap/decap resource structure. */ struct mlx5_flow_dv_encap_decap_resource { LIST_ENTRY(mlx5_flow_dv_encap_decap_resource) next; @@ -200,6 +219,29 @@ struct mlx5_flow_dv_encap_decap_resource { uint8_t ft_type; }; +/* Number of modification commands. */ +#define MLX5_MODIFY_NUM 8 + +/* Modify resource structure */ +struct mlx5_flow_dv_modify_hdr_resource { + LIST_ENTRY(mlx5_flow_dv_modify_hdr_resource) next; + /* Pointer to next element. */ + rte_atomic32_t refcnt; /**< Reference counter. */ + struct ibv_flow_action *verbs_action; + /**< Verbs modify header action object. */ + uint8_t ft_type; /**< Flow table type, Rx or Tx. */ + uint32_t actions_num; /**< Number of modification actions. */ + struct mlx5_modification_cmd actions[MLX5_MODIFY_NUM]; + /**< Modification actions. */ +}; + +/* + * Max number of actions per DV flow. + * See CREATE_FLOW_MAX_FLOW_ACTIONS_SUPPORTED + * In rdma-core file providers/mlx5/verbs.c + */ +#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8 + /* DV flows structure. */ struct mlx5_flow_dv { uint64_t hash_fields; /**< Fields that participate in the hash. */ @@ -210,6 +252,8 @@ struct mlx5_flow_dv { /**< Holds the value that the packet is compared to. */ struct mlx5_flow_dv_encap_decap_resource *encap_decap; /**< Pointer to encap/decap resource in cache. */ + struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; + /**< Pointer to modify header resource in cache. */ struct ibv_flow *flow; /**< Installed flow. */ #ifdef HAVE_IBV_FLOW_DV_SUPPORT struct mlx5dv_flow_action_attr actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS]; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 1f31874879..4c0b7ed67f 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -35,6 +35,478 @@ #ifdef HAVE_IBV_FLOW_DV_SUPPORT +union flow_dv_attr { + struct { + uint32_t valid:1; + uint32_t ipv4:1; + uint32_t ipv6:1; + uint32_t tcp:1; + uint32_t udp:1; + uint32_t reserved:27; + }; + uint32_t attr; +}; + +/** + * Initialize flow attributes structure according to flow items' types. + * + * @param[in] item + * Pointer to item specification. + * @param[out] attr + * Pointer to flow attributes structure. + */ +static void +flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr) +{ + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + switch (item->type) { + case RTE_FLOW_ITEM_TYPE_IPV4: + attr->ipv4 = 1; + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + attr->ipv6 = 1; + break; + case RTE_FLOW_ITEM_TYPE_UDP: + attr->udp = 1; + break; + case RTE_FLOW_ITEM_TYPE_TCP: + attr->tcp = 1; + break; + default: + break; + } + } + attr->valid = 1; +} + +struct field_modify_info { + uint32_t size; /* Size of field in protocol header, in bytes. */ + uint32_t offset; /* Offset of field in protocol header, in bytes. */ + enum mlx5_modification_field id; +}; + +struct field_modify_info modify_eth[] = { + {4, 0, MLX5_MODI_OUT_DMAC_47_16}, + {2, 4, MLX5_MODI_OUT_DMAC_15_0}, + {4, 6, MLX5_MODI_OUT_SMAC_47_16}, + {2, 10, MLX5_MODI_OUT_SMAC_15_0}, + {0, 0, 0}, +}; + +struct field_modify_info modify_ipv4[] = { + {1, 8, MLX5_MODI_OUT_IPV4_TTL}, + {4, 12, MLX5_MODI_OUT_SIPV4}, + {4, 16, MLX5_MODI_OUT_DIPV4}, + {0, 0, 0}, +}; + +struct field_modify_info modify_ipv6[] = { + {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT}, + {4, 8, MLX5_MODI_OUT_SIPV6_127_96}, + {4, 12, MLX5_MODI_OUT_SIPV6_95_64}, + {4, 16, MLX5_MODI_OUT_SIPV6_63_32}, + {4, 20, MLX5_MODI_OUT_SIPV6_31_0}, + {4, 24, MLX5_MODI_OUT_DIPV6_127_96}, + {4, 28, MLX5_MODI_OUT_DIPV6_95_64}, + {4, 32, MLX5_MODI_OUT_DIPV6_63_32}, + {4, 36, MLX5_MODI_OUT_DIPV6_31_0}, + {0, 0, 0}, +}; + +struct field_modify_info modify_udp[] = { + {2, 0, MLX5_MODI_OUT_UDP_SPORT}, + {2, 2, MLX5_MODI_OUT_UDP_DPORT}, + {0, 0, 0}, +}; + +struct field_modify_info modify_tcp[] = { + {2, 0, MLX5_MODI_OUT_TCP_SPORT}, + {2, 2, MLX5_MODI_OUT_TCP_DPORT}, + {0, 0, 0}, +}; + +/** + * Convert modify-header action to DV specification. + * + * @param[in] item + * Pointer to item specification. + * @param[in] field + * Pointer to field modification information. + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] type + * Type of modification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_modify_action(struct rte_flow_item *item, + struct field_modify_info *field, + struct mlx5_flow_dv_modify_hdr_resource *resource, + uint32_t type, + struct rte_flow_error *error) +{ + uint32_t i = resource->actions_num; + struct mlx5_modification_cmd *actions = resource->actions; + const uint8_t *spec = item->spec; + const uint8_t *mask = item->mask; + uint32_t set; + + while (field->size) { + set = 0; + /* Generate modify command for each mask segment. */ + memcpy(&set, &mask[field->offset], field->size); + if (set) { + if (i >= MLX5_MODIFY_NUM) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "too many items to modify"); + actions[i].action_type = type; + actions[i].field = field->id; + actions[i].length = field->size == + 4 ? 0 : field->size * 8; + rte_memcpy(&actions[i].data[4 - field->size], + &spec[field->offset], field->size); + actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); + ++i; + } + if (resource->actions_num != i) + resource->actions_num = i; + field++; + } + if (!resource->actions_num) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "invalid modification flow item"); + return 0; +} + +/** + * Convert modify-header set IPv4 address action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_ipv4 + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_ipv4 *conf = + (const struct rte_flow_action_set_ipv4 *)(action->conf); + struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; + struct rte_flow_item_ipv4 ipv4; + struct rte_flow_item_ipv4 ipv4_mask; + + memset(&ipv4, 0, sizeof(ipv4)); + memset(&ipv4_mask, 0, sizeof(ipv4_mask)); + if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) { + ipv4.hdr.src_addr = conf->ipv4_addr; + ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr; + } else { + ipv4.hdr.dst_addr = conf->ipv4_addr; + ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr; + } + item.spec = &ipv4; + item.mask = &ipv4_mask; + return flow_dv_convert_modify_action(&item, modify_ipv4, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Convert modify-header set IPv6 address action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_ipv6 + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_ipv6 *conf = + (const struct rte_flow_action_set_ipv6 *)(action->conf); + struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; + struct rte_flow_item_ipv6 ipv6; + struct rte_flow_item_ipv6 ipv6_mask; + + memset(&ipv6, 0, sizeof(ipv6)); + memset(&ipv6_mask, 0, sizeof(ipv6_mask)); + if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) { + memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr, + sizeof(ipv6.hdr.src_addr)); + memcpy(&ipv6_mask.hdr.src_addr, + &rte_flow_item_ipv6_mask.hdr.src_addr, + sizeof(ipv6.hdr.src_addr)); + } else { + memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr, + sizeof(ipv6.hdr.dst_addr)); + memcpy(&ipv6_mask.hdr.dst_addr, + &rte_flow_item_ipv6_mask.hdr.dst_addr, + sizeof(ipv6.hdr.dst_addr)); + } + item.spec = &ipv6; + item.mask = &ipv6_mask; + return flow_dv_convert_modify_action(&item, modify_ipv6, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Convert modify-header set MAC address action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_mac + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_mac *conf = + (const struct rte_flow_action_set_mac *)(action->conf); + struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH }; + struct rte_flow_item_eth eth; + struct rte_flow_item_eth eth_mask; + + memset(ð, 0, sizeof(eth)); + memset(ð_mask, 0, sizeof(eth_mask)); + if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) { + memcpy(ð.src.addr_bytes, &conf->mac_addr, + sizeof(eth.src.addr_bytes)); + memcpy(ð_mask.src.addr_bytes, + &rte_flow_item_eth_mask.src.addr_bytes, + sizeof(eth_mask.src.addr_bytes)); + } else { + memcpy(ð.dst.addr_bytes, &conf->mac_addr, + sizeof(eth.dst.addr_bytes)); + memcpy(ð_mask.dst.addr_bytes, + &rte_flow_item_eth_mask.dst.addr_bytes, + sizeof(eth_mask.dst.addr_bytes)); + } + item.spec = ð + item.mask = ð_mask; + return flow_dv_convert_modify_action(&item, modify_eth, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Convert modify-header set TP action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[in] items + * Pointer to rte_flow_item objects list. + * @param[in] attr + * Pointer to flow attributes structure. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_tp + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + const struct rte_flow_item *items, + union flow_dv_attr *attr, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_tp *conf = + (const struct rte_flow_action_set_tp *)(action->conf); + struct rte_flow_item item; + struct rte_flow_item_udp udp; + struct rte_flow_item_udp udp_mask; + struct rte_flow_item_tcp tcp; + struct rte_flow_item_tcp tcp_mask; + struct field_modify_info *field; + + if (!attr->valid) + flow_dv_attr_init(items, attr); + if (attr->udp) { + memset(&udp, 0, sizeof(udp)); + memset(&udp_mask, 0, sizeof(udp_mask)); + if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { + udp.hdr.src_port = conf->port; + udp_mask.hdr.src_port = + rte_flow_item_udp_mask.hdr.src_port; + } else { + udp.hdr.dst_port = conf->port; + udp_mask.hdr.dst_port = + rte_flow_item_udp_mask.hdr.dst_port; + } + item.type = RTE_FLOW_ITEM_TYPE_UDP; + item.spec = &udp; + item.mask = &udp_mask; + field = modify_udp; + } + if (attr->tcp) { + memset(&tcp, 0, sizeof(tcp)); + memset(&tcp_mask, 0, sizeof(tcp_mask)); + if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { + tcp.hdr.src_port = conf->port; + tcp_mask.hdr.src_port = + rte_flow_item_tcp_mask.hdr.src_port; + } else { + tcp.hdr.dst_port = conf->port; + tcp_mask.hdr.dst_port = + rte_flow_item_tcp_mask.hdr.dst_port; + } + item.type = RTE_FLOW_ITEM_TYPE_TCP; + item.spec = &tcp; + item.mask = &tcp_mask; + field = modify_tcp; + } + return flow_dv_convert_modify_action(&item, field, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Convert modify-header set TTL action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[in] items + * Pointer to rte_flow_item objects list. + * @param[in] attr + * Pointer to flow attributes structure. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_ttl + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + const struct rte_flow_item *items, + union flow_dv_attr *attr, + struct rte_flow_error *error) +{ + const struct rte_flow_action_set_ttl *conf = + (const struct rte_flow_action_set_ttl *)(action->conf); + struct rte_flow_item item; + struct rte_flow_item_ipv4 ipv4; + struct rte_flow_item_ipv4 ipv4_mask; + struct rte_flow_item_ipv6 ipv6; + struct rte_flow_item_ipv6 ipv6_mask; + struct field_modify_info *field; + + if (!attr->valid) + flow_dv_attr_init(items, attr); + if (attr->ipv4) { + memset(&ipv4, 0, sizeof(ipv4)); + memset(&ipv4_mask, 0, sizeof(ipv4_mask)); + ipv4.hdr.time_to_live = conf->ttl_value; + ipv4_mask.hdr.time_to_live = 0xFF; + item.type = RTE_FLOW_ITEM_TYPE_IPV4; + item.spec = &ipv4; + item.mask = &ipv4_mask; + field = modify_ipv4; + } + if (attr->ipv6) { + memset(&ipv6, 0, sizeof(ipv6)); + memset(&ipv6_mask, 0, sizeof(ipv6_mask)); + ipv6.hdr.hop_limits = conf->ttl_value; + ipv6_mask.hdr.hop_limits = 0xFF; + item.type = RTE_FLOW_ITEM_TYPE_IPV6; + item.spec = &ipv6; + item.mask = &ipv6_mask; + field = modify_ipv6; + } + return flow_dv_convert_modify_action(&item, field, resource, + MLX5_MODIFICATION_TYPE_SET, error); +} + +/** + * Convert modify-header decrement TTL action to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[in] items + * Pointer to rte_flow_item objects list. + * @param[in] attr + * Pointer to flow attributes structure. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_dec_ttl + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_item *items, + union flow_dv_attr *attr, + struct rte_flow_error *error) +{ + struct rte_flow_item item; + struct rte_flow_item_ipv4 ipv4; + struct rte_flow_item_ipv4 ipv4_mask; + struct rte_flow_item_ipv6 ipv6; + struct rte_flow_item_ipv6 ipv6_mask; + struct field_modify_info *field; + + if (!attr->valid) + flow_dv_attr_init(items, attr); + if (attr->ipv4) { + memset(&ipv4, 0, sizeof(ipv4)); + memset(&ipv4_mask, 0, sizeof(ipv4_mask)); + ipv4.hdr.time_to_live = 0xFF; + ipv4_mask.hdr.time_to_live = 0xFF; + item.type = RTE_FLOW_ITEM_TYPE_IPV4; + item.spec = &ipv4; + item.mask = &ipv4_mask; + field = modify_ipv4; + } + if (attr->ipv6) { + memset(&ipv6, 0, sizeof(ipv6)); + memset(&ipv6_mask, 0, sizeof(ipv6_mask)); + ipv6.hdr.hop_limits = 0xFF; + ipv6_mask.hdr.hop_limits = 0xFF; + item.type = RTE_FLOW_ITEM_TYPE_IPV6; + item.spec = &ipv6; + item.mask = &ipv6_mask; + field = modify_ipv6; + } + return flow_dv_convert_modify_action(&item, field, resource, + MLX5_MODIFICATION_TYPE_ADD, error); +} + /** * Validate META item. * @@ -166,6 +638,11 @@ flow_dv_validate_action_l2_decap(uint64_t action_flags, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can only have a single encap or" " decap action in a flow"); + if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can't have decap action after" + " modify action"); if (attr->egress) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, @@ -254,6 +731,11 @@ flow_dv_validate_action_raw_decap(uint64_t action_flags, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can only have a single decap" " action in a flow"); + if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can't have decap action after" + " modify action"); /* decap action is valid on egress only if it is followed by encap */ if (attr->egress) { for (; action->type != RTE_FLOW_ACTION_TYPE_END && @@ -270,7 +752,6 @@ flow_dv_validate_action_raw_decap(uint64_t action_flags, return 0; } - /** * Find existing encap/decap resource or create and register a new one. * @@ -703,6 +1184,277 @@ flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, return 0; } +/** + * Validate the modify-header actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_hdr(const uint64_t action_flags, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, "action configuration not set"); + if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can't have encap action before" + " modify action"); + return 0; +} + +/** + * Validate the modify-header MAC address actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_mac(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L2)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no L2 item in pattern"); + } + return ret; +} + +/** + * Validate the modify-header IPv4 address actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_ipv4(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no ipv4 item in pattern"); + } + return ret; +} + +/** + * Validate the modify-header IPv6 address actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_ipv6(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no ipv6 item in pattern"); + } + return ret; +} + +/** + * Validate the modify-header TP actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_tp(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L4)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "no transport layer " + "in pattern"); + } + return ret; +} + +/** + * Validate the modify-header TTL actions. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_ttl(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_L3)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "no IP protocol in pattern"); + } + return ret; +} + +/** + * Find existing modify-header resource or create and register a new one. + * + * @param dev[in, out] + * Pointer to rte_eth_dev structure. + * @param[in, out] resource + * Pointer to modify-header resource. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ +static int +flow_dv_modify_hdr_resource_register + (struct rte_eth_dev *dev, + struct mlx5_flow_dv_modify_hdr_resource *resource, + struct mlx5_flow *dev_flow, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + struct mlx5_flow_dv_modify_hdr_resource *cache_resource; + + /* Lookup a matching resource from cache. */ + LIST_FOREACH(cache_resource, &priv->modify_cmds, next) { + if (resource->ft_type == cache_resource->ft_type && + resource->actions_num == cache_resource->actions_num && + !memcmp((const void *)resource->actions, + (const void *)cache_resource->actions, + (resource->actions_num * + sizeof(resource->actions[0])))) { + DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + rte_atomic32_inc(&cache_resource->refcnt); + dev_flow->dv.modify_hdr = cache_resource; + return 0; + } + } + /* Register new modify-header resource. */ + cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + if (!cache_resource) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + *cache_resource = *resource; + cache_resource->verbs_action = + mlx5_glue->dv_create_flow_action_modify_header + (priv->ctx, + cache_resource->actions_num * + sizeof(cache_resource->actions[0]), + (uint64_t *)cache_resource->actions, + cache_resource->ft_type); + if (!cache_resource->verbs_action) { + rte_free(cache_resource); + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create action"); + } + rte_atomic32_init(&cache_resource->refcnt); + rte_atomic32_inc(&cache_resource->refcnt); + LIST_INSERT_HEAD(&priv->modify_cmds, cache_resource, next); + dev_flow->dv.modify_hdr = cache_resource; + DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + return 0; +} + /** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. @@ -1014,6 +1766,87 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; ++actions_n; break; + case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: + case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: + ret = flow_dv_validate_action_modify_mac(action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? + MLX5_FLOW_ACTION_SET_MAC_SRC : + MLX5_FLOW_ACTION_SET_MAC_DST; + break; + + case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: + ret = flow_dv_validate_action_modify_ipv4(action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? + MLX5_FLOW_ACTION_SET_IPV4_SRC : + MLX5_FLOW_ACTION_SET_IPV4_DST; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: + ret = flow_dv_validate_action_modify_ipv6(action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? + MLX5_FLOW_ACTION_SET_IPV6_SRC : + MLX5_FLOW_ACTION_SET_IPV6_DST; + break; + case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: + case RTE_FLOW_ACTION_TYPE_SET_TP_DST: + ret = flow_dv_validate_action_modify_tp(action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? + MLX5_FLOW_ACTION_SET_TP_SRC : + MLX5_FLOW_ACTION_SET_TP_DST; + break; + case RTE_FLOW_ACTION_TYPE_DEC_TTL: + case RTE_FLOW_ACTION_TYPE_SET_TTL: + ret = flow_dv_validate_action_modify_ttl(action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_TTL ? + MLX5_FLOW_ACTION_SET_TTL : + MLX5_FLOW_ACTION_DEC_TTL; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1895,10 +2728,16 @@ flow_dv_translate(struct rte_eth_dev *dev, }, }; int actions_n = 0; + bool actions_end = false; + struct mlx5_flow_dv_modify_hdr_resource res = { + .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : + MLX5DV_FLOW_TABLE_TYPE_NIC_RX + }; + union flow_dv_attr flow_attr = { .attr = 0 }; if (priority == MLX5_FLOW_PRIO_RSVD) priority = priv->config.flow_prio - 1; - for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + for (; !actions_end ; actions++) { const struct rte_flow_action_queue *queue; const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; @@ -2025,6 +2864,77 @@ flow_dv_translate(struct rte_eth_dev *dev, /* If decap is followed by encap, handle it at encap. */ action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; break; + case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: + case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: + if (flow_dv_convert_action_modify_mac(&res, actions, + error)) + return -rte_errno; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? + MLX5_FLOW_ACTION_SET_MAC_SRC : + MLX5_FLOW_ACTION_SET_MAC_DST; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: + if (flow_dv_convert_action_modify_ipv4(&res, actions, + error)) + return -rte_errno; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? + MLX5_FLOW_ACTION_SET_IPV4_SRC : + MLX5_FLOW_ACTION_SET_IPV4_DST; + break; + case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: + case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: + if (flow_dv_convert_action_modify_ipv6(&res, actions, + error)) + return -rte_errno; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? + MLX5_FLOW_ACTION_SET_IPV6_SRC : + MLX5_FLOW_ACTION_SET_IPV6_DST; + break; + case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: + case RTE_FLOW_ACTION_TYPE_SET_TP_DST: + if (flow_dv_convert_action_modify_tp(&res, actions, + items, &flow_attr, + error)) + return -rte_errno; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? + MLX5_FLOW_ACTION_SET_TP_SRC : + MLX5_FLOW_ACTION_SET_TP_DST; + break; + case RTE_FLOW_ACTION_TYPE_DEC_TTL: + if (flow_dv_convert_action_modify_dec_ttl(&res, items, + &flow_attr, + error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_DEC_TTL; + break; + case RTE_FLOW_ACTION_TYPE_SET_TTL: + if (flow_dv_convert_action_modify_ttl(&res, actions, + items, &flow_attr, + error)) + return -rte_errno; + action_flags |= MLX5_FLOW_ACTION_SET_TTL; + break; + case RTE_FLOW_ACTION_TYPE_END: + actions_end = true; + if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) { + /* create modify action if needed. */ + if (flow_dv_modify_hdr_resource_register + (dev, &res, + dev_flow, + error)) + return -rte_errno; + dev_flow->dv.actions[actions_n].type = + MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; + dev_flow->dv.actions[actions_n].action = + dev_flow->dv.modify_hdr->verbs_action; + actions_n++; + } + break; default: break; } @@ -2308,6 +3218,37 @@ flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) return 1; } +/** + * Release a modify-header resource. + * + * @param flow + * Pointer to mlx5_flow. + * + * @return + * 1 while a reference on it exists, 0 when freed. + */ +static int +flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow) +{ + struct mlx5_flow_dv_modify_hdr_resource *cache_resource = + flow->dv.modify_hdr; + + assert(cache_resource->verbs_action); + DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { + claim_zero(mlx5_glue->destroy_flow_action + (cache_resource->verbs_action)); + LIST_REMOVE(cache_resource, next); + rte_free(cache_resource); + DRV_LOG(DEBUG, "modify-header resource %p: removed", + (void *)cache_resource); + return 0; + } + return 1; +} + /** * Remove the flow from the NIC but keeps it in memory. * @@ -2365,6 +3306,8 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_matcher_release(dev, dev_flow); if (dev_flow->dv.encap_decap) flow_dv_encap_decap_resource_release(dev_flow); + if (dev_flow->dv.modify_hdr) + flow_dv_modify_hdr_resource_release(dev_flow); rte_free(dev_flow); } } diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c index dd10ad6de4..a806d92a10 100644 --- a/drivers/net/mlx5/mlx5_glue.c +++ b/drivers/net/mlx5/mlx5_glue.c @@ -479,6 +479,26 @@ mlx5_glue_dv_create_flow_action_packet_reformat #endif } +static struct ibv_flow_action * +mlx5_glue_dv_create_flow_action_modify_header + (struct ibv_context *ctx, + size_t actions_sz, + uint64_t actions[], + enum mlx5dv_flow_table_type ft_type) +{ +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + return mlx5dv_create_flow_action_modify_header(ctx, actions_sz, + actions, ft_type); +#else + (void)ctx; + (void)actions_sz; + (void)actions; + (void)ft_type; + return NULL; +#endif +} + + alignas(RTE_CACHE_LINE_SIZE) const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){ .version = MLX5_GLUE_VERSION, @@ -535,4 +555,6 @@ const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){ .dv_create_flow = mlx5_glue_dv_create_flow, .dv_create_flow_action_packet_reformat = mlx5_glue_dv_create_flow_action_packet_reformat, + .dv_create_flow_action_modify_header = + mlx5_glue_dv_create_flow_action_modify_header, }; diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h index 2d92ba8bcf..9cfe836360 100644 --- a/drivers/net/mlx5/mlx5_glue.h +++ b/drivers/net/mlx5/mlx5_glue.h @@ -164,6 +164,11 @@ struct mlx5_glue { void *data, enum mlx5dv_flow_action_packet_reformat_type reformat_type, enum mlx5dv_flow_table_type ft_type); + struct ibv_flow_action *(*dv_create_flow_action_modify_header) + (struct ibv_context *ctx, + size_t actions_sz, + uint64_t actions[], + enum mlx5dv_flow_table_type ft_type); }; const struct mlx5_glue *mlx5_glue; diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h index 29742b1350..5c39036bcb 100644 --- a/drivers/net/mlx5/mlx5_prm.h +++ b/drivers/net/mlx5/mlx5_prm.h @@ -280,8 +280,14 @@ struct mlx5_cqe { /* CQE format value. */ #define MLX5_COMPRESSED 0x3 +/* Write a specific data value to a field. */ +#define MLX5_MODIFICATION_TYPE_SET 1 + +/* Add a specific data value to a field. */ +#define MLX5_MODIFICATION_TYPE_ADD 2 + /* The field of packet to be modified. */ -enum mlx5_modificaiton_field { +enum mlx5_modification_field { MLX5_MODI_OUT_SMAC_47_16 = 1, MLX5_MODI_OUT_SMAC_15_0, MLX5_MODI_OUT_ETHERTYPE, @@ -337,23 +343,17 @@ struct mlx5_modification_cmd { union { uint32_t data0; struct { - unsigned int bits:5; + unsigned int length:5; unsigned int rsvd0:3; - unsigned int src_offset:5; /* Start bit offset. */ + unsigned int offset:5; unsigned int rsvd1:3; - unsigned int src_field:12; - unsigned int type:4; + unsigned int field:12; + unsigned int action_type:4; }; }; union { uint32_t data1; uint8_t data[4]; - struct { - unsigned int rsvd2:8; - unsigned int dst_offset:8; - unsigned int dst_field:12; - unsigned int rsvd3:4; - }; }; }; -- 2.20.1