X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=1d1ff909a5ff87fa2af196223626955074daf0c6;hb=31538ef62ceb23d8cb655b1598c7a6f2da2ab04c;hp=f048920c1da0f5955a25ce1ce86ad70c7ce33b58;hpb=2e4c987aad91b836bfaf8f23d281c04652d7ddad;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index f048920c1d..1d1ff909a5 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -6,6 +6,7 @@ #include #include #include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ @@ -39,6 +40,16 @@ #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 #endif +#ifndef HAVE_MLX5DV_DR_ESWITCH +#ifndef MLX5DV_FLOW_TABLE_TYPE_FDB +#define MLX5DV_FLOW_TABLE_TYPE_FDB 0 +#endif +#endif + +#ifndef HAVE_MLX5DV_DR +#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1 +#endif + union flow_dv_attr { struct { uint32_t valid:1; @@ -126,9 +137,44 @@ struct field_modify_info modify_udp[] = { struct field_modify_info modify_tcp[] = { {2, 0, MLX5_MODI_OUT_TCP_SPORT}, {2, 2, MLX5_MODI_OUT_TCP_DPORT}, + {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM}, + {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM}, {0, 0, 0}, }; +static void +mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item, uint64_t *flags) +{ + uint8_t next_protocol = 0xFF; + + if (item->mask != NULL) { + switch (item->type) { + case RTE_FLOW_ITEM_TYPE_IPV4: + next_protocol = + ((const struct rte_flow_item_ipv4 *) + (item->spec))->hdr.next_proto_id; + next_protocol &= + ((const struct rte_flow_item_ipv4 *) + (item->mask))->hdr.next_proto_id; + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + next_protocol = + ((const struct rte_flow_item_ipv6 *) + (item->spec))->hdr.proto; + next_protocol &= + ((const struct rte_flow_item_ipv6 *) + (item->mask))->hdr.proto; + break; + default: + break; + } + } + if (next_protocol == IPPROTO_IPIP) + *flags |= MLX5_FLOW_LAYER_IPIP; + if (next_protocol == IPPROTO_IPV6) + *flags |= MLX5_FLOW_LAYER_IPV6_ENCAP; +} + /** * Acquire the synchronizing object to protect multithreaded access * to shared dv context. Lock occurs only if context is actually @@ -550,6 +596,96 @@ flow_dv_convert_action_modify_dec_ttl MLX5_MODIFICATION_TYPE_ADD, error); } +/** + * Convert modify-header increment/decrement TCP Sequence number + * to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_tcp_seq + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const rte_be32_t *conf = (const rte_be32_t *)(action->conf); + uint64_t value = rte_be_to_cpu_32(*conf); + struct rte_flow_item item; + struct rte_flow_item_tcp tcp; + struct rte_flow_item_tcp tcp_mask; + + memset(&tcp, 0, sizeof(tcp)); + memset(&tcp_mask, 0, sizeof(tcp_mask)); + if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ) + /* + * The HW has no decrement operation, only increment operation. + * To simulate decrement X from Y using increment operation + * we need to add UINT32_MAX X times to Y. + * Each adding of UINT32_MAX decrements Y by 1. + */ + value *= UINT32_MAX; + tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value); + tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX); + item.type = RTE_FLOW_ITEM_TYPE_TCP; + item.spec = &tcp; + item.mask = &tcp_mask; + return flow_dv_convert_modify_action(&item, modify_tcp, resource, + MLX5_MODIFICATION_TYPE_ADD, error); +} + +/** + * Convert modify-header increment/decrement TCP Acknowledgment number + * to DV specification. + * + * @param[in,out] resource + * Pointer to the modify-header resource. + * @param[in] action + * Pointer to action specification. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_convert_action_modify_tcp_ack + (struct mlx5_flow_dv_modify_hdr_resource *resource, + const struct rte_flow_action *action, + struct rte_flow_error *error) +{ + const rte_be32_t *conf = (const rte_be32_t *)(action->conf); + uint64_t value = rte_be_to_cpu_32(*conf); + struct rte_flow_item item; + struct rte_flow_item_tcp tcp; + struct rte_flow_item_tcp tcp_mask; + + memset(&tcp, 0, sizeof(tcp)); + memset(&tcp_mask, 0, sizeof(tcp_mask)); + if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK) + /* + * The HW has no decrement operation, only increment operation. + * To simulate decrement X from Y using increment operation + * we need to add UINT32_MAX X times to Y. + * Each adding of UINT32_MAX decrements Y by 1. + */ + value *= UINT32_MAX; + tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value); + tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX); + item.type = RTE_FLOW_ITEM_TYPE_TCP; + item.spec = &tcp; + item.mask = &tcp_mask; + return flow_dv_convert_modify_action(&item, modify_tcp, resource, + MLX5_MODIFICATION_TYPE_ADD, error); +} + /** * Validate META item. * @@ -935,13 +1071,15 @@ flow_dv_encap_decap_resource_register struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_encap_decap_resource *cache_resource; struct rte_flow *flow = dev_flow->flow; - struct mlx5dv_dr_ns *ns; + struct mlx5dv_dr_domain *domain; resource->flags = flow->group ? 0 : 1; - if (flow->ingress) - ns = sh->rx_ns; + if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + domain = sh->fdb_domain; + else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) + domain = sh->rx_domain; else - ns = sh->tx_ns; + domain = sh->tx_domain; /* Lookup a matching resource from cache. */ LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) { @@ -970,7 +1108,7 @@ flow_dv_encap_decap_resource_register cache_resource->verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat (sh->ctx, cache_resource->reformat_type, - cache_resource->ft_type, ns, cache_resource->flags, + cache_resource->ft_type, domain, cache_resource->flags, cache_resource->size, (cache_resource->size ? cache_resource->buf : NULL)); if (!cache_resource->verbs_action) { @@ -1052,6 +1190,70 @@ flow_dv_jump_tbl_resource_register return 0; } +/** + * Find existing table port ID resource or create and register a new one. + * + * @param dev[in, out] + * Pointer to rte_eth_dev structure. + * @param[in, out] resource + * Pointer to port ID action resource. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ +static int +flow_dv_port_id_action_resource_register + (struct rte_eth_dev *dev, + struct mlx5_flow_dv_port_id_action_resource *resource, + struct mlx5_flow *dev_flow, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_dv_port_id_action_resource *cache_resource; + + /* Lookup a matching resource from cache. */ + LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) { + if (resource->port_id == cache_resource->port_id) { + DRV_LOG(DEBUG, "port id action resource resource %p: " + "refcnt %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + rte_atomic32_inc(&cache_resource->refcnt); + dev_flow->dv.port_id_action = cache_resource; + return 0; + } + } + /* Register new port id action resource. */ + cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + if (!cache_resource) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + *cache_resource = *resource; + cache_resource->action = + mlx5_glue->dr_create_flow_action_dest_vport + (priv->sh->fdb_domain, resource->port_id); + if (!cache_resource->action) { + rte_free(cache_resource); + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create action"); + } + rte_atomic32_init(&cache_resource->refcnt); + rte_atomic32_inc(&cache_resource->refcnt); + LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next); + dev_flow->dv.port_id_action = cache_resource; + DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + return 0; +} + /** * Get the size of specific rte_flow_item_type * @@ -1135,14 +1337,14 @@ static int flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, size_t *size, struct rte_flow_error *error) { - struct ether_hdr *eth = NULL; - struct vlan_hdr *vlan = NULL; - struct ipv4_hdr *ipv4 = NULL; - struct ipv6_hdr *ipv6 = NULL; - struct udp_hdr *udp = NULL; - struct vxlan_hdr *vxlan = NULL; - struct vxlan_gpe_hdr *vxlan_gpe = NULL; - struct gre_hdr *gre = NULL; + struct rte_ether_hdr *eth = NULL; + struct rte_vlan_hdr *vlan = NULL; + struct rte_ipv4_hdr *ipv4 = NULL; + struct rte_ipv6_hdr *ipv6 = NULL; + struct rte_udp_hdr *udp = NULL; + struct rte_vxlan_hdr *vxlan = NULL; + struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL; + struct rte_gre_hdr *gre = NULL; size_t len; size_t temp_size = 0; @@ -1161,20 +1363,20 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, rte_memcpy((void *)&buf[temp_size], items->spec, len); switch (items->type) { case RTE_FLOW_ITEM_TYPE_ETH: - eth = (struct ether_hdr *)&buf[temp_size]; + eth = (struct rte_ether_hdr *)&buf[temp_size]; break; case RTE_FLOW_ITEM_TYPE_VLAN: - vlan = (struct vlan_hdr *)&buf[temp_size]; + vlan = (struct rte_vlan_hdr *)&buf[temp_size]; if (!eth) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, (void *)items->type, "eth header not found"); if (!eth->ether_type) - eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN); + eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN); break; case RTE_FLOW_ITEM_TYPE_IPV4: - ipv4 = (struct ipv4_hdr *)&buf[temp_size]; + ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size]; if (!vlan && !eth) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1182,9 +1384,9 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, "neither eth nor vlan" " header found"); if (vlan && !vlan->eth_proto) - vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4); + vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4); else if (eth && !eth->ether_type) - eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4); + eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4); if (!ipv4->version_ihl) ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION | MLX5_ENCAP_IPV4_IHL_MIN; @@ -1192,7 +1394,7 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF; break; case RTE_FLOW_ITEM_TYPE_IPV6: - ipv6 = (struct ipv6_hdr *)&buf[temp_size]; + ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size]; if (!vlan && !eth) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1200,9 +1402,9 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, "neither eth nor vlan" " header found"); if (vlan && !vlan->eth_proto) - vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6); + vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6); else if (eth && !eth->ether_type) - eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6); + eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6); if (!ipv6->vtc_flow) ipv6->vtc_flow = RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW); @@ -1210,7 +1412,7 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT; break; case RTE_FLOW_ITEM_TYPE_UDP: - udp = (struct udp_hdr *)&buf[temp_size]; + udp = (struct rte_udp_hdr *)&buf[temp_size]; if (!ipv4 && !ipv6) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1222,7 +1424,7 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, ipv6->proto = IPPROTO_UDP; break; case RTE_FLOW_ITEM_TYPE_VXLAN: - vxlan = (struct vxlan_hdr *)&buf[temp_size]; + vxlan = (struct rte_vxlan_hdr *)&buf[temp_size]; if (!udp) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1235,7 +1437,7 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS); break; case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: - vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size]; + vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size]; if (!udp) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1255,7 +1457,7 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, break; case RTE_FLOW_ITEM_TYPE_GRE: case RTE_FLOW_ITEM_TYPE_NVGRE: - gre = (struct gre_hdr *)&buf[temp_size]; + gre = (struct rte_gre_hdr *)&buf[temp_size]; if (!gre->proto) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1286,6 +1488,50 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, return 0; } +static int +flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error) +{ + struct rte_ether_hdr *eth = NULL; + struct rte_vlan_hdr *vlan = NULL; + struct rte_ipv6_hdr *ipv6 = NULL; + struct rte_udp_hdr *udp = NULL; + char *next_hdr; + uint16_t proto; + + eth = (struct rte_ether_hdr *)data; + next_hdr = (char *)(eth + 1); + proto = RTE_BE16(eth->ether_type); + + /* VLAN skipping */ + while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) { + next_hdr += sizeof(struct rte_vlan_hdr); + vlan = (struct rte_vlan_hdr *)next_hdr; + proto = RTE_BE16(vlan->eth_proto); + } + + /* HW calculates IPv4 csum. no need to proceed */ + if (proto == RTE_ETHER_TYPE_IPV4) + return 0; + + /* non IPv4/IPv6 header. not supported */ + if (proto != RTE_ETHER_TYPE_IPV6) { + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Cannot offload non IPv4/IPv6"); + } + + ipv6 = (struct rte_ipv6_hdr *)next_hdr; + + /* ignore non UDP */ + if (ipv6->proto != IPPROTO_UDP) + return 0; + + udp = (struct rte_udp_hdr *)(ipv6 + 1); + udp->dgram_cksum = 0; + + return 0; +} + /** * Convert L2 encap action to DV specification. * @@ -1295,6 +1541,8 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, * Pointer to action structure. * @param[in, out] dev_flow * Pointer to the mlx5_flow. + * @param[in] transfer + * Mark if the flow is E-Switch flow. * @param[out] error * Pointer to the error structure. * @@ -1305,6 +1553,7 @@ static int flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, const struct rte_flow_action *action, struct mlx5_flow *dev_flow, + uint8_t transfer, struct rte_flow_error *error) { const struct rte_flow_item *encap_data; @@ -1312,7 +1561,8 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, struct mlx5_flow_dv_encap_decap_resource res = { .reformat_type = MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL, - .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX, + .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : + MLX5DV_FLOW_TABLE_TYPE_NIC_TX, }; if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { @@ -1320,6 +1570,8 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, (const struct rte_flow_action_raw_encap *)action->conf; res.size = raw_encap_data->size; memcpy(res.buf, raw_encap_data->data, res.size); + if (flow_dv_zero_encap_udp_csum(res.buf, error)) + return -rte_errno; } else { if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) encap_data = @@ -1347,6 +1599,8 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, * Pointer to rte_eth_dev structure. * @param[in, out] dev_flow * Pointer to the mlx5_flow. + * @param[in] transfer + * Mark if the flow is E-Switch flow. * @param[out] error * Pointer to the error structure. * @@ -1356,13 +1610,15 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, static int flow_dv_create_action_l2_decap(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, + uint8_t transfer, struct rte_flow_error *error) { struct mlx5_flow_dv_encap_decap_resource res = { .size = 0, .reformat_type = MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2, - .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX, + .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : + MLX5DV_FLOW_TABLE_TYPE_NIC_RX, }; if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) @@ -1405,8 +1661,11 @@ flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, res.reformat_type = attr->egress ? MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL : MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2; - res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : - MLX5DV_FLOW_TABLE_TYPE_NIC_RX; + if (attr->transfer) + res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; + else + res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : + MLX5DV_FLOW_TABLE_TYPE_NIC_RX; if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1580,6 +1839,96 @@ flow_dv_validate_action_modify_tp(const uint64_t action_flags, return ret; } +/** + * Validate the modify-header actions of increment/decrement + * TCP Sequence-number. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "no TCP item in" + " pattern"); + if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ && + (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) || + (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ && + (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot decrease and increase" + " TCP sequence number" + " at the same time"); + } + return ret; +} + +/** + * Validate the modify-header actions of increment/decrement + * TCP Acknowledgment number. + * + * @param[in] action_flags + * Holds the actions detected until now. + * @param[in] action + * Pointer to the modify action. + * @param[in] item_flags + * Holds the items detected. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags, + const struct rte_flow_action *action, + const uint64_t item_flags, + struct rte_flow_error *error) +{ + int ret = 0; + + ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); + if (!ret) { + if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "no TCP item in" + " pattern"); + if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK && + (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) || + (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK && + (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK))) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot decrease and increase" + " TCP acknowledgment number" + " at the same time"); + } + return ret; +} + /** * Validate the modify-header TTL actions. * @@ -1741,15 +2090,21 @@ flow_dv_modify_hdr_resource_register struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_modify_hdr_resource *cache_resource; + struct mlx5dv_dr_domain *ns; - struct mlx5dv_dr_ns *ns = - resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX ? - sh->tx_ns : sh->rx_ns; - + if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + ns = sh->fdb_domain; + else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) + ns = sh->tx_domain; + else + ns = sh->rx_domain; + resource->flags = + dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; /* Lookup a matching resource from cache. */ LIST_FOREACH(cache_resource, &sh->modify_cmds, next) { if (resource->ft_type == cache_resource->ft_type && resource->actions_num == cache_resource->actions_num && + resource->flags == cache_resource->flags && !memcmp((const void *)resource->actions, (const void *)cache_resource->actions, (resource->actions_num * @@ -1772,7 +2127,7 @@ flow_dv_modify_hdr_resource_register cache_resource->verbs_action = mlx5_glue->dv_create_flow_action_modify_header (sh->ctx, cache_resource->ft_type, - ns, 0, + ns, cache_resource->flags, cache_resource->actions_num * sizeof(cache_resource->actions[0]), (uint64_t *)cache_resource->actions); @@ -1792,6 +2147,8 @@ flow_dv_modify_hdr_resource_register return 0; } +#define MLX5_CNT_CONTAINER_RESIZE 64 + /** * Get or create a flow counter. * @@ -1806,34 +2163,34 @@ flow_dv_modify_hdr_resource_register * pointer to flow counter on success, NULL otherwise and rte_errno is set. */ static struct mlx5_flow_counter * -flow_dv_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) +flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared, + uint32_t id) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_counter *cnt = NULL; - struct mlx5_devx_counter_set *dcs = NULL; - int ret; + struct mlx5_devx_obj *dcs = NULL; if (!priv->config.devx) { - ret = -ENOTSUP; - goto error_exit; + rte_errno = ENOTSUP; + return NULL; } if (shared) { - LIST_FOREACH(cnt, &priv->flow_counters, next) { + TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) { if (cnt->shared && cnt->id == id) { cnt->ref_cnt++; return cnt; } } } + dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); + if (!dcs) + return NULL; cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0); - dcs = rte_calloc(__func__, 1, sizeof(*dcs), 0); - if (!dcs || !cnt) { - ret = -ENOMEM; - goto error_exit; + if (!cnt) { + claim_zero(mlx5_devx_cmd_destroy(cnt->dcs)); + rte_errno = ENOMEM; + return NULL; } - ret = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, dcs); - if (ret) - goto error_exit; struct mlx5_flow_counter tmpl = { .shared = shared, .ref_cnt = 1, @@ -1842,42 +2199,573 @@ flow_dv_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) }; tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0); if (!tmpl.action) { - ret = errno; - goto error_exit; + claim_zero(mlx5_devx_cmd_destroy(cnt->dcs)); + rte_errno = errno; + rte_free(cnt); + return NULL; } *cnt = tmpl; - LIST_INSERT_HEAD(&priv->flow_counters, cnt, next); + TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next); return cnt; -error_exit: - rte_free(cnt); - rte_free(dcs); - rte_errno = -ret; - return NULL; } /** * Release a flow counter. * + * @param[in] dev + * Pointer to the Ethernet device structure. * @param[in] counter * Pointer to the counter handler. */ static void -flow_dv_counter_release(struct mlx5_flow_counter *counter) +flow_dv_counter_release_fallback(struct rte_eth_dev *dev, + struct mlx5_flow_counter *counter) { - int ret; + struct mlx5_priv *priv = dev->data->dev_private; if (!counter) return; if (--counter->ref_cnt == 0) { - ret = mlx5_devx_cmd_flow_counter_free(counter->dcs->obj); - if (ret) - DRV_LOG(ERR, "Failed to free devx counters, %d", ret); - LIST_REMOVE(counter, next); - rte_free(counter->dcs); + TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next); + claim_zero(mlx5_devx_cmd_destroy(counter->dcs)); rte_free(counter); } } +/** + * Query a devx flow counter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] cnt + * Pointer to the flow counter. + * @param[out] pkts + * The statistics value of packets. + * @param[out] bytes + * The statistics value of bytes. + * + * @return + * 0 on success, otherwise a negative errno value and rte_errno is set. + */ +static inline int +_flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused, + struct mlx5_flow_counter *cnt, uint64_t *pkts, + uint64_t *bytes) +{ + return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes, + 0, NULL, NULL, 0); +} + +/** + * Get a pool by a counter. + * + * @param[in] cnt + * Pointer to the counter. + * + * @return + * The counter pool. + */ +static struct mlx5_flow_counter_pool * +flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt) +{ + if (!cnt->batch) { + cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL; + return (struct mlx5_flow_counter_pool *)cnt - 1; + } + return cnt->pool; +} + +/** + * Get a pool by devx counter ID. + * + * @param[in] cont + * Pointer to the counter container. + * @param[in] id + * The counter devx ID. + * + * @return + * The counter pool pointer if exists, NULL otherwise, + */ +static struct mlx5_flow_counter_pool * +flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id) +{ + struct mlx5_flow_counter_pool *pool; + + TAILQ_FOREACH(pool, &cont->pool_list, next) { + int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) * + MLX5_COUNTERS_PER_POOL; + + if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) + return pool; + }; + return NULL; +} + +/** + * Allocate a new memory for the counter values wrapped by all the needed + * management. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] raws_n + * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters. + * + * @return + * The new memory management pointer on success, otherwise NULL and rte_errno + * is set. + */ +static struct mlx5_counter_stats_mem_mng * +flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n) +{ + struct mlx5_ibv_shared *sh = ((struct mlx5_priv *) + (dev->data->dev_private))->sh; + struct mlx5dv_pd dv_pd; + struct mlx5dv_obj dv_obj; + struct mlx5_devx_mkey_attr mkey_attr; + struct mlx5_counter_stats_mem_mng *mem_mng; + volatile struct flow_counter_stats *raw_data; + int size = (sizeof(struct flow_counter_stats) * + MLX5_COUNTERS_PER_POOL + + sizeof(struct mlx5_counter_stats_raw)) * raws_n + + sizeof(struct mlx5_counter_stats_mem_mng); + uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE)); + int i; + + if (!mem) { + rte_errno = ENOMEM; + return NULL; + } + mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; + size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; + mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, + IBV_ACCESS_LOCAL_WRITE); + if (!mem_mng->umem) { + rte_errno = errno; + rte_free(mem); + return NULL; + } + dv_obj.pd.in = sh->pd; + dv_obj.pd.out = &dv_pd; + mlx5_glue->dv_init_obj(&dv_obj, MLX5DV_OBJ_PD); + mkey_attr.addr = (uintptr_t)mem; + mkey_attr.size = size; + mkey_attr.umem_id = mem_mng->umem->umem_id; + mkey_attr.pd = dv_pd.pdn; + mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); + if (!mem_mng->dm) { + mlx5_glue->devx_umem_dereg(mem_mng->umem); + rte_errno = errno; + rte_free(mem); + return NULL; + } + mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); + raw_data = (volatile struct flow_counter_stats *)mem; + for (i = 0; i < raws_n; ++i) { + mem_mng->raws[i].mem_mng = mem_mng; + mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; + } + LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); + return mem_mng; +} + +/** + * Resize a counter container. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] batch + * Whether the pool is for counter that was allocated by batch command. + * + * @return + * The new container pointer on success, otherwise NULL and rte_errno is set. + */ +static struct mlx5_pools_container * +flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_pools_container *cont = + MLX5_CNT_CONTAINER(priv->sh, batch, 0); + struct mlx5_pools_container *new_cont = + MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0); + struct mlx5_counter_stats_mem_mng *mem_mng; + uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE; + uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize; + int i; + + if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) { + /* The last resize still hasn't detected by the host thread. */ + rte_errno = EAGAIN; + return NULL; + } + new_cont->pools = rte_calloc(__func__, 1, mem_size, 0); + if (!new_cont->pools) { + rte_errno = ENOMEM; + return NULL; + } + if (cont->n) + memcpy(new_cont->pools, cont->pools, cont->n * + sizeof(struct mlx5_flow_counter_pool *)); + mem_mng = flow_dv_create_counter_stat_mem_mng(dev, + MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES); + if (!mem_mng) { + rte_free(new_cont->pools); + return NULL; + } + for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) + LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws, + mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + + i, next); + new_cont->n = resize; + rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid)); + TAILQ_INIT(&new_cont->pool_list); + TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next); + new_cont->init_mem_mng = mem_mng; + rte_cio_wmb(); + /* Flip the master container. */ + priv->sh->cmng.mhi[batch] ^= (uint8_t)1; + return new_cont; +} + +/** + * Query a devx flow counter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] cnt + * Pointer to the flow counter. + * @param[out] pkts + * The statistics value of packets. + * @param[out] bytes + * The statistics value of bytes. + * + * @return + * 0 on success, otherwise a negative errno value and rte_errno is set. + */ +static inline int +_flow_dv_query_count(struct rte_eth_dev *dev, + struct mlx5_flow_counter *cnt, uint64_t *pkts, + uint64_t *bytes) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_counter_pool *pool = + flow_dv_counter_pool_get(cnt); + int offset = cnt - &pool->counters_raw[0]; + + if (priv->counter_fallback) + return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes); + + rte_spinlock_lock(&pool->sl); + /* + * The single counters allocation may allocate smaller ID than the + * current allocated in parallel to the host reading. + * In this case the new counter values must be reported as 0. + */ + if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) { + *pkts = 0; + *bytes = 0; + } else { + *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits); + *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes); + } + rte_spinlock_unlock(&pool->sl); + return 0; +} + +/** + * Create and initialize a new counter pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] dcs + * The devX counter handle. + * @param[in] batch + * Whether the pool is for counter that was allocated by batch command. + * + * @return + * A new pool pointer on success, NULL otherwise and rte_errno is set. + */ +static struct mlx5_flow_counter_pool * +flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, + uint32_t batch) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_counter_pool *pool; + struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, + 0); + int16_t n_valid = rte_atomic16_read(&cont->n_valid); + uint32_t size; + + if (cont->n == n_valid) { + cont = flow_dv_container_resize(dev, batch); + if (!cont) + return NULL; + } + size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL * + sizeof(struct mlx5_flow_counter); + pool = rte_calloc(__func__, 1, size, 0); + if (!pool) { + rte_errno = ENOMEM; + return NULL; + } + pool->min_dcs = dcs; + pool->raw = cont->init_mem_mng->raws + n_valid % + MLX5_CNT_CONTAINER_RESIZE; + pool->raw_hw = NULL; + rte_spinlock_init(&pool->sl); + /* + * The generation of the new allocated counters in this pool is 0, 2 in + * the pool generation makes all the counters valid for allocation. + */ + rte_atomic64_set(&pool->query_gen, 0x2); + TAILQ_INIT(&pool->counters); + TAILQ_INSERT_TAIL(&cont->pool_list, pool, next); + cont->pools[n_valid] = pool; + /* Pool initialization must be updated before host thread access. */ + rte_cio_wmb(); + rte_atomic16_add(&cont->n_valid, 1); + return pool; +} + +/** + * Prepare a new counter and/or a new counter pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] cnt_free + * Where to put the pointer of a new counter. + * @param[in] batch + * Whether the pool is for counter that was allocated by batch command. + * + * @return + * The free counter pool pointer and @p cnt_free is set on success, + * NULL otherwise and rte_errno is set. + */ +static struct mlx5_flow_counter_pool * +flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, + struct mlx5_flow_counter **cnt_free, + uint32_t batch) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_counter_pool *pool; + struct mlx5_devx_obj *dcs = NULL; + struct mlx5_flow_counter *cnt; + uint32_t i; + + if (!batch) { + /* bulk_bitmap must be 0 for single counter allocation. */ + dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); + if (!dcs) + return NULL; + pool = flow_dv_find_pool_by_id + (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id); + if (!pool) { + pool = flow_dv_pool_create(dev, dcs, batch); + if (!pool) { + mlx5_devx_cmd_destroy(dcs); + return NULL; + } + } else if (dcs->id < pool->min_dcs->id) { + rte_atomic64_set(&pool->a64_dcs, + (int64_t)(uintptr_t)dcs); + } + cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL]; + TAILQ_INSERT_HEAD(&pool->counters, cnt, next); + cnt->dcs = dcs; + *cnt_free = cnt; + return pool; + } + /* bulk_bitmap is in 128 counters units. */ + if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4) + dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); + if (!dcs) { + rte_errno = ENODATA; + return NULL; + } + pool = flow_dv_pool_create(dev, dcs, batch); + if (!pool) { + mlx5_devx_cmd_destroy(dcs); + return NULL; + } + for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { + cnt = &pool->counters_raw[i]; + cnt->pool = pool; + TAILQ_INSERT_HEAD(&pool->counters, cnt, next); + } + *cnt_free = &pool->counters_raw[0]; + return pool; +} + +/** + * Search for existed shared counter. + * + * @param[in] cont + * Pointer to the relevant counter pool container. + * @param[in] id + * The shared counter ID to search. + * + * @return + * NULL if not existed, otherwise pointer to the shared counter. + */ +static struct mlx5_flow_counter * +flow_dv_counter_shared_search(struct mlx5_pools_container *cont, + uint32_t id) +{ + static struct mlx5_flow_counter *cnt; + struct mlx5_flow_counter_pool *pool; + int i; + + TAILQ_FOREACH(pool, &cont->pool_list, next) { + for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { + cnt = &pool->counters_raw[i]; + if (cnt->ref_cnt && cnt->shared && cnt->id == id) + return cnt; + } + } + return NULL; +} + +/** + * Allocate a flow counter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] shared + * Indicate if this counter is shared with other flows. + * @param[in] id + * Counter identifier. + * @param[in] group + * Counter flow group. + * + * @return + * pointer to flow counter on success, NULL otherwise and rte_errno is set. + */ +static struct mlx5_flow_counter * +flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, + uint16_t group) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_counter_pool *pool = NULL; + struct mlx5_flow_counter *cnt_free = NULL; + /* + * Currently group 0 flow counter cannot be assigned to a flow if it is + * not the first one in the batch counter allocation, so it is better + * to allocate counters one by one for these flows in a separate + * container. + * A counter can be shared between different groups so need to take + * shared counters from the single container. + */ + uint32_t batch = (group && !shared) ? 1 : 0; + struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, + 0); + + if (priv->counter_fallback) + return flow_dv_counter_alloc_fallback(dev, shared, id); + if (!priv->config.devx) { + rte_errno = ENOTSUP; + return NULL; + } + if (shared) { + cnt_free = flow_dv_counter_shared_search(cont, id); + if (cnt_free) { + if (cnt_free->ref_cnt + 1 == 0) { + rte_errno = E2BIG; + return NULL; + } + cnt_free->ref_cnt++; + return cnt_free; + } + } + /* Pools which has a free counters are in the start. */ + TAILQ_FOREACH(pool, &cont->pool_list, next) { + /* + * The free counter reset values must be updated between the + * counter release to the counter allocation, so, at least one + * query must be done in this time. ensure it by saving the + * query generation in the release time. + * The free list is sorted according to the generation - so if + * the first one is not updated, all the others are not + * updated too. + */ + cnt_free = TAILQ_FIRST(&pool->counters); + if (cnt_free && cnt_free->query_gen + 1 < + rte_atomic64_read(&pool->query_gen)) + break; + cnt_free = NULL; + } + if (!cnt_free) { + pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch); + if (!pool) + return NULL; + } + cnt_free->batch = batch; + /* Create a DV counter action only in the first time usage. */ + if (!cnt_free->action) { + uint16_t offset; + struct mlx5_devx_obj *dcs; + + if (batch) { + offset = cnt_free - &pool->counters_raw[0]; + dcs = pool->min_dcs; + } else { + offset = 0; + dcs = cnt_free->dcs; + } + cnt_free->action = mlx5_glue->dv_create_flow_action_counter + (dcs->obj, offset); + if (!cnt_free->action) { + rte_errno = errno; + return NULL; + } + } + /* Update the counter reset values. */ + if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits, + &cnt_free->bytes)) + return NULL; + cnt_free->shared = shared; + cnt_free->ref_cnt = 1; + cnt_free->id = id; + if (!priv->sh->cmng.query_thread_on) + /* Start the asynchronous batch query by the host thread. */ + mlx5_set_query_alarm(priv->sh); + TAILQ_REMOVE(&pool->counters, cnt_free, next); + if (TAILQ_EMPTY(&pool->counters)) { + /* Move the pool to the end of the container pool list. */ + TAILQ_REMOVE(&cont->pool_list, pool, next); + TAILQ_INSERT_TAIL(&cont->pool_list, pool, next); + } + return cnt_free; +} + +/** + * Release a flow counter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] counter + * Pointer to the counter handler. + */ +static void +flow_dv_counter_release(struct rte_eth_dev *dev, + struct mlx5_flow_counter *counter) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + if (!counter) + return; + if (priv->counter_fallback) { + flow_dv_counter_release_fallback(dev, counter); + return; + } + if (--counter->ref_cnt == 0) { + struct mlx5_flow_counter_pool *pool = + flow_dv_counter_pool_get(counter); + + /* Put the counter in the end - the last updated one. */ + TAILQ_INSERT_TAIL(&pool->counters, counter, next); + counter->query_gen = rte_atomic64_read(&pool->query_gen); + } +} + /** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. @@ -1973,6 +2861,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, uint64_t last_item = 0; uint8_t next_protocol = 0xff; int actions_n = 0; + const struct rte_flow_item *gre_item = NULL; + struct rte_flow_item_tcp nic_tcp_mask = { + .hdr = { + .tcp_flags = 0xFF, + .src_port = RTE_BE16(UINT16_MAX), + .dst_port = RTE_BE16(UINT16_MAX), + } + }; if (items == NULL) return -1; @@ -2027,6 +2923,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Reset for inner layer. */ next_protocol = 0xff; } + mlx5_flow_tunnel_ip_check(items, &last_item); break; case RTE_FLOW_ITEM_TYPE_IPV6: ret = mlx5_flow_validate_item_ipv6(items, item_flags, @@ -2048,12 +2945,13 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, /* Reset for inner layer. */ next_protocol = 0xff; } + mlx5_flow_tunnel_ip_check(items, &last_item); break; case RTE_FLOW_ITEM_TYPE_TCP: ret = mlx5_flow_validate_item_tcp (items, item_flags, next_protocol, - &rte_flow_item_tcp_mask, + &nic_tcp_mask, error); if (ret < 0) return ret; @@ -2075,8 +2973,16 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, next_protocol, error); if (ret < 0) return ret; + gre_item = items; last_item = MLX5_FLOW_LAYER_GRE; break; + case RTE_FLOW_ITEM_TYPE_GRE_KEY: + ret = mlx5_flow_validate_item_gre_key + (items, item_flags, gre_item, error); + if (ret < 0) + return ret; + item_flags |= MLX5_FLOW_LAYER_GRE_KEY; + break; case RTE_FLOW_ITEM_TYPE_VXLAN: ret = mlx5_flow_validate_item_vxlan(items, item_flags, error); @@ -2105,7 +3011,23 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, error); if (ret < 0) return ret; - last_item = MLX5_FLOW_ITEM_METADATA; + last_item = MLX5_FLOW_ITEM_METADATA; + break; + case RTE_FLOW_ITEM_TYPE_ICMP: + ret = mlx5_flow_validate_item_icmp(items, item_flags, + next_protocol, + error); + if (ret < 0) + return ret; + item_flags |= MLX5_FLOW_LAYER_ICMP; + break; + case RTE_FLOW_ITEM_TYPE_ICMP6: + ret = mlx5_flow_validate_item_icmp6(items, item_flags, + next_protocol, + error); + if (ret < 0) + return ret; + item_flags |= MLX5_FLOW_LAYER_ICMP6; break; default: return rte_flow_error_set(error, ENOTSUP, @@ -2316,6 +3238,40 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, ++actions_n; action_flags |= MLX5_FLOW_ACTION_JUMP; break; + case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: + ret = flow_dv_validate_action_modify_tcp_seq + (action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? + MLX5_FLOW_ACTION_INC_TCP_SEQ : + MLX5_FLOW_ACTION_DEC_TCP_SEQ; + break; + case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: + ret = flow_dv_validate_action_modify_tcp_ack + (action_flags, + actions, + item_flags, + error); + if (ret < 0) + return ret; + /* Count all modify-header actions as one action. */ + if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) + ++actions_n; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? + MLX5_FLOW_ACTION_INC_TCP_ACK : + MLX5_FLOW_ACTION_DEC_TCP_ACK; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -2393,7 +3349,7 @@ flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused, "not enough memory to create flow"); return NULL; } - flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param); + flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); return flow; } @@ -2417,7 +3373,7 @@ flow_dv_check_valid_spec(void *match_mask, void *match_value) uint8_t *v = match_value; unsigned int i; - for (i = 0; i < MLX5_ST_SZ_DB(fte_match_param); ++i) { + for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) { if (v[i] & ~m[i]) { DRV_LOG(ERR, "match_value differs from match_criteria" @@ -2769,6 +3725,10 @@ flow_dv_translate_item_tcp(void *matcher, void *key, rte_be_to_cpu_16(tcp_m->hdr.dst_port)); MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port)); + MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags, + tcp_m->hdr.tcp_flags); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags, + (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags)); } /** @@ -2818,6 +3778,45 @@ flow_dv_translate_item_udp(void *matcher, void *key, rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port)); } +/** + * Add GRE optional Key item to matcher and to the value. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[in] inner + * Item is inner pattern. + */ +static void +flow_dv_translate_item_gre_key(void *matcher, void *key, + const struct rte_flow_item *item) +{ + const rte_be32_t *key_m = item->mask; + const rte_be32_t *key_v = item->spec; + void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); + void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); + rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); + + if (!key_v) + return; + if (!key_m) + key_m = &gre_key_default_mask; + /* GRE K bit must be on and should already be validated */ + MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1); + MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1); + MLX5_SET(fte_match_set_misc, misc_m, gre_key_h, + rte_be_to_cpu_32(*key_m) >> 8); + MLX5_SET(fte_match_set_misc, misc_v, gre_key_h, + rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8); + MLX5_SET(fte_match_set_misc, misc_m, gre_key_l, + rte_be_to_cpu_32(*key_m) & 0xFF); + MLX5_SET(fte_match_set_misc, misc_v, gre_key_l, + rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF); +} + /** * Add GRE item to matcher and to the value. * @@ -2841,6 +3840,20 @@ flow_dv_translate_item_gre(void *matcher, void *key, void *headers_v; void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); + struct { + union { + __extension__ + struct { + uint16_t version:3; + uint16_t rsvd0:9; + uint16_t s_present:1; + uint16_t k_present:1; + uint16_t rsvd_bit1:1; + uint16_t c_present:1; + }; + uint16_t value; + }; + } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v; if (inner) { headers_m = MLX5_ADDR_OF(fte_match_param, matcher, @@ -2861,6 +3874,23 @@ flow_dv_translate_item_gre(void *matcher, void *key, rte_be_to_cpu_16(gre_m->protocol)); MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol)); + gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver); + gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver); + MLX5_SET(fte_match_set_misc, misc_m, gre_c_present, + gre_crks_rsvd0_ver_m.c_present); + MLX5_SET(fte_match_set_misc, misc_v, gre_c_present, + gre_crks_rsvd0_ver_v.c_present & + gre_crks_rsvd0_ver_m.c_present); + MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, + gre_crks_rsvd0_ver_m.k_present); + MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, + gre_crks_rsvd0_ver_v.k_present & + gre_crks_rsvd0_ver_m.k_present); + MLX5_SET(fte_match_set_misc, misc_m, gre_s_present, + gre_crks_rsvd0_ver_m.s_present); + MLX5_SET(fte_match_set_misc, misc_v, gre_s_present, + gre_crks_rsvd0_ver_v.s_present & + gre_crks_rsvd0_ver_m.s_present); } /** @@ -3001,7 +4031,7 @@ flow_dv_translate_item_mpls(void *matcher, void *key, case MLX5_FLOW_LAYER_GRE: MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff); MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, - ETHER_TYPE_MPLS); + RTE_ETHER_TYPE_MPLS); break; default: MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); @@ -3085,6 +4115,158 @@ flow_dv_translate_item_meta(void *matcher, void *key, } } +/** + * Add source vport match to the specified matcher. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] port + * Source vport value to match + * @param[in] mask + * Mask + */ +static void +flow_dv_translate_item_source_vport(void *matcher, void *key, + int16_t port, uint16_t mask) +{ + void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); + void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); + + MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); + MLX5_SET(fte_match_set_misc, misc_v, source_port, port); +} + +/** + * Translate port-id item to eswitch match on port-id. + * + * @param[in] dev + * The devich to configure through. + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, + void *key, const struct rte_flow_item *item) +{ + const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; + const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; + uint16_t mask, val, id; + int ret; + + mask = pid_m ? pid_m->id : 0xffff; + id = pid_v ? pid_v->id : dev->data->port_id; + ret = mlx5_port_to_eswitch_info(id, NULL, &val); + if (ret) + return ret; + flow_dv_translate_item_source_vport(matcher, key, val, mask); + return 0; +} + +/** + * Add ICMP6 item to matcher and to the value. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[in] inner + * Item is inner pattern. + */ +static void +flow_dv_translate_item_icmp6(void *matcher, void *key, + const struct rte_flow_item *item, + int inner) +{ + const struct rte_flow_item_icmp6 *icmp6_m = item->mask; + const struct rte_flow_item_icmp6 *icmp6_v = item->spec; + void *headers_m; + void *headers_v; + void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, + misc_parameters_3); + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + if (inner) { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + inner_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); + } else { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + outer_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); + } + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6); + if (!icmp6_v) + return; + if (!icmp6_m) + icmp6_m = &rte_flow_item_icmp6_mask; + MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type); + MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type, + icmp6_v->type & icmp6_m->type); + MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code); + MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code, + icmp6_v->code & icmp6_m->code); +} + +/** + * Add ICMP item to matcher and to the value. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * @param[in] inner + * Item is inner pattern. + */ +static void +flow_dv_translate_item_icmp(void *matcher, void *key, + const struct rte_flow_item *item, + int inner) +{ + const struct rte_flow_item_icmp *icmp_m = item->mask; + const struct rte_flow_item_icmp *icmp_v = item->spec; + void *headers_m; + void *headers_v; + void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, + misc_parameters_3); + void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); + if (inner) { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + inner_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); + } else { + headers_m = MLX5_ADDR_OF(fte_match_param, matcher, + outer_headers); + headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); + } + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); + MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP); + if (!icmp_v) + return; + if (!icmp_m) + icmp_m = &rte_flow_item_icmp_mask; + MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type, + icmp_m->hdr.icmp_type); + MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type, + icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type); + MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code, + icmp_m->hdr.icmp_code); + MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code, + icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code); +} + static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; #define HEADER_IS_ZERO(match_criteria, headers) \ @@ -3135,6 +4317,8 @@ flow_dv_matcher_enable(uint32_t *match_criteria) * Table id to use. * @param[in] egress * Direction of the table. + * @param[in] transfer + * E-Switch or NIC flow. * @param[out] error * pointer to error structure. * @@ -3144,6 +4328,7 @@ flow_dv_matcher_enable(uint32_t *match_criteria) static struct mlx5_flow_tbl_resource * flow_dv_tbl_resource_get(struct rte_eth_dev *dev, uint32_t table_id, uint8_t egress, + uint8_t transfer, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; @@ -3151,16 +4336,21 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, struct mlx5_flow_tbl_resource *tbl; #ifdef HAVE_MLX5DV_DR - if (egress) { + if (transfer) { + tbl = &sh->fdb_tbl[table_id]; + if (!tbl->obj) + tbl->obj = mlx5_glue->dr_create_flow_tbl + (sh->fdb_domain, table_id); + } else if (egress) { tbl = &sh->tx_tbl[table_id]; if (!tbl->obj) tbl->obj = mlx5_glue->dr_create_flow_tbl - (sh->tx_ns, table_id); + (sh->tx_domain, table_id); } else { tbl = &sh->rx_tbl[table_id]; if (!tbl->obj) tbl->obj = mlx5_glue->dr_create_flow_tbl - (sh->rx_ns, table_id); + (sh->rx_domain, table_id); } if (!tbl->obj) { rte_flow_error_set(error, ENOMEM, @@ -3173,7 +4363,9 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, #else (void)error; (void)tbl; - if (egress) + if (transfer) + return &sh->fdb_tbl[table_id]; + else if (egress) return &sh->tx_tbl[table_id]; else return &sh->rx_tbl[table_id]; @@ -3238,6 +4430,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, matcher->priority == cache_matcher->priority && matcher->egress == cache_matcher->egress && matcher->group == cache_matcher->group && + matcher->transfer == cache_matcher->transfer && !memcmp((const void *)matcher->mask.buf, (const void *)cache_matcher->mask.buf, cache_matcher->mask.size)) { @@ -3259,7 +4452,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate matcher memory"); tbl = flow_dv_tbl_resource_get(dev, matcher->group * MLX5_GROUP_FACTOR, - matcher->egress, error); + matcher->egress, matcher->transfer, + error); if (!tbl) { rte_free(cache_matcher); return rte_flow_error_set(error, ENOMEM, @@ -3294,29 +4488,6 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, return 0; } -/** - * Add source vport match to the specified matcher. - * - * @param[in, out] matcher - * Flow matcher. - * @param[in, out] key - * Flow matcher value. - * @param[in] port - * Source vport value to match - * @param[in] mask - * Mask - */ -static void -flow_dv_translate_item_source_vport(void *matcher, void *key, - int16_t port, uint16_t mask) -{ - void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); - void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); - - MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); - MLX5_SET(fte_match_set_misc, misc_v, source_port, port); -} - /** * Find existing tag resource or create and register a new one. * @@ -3409,6 +4580,44 @@ flow_dv_tag_release(struct rte_eth_dev *dev, return 1; } +/** + * Translate port ID action to vport. + * + * @param[in] dev + * Pointer to rte_eth_dev structure. + * @param[in] action + * Pointer to the port ID action. + * @param[out] dst_port_id + * The target port ID. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_translate_action_port_id(struct rte_eth_dev *dev, + const struct rte_flow_action *action, + uint32_t *dst_port_id, + struct rte_flow_error *error) +{ + uint32_t port; + uint16_t port_id; + int ret; + const struct rte_flow_action_port_id *conf = + (const struct rte_flow_action_port_id *)action->conf; + + port = conf->original ? dev->data->port_id : conf->id; + ret = mlx5_port_to_eswitch_info(port, NULL, &port_id); + if (ret) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "No eswitch info was found for port"); + *dst_port_id = port_id; + return 0; +} + /** * Fill the flow with DV spec. * @@ -3456,7 +4665,12 @@ flow_dv_translate(struct rte_eth_dev *dev, union flow_dv_attr flow_attr = { .attr = 0 }; struct mlx5_flow_dv_tag_resource tag_resource; uint32_t modify_action_position = UINT32_MAX; + void *match_mask = matcher.mask.buf; + void *match_value = dev_flow->dv.value.buf; + flow->group = attr->group; + if (attr->transfer) + res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; if (priority == MLX5_FLOW_PRIO_RSVD) priority = priv->config.flow_prio - 1; for (; !actions_end ; actions++) { @@ -3468,10 +4682,24 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_jump *jump_data; struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource; struct mlx5_flow_tbl_resource *tbl; + uint32_t port_id = 0; + struct mlx5_flow_dv_port_id_action_resource port_id_resource; switch (actions->type) { case RTE_FLOW_ACTION_TYPE_VOID: break; + case RTE_FLOW_ACTION_TYPE_PORT_ID: + if (flow_dv_translate_action_port_id(dev, action, + &port_id, error)) + return -rte_errno; + port_id_resource.port_id = port_id; + if (flow_dv_port_id_action_resource_register + (dev, &port_id_resource, dev_flow, error)) + return -rte_errno; + dev_flow->dv.actions[actions_n++] = + dev_flow->dv.port_id_action->action; + action_flags |= MLX5_FLOW_ACTION_PORT_ID; + break; case RTE_FLOW_ACTION_TYPE_FLAG: tag_resource.tag = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); @@ -3523,8 +4751,10 @@ flow_dv_translate(struct rte_eth_dev *dev, rte_errno = ENOTSUP; goto cnt_err; } - flow->counter = flow_dv_counter_new(dev, count->shared, - count->id); + flow->counter = flow_dv_counter_alloc(dev, + count->shared, + count->id, + attr->group); if (flow->counter == NULL) goto cnt_err; dev_flow->dv.actions[actions_n++] = @@ -3548,7 +4778,9 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: if (flow_dv_create_action_l2_encap(dev, actions, - dev_flow, error)) + dev_flow, + attr->transfer, + error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = dev_flow->dv.encap_decap->verbs_action; @@ -3560,6 +4792,7 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: if (flow_dv_create_action_l2_decap(dev, dev_flow, + attr->transfer, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = @@ -3579,9 +4812,9 @@ cnt_err: dev_flow->dv.encap_decap->verbs_action; } else { /* Handle encap without preceding decap. */ - if (flow_dv_create_action_l2_encap(dev, actions, - dev_flow, - error)) + if (flow_dv_create_action_l2_encap + (dev, actions, dev_flow, attr->transfer, + error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = dev_flow->dv.encap_decap->verbs_action; @@ -3596,9 +4829,8 @@ cnt_err: } /* Handle decap only if it isn't followed by encap. */ if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { - if (flow_dv_create_action_l2_decap(dev, - dev_flow, - error)) + if (flow_dv_create_action_l2_decap + (dev, dev_flow, attr->transfer, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = dev_flow->dv.encap_decap->verbs_action; @@ -3610,7 +4842,8 @@ cnt_err: jump_data = action->conf; tbl = flow_dv_tbl_resource_get(dev, jump_data->group * MLX5_GROUP_FACTOR, - attr->egress, error); + attr->egress, + attr->transfer, error); if (!tbl) return rte_flow_error_set (error, errno, @@ -3686,6 +4919,27 @@ cnt_err: return -rte_errno; action_flags |= MLX5_FLOW_ACTION_SET_TTL; break; + case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: + if (flow_dv_convert_action_modify_tcp_seq(&res, actions, + error)) + return -rte_errno; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? + MLX5_FLOW_ACTION_INC_TCP_SEQ : + MLX5_FLOW_ACTION_DEC_TCP_SEQ; + break; + + case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: + case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: + if (flow_dv_convert_action_modify_tcp_ack(&res, actions, + error)) + return -rte_errno; + action_flags |= actions->type == + RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? + MLX5_FLOW_ACTION_INC_TCP_ACK : + MLX5_FLOW_ACTION_DEC_TCP_ACK; + break; case RTE_FLOW_ACTION_TYPE_END: actions_end = true; if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) { @@ -3708,25 +4962,15 @@ cnt_err: } dev_flow->dv.actions_n = actions_n; flow->actions = action_flags; - if (attr->ingress && !attr->transfer && - (priv->representor || priv->master)) { - /* It was validated - we support unidirection flows only. */ - assert(!attr->egress); - /* - * Add matching on source vport index only - * for ingress rules in E-Switch configurations. - */ - flow_dv_translate_item_source_vport(matcher.mask.buf, - dev_flow->dv.value.buf, - priv->vport_id, - 0xffff); - } for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); - void *match_mask = matcher.mask.buf; - void *match_value = dev_flow->dv.value.buf; switch (items->type) { + case RTE_FLOW_ITEM_TYPE_PORT_ID: + flow_dv_translate_item_port_id(dev, match_mask, + match_value, items); + last_item = MLX5_FLOW_ITEM_PORT_ID; + break; case RTE_FLOW_ITEM_TYPE_ETH: flow_dv_translate_item_eth(match_mask, match_value, items, tunnel); @@ -3754,6 +4998,7 @@ cnt_err: MLX5_IPV4_IBV_RX_HASH); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : MLX5_FLOW_LAYER_OUTER_L3_IPV4; + mlx5_flow_tunnel_ip_check(items, &last_item); break; case RTE_FLOW_ITEM_TYPE_IPV6: flow_dv_translate_item_ipv6(match_mask, match_value, @@ -3766,6 +5011,7 @@ cnt_err: MLX5_IPV6_IBV_RX_HASH); last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : MLX5_FLOW_LAYER_OUTER_L3_IPV6; + mlx5_flow_tunnel_ip_check(items, &last_item); break; case RTE_FLOW_ITEM_TYPE_TCP: flow_dv_translate_item_tcp(match_mask, match_value, @@ -3796,6 +5042,11 @@ cnt_err: items, tunnel); last_item = MLX5_FLOW_LAYER_GRE; break; + case RTE_FLOW_ITEM_TYPE_GRE_KEY: + flow_dv_translate_item_gre_key(match_mask, + match_value, items); + item_flags |= MLX5_FLOW_LAYER_GRE_KEY; + break; case RTE_FLOW_ITEM_TYPE_NVGRE: flow_dv_translate_item_nvgre(match_mask, match_value, items, tunnel); @@ -3821,11 +5072,34 @@ cnt_err: items); last_item = MLX5_FLOW_ITEM_METADATA; break; + case RTE_FLOW_ITEM_TYPE_ICMP: + flow_dv_translate_item_icmp(match_mask, match_value, + items, tunnel); + item_flags |= MLX5_FLOW_LAYER_ICMP; + break; + case RTE_FLOW_ITEM_TYPE_ICMP6: + flow_dv_translate_item_icmp6(match_mask, match_value, + items, tunnel); + item_flags |= MLX5_FLOW_LAYER_ICMP6; + break; default: break; } item_flags |= last_item; } + /* + * In case of ingress traffic when E-Switch mode is enabled, + * we have two cases where we need to set the source port manually. + * The first one, is in case of Nic steering rule, and the second is + * E-Switch rule where no port_id item was found. In both cases + * the source port is set according the current port in use. + */ + if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) && + (priv->representor || priv->master)) { + if (flow_dv_translate_item_port_id(dev, match_mask, + match_value, NULL)) + return -rte_errno; + } assert(!flow_dv_check_valid_spec(matcher.mask.buf, dev_flow->dv.value.buf)); dev_flow->layers = item_flags; @@ -3836,6 +5110,7 @@ cnt_err: matcher.priority); matcher.egress = attr->egress; matcher.group = attr->group; + matcher.transfer = attr->transfer; if (flow_dv_matcher_register(dev, &matcher, dev_flow, error)) return -rte_errno; return 0; @@ -3860,6 +5135,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, { struct mlx5_flow_dv *dv; struct mlx5_flow *dev_flow; + struct mlx5_priv *priv = dev->data->dev_private; int n; int err; @@ -3867,15 +5143,20 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, dv = &dev_flow->dv; n = dv->actions_n; if (flow->actions & MLX5_FLOW_ACTION_DROP) { - dv->hrxq = mlx5_hrxq_drop_new(dev); - if (!dv->hrxq) { - rte_flow_error_set - (error, errno, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot get drop hash queue"); - goto error; + if (flow->transfer) { + dv->actions[n++] = priv->sh->esw_drop_action; + } else { + dv->hrxq = mlx5_hrxq_drop_new(dev); + if (!dv->hrxq) { + rte_flow_error_set + (error, errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot get drop hash queue"); + goto error; + } + dv->actions[n++] = dv->hrxq->action; } - dv->actions[n++] = dv->hrxq->action; } else if (flow->actions & (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) { struct mlx5_hrxq *hrxq; @@ -4066,6 +5347,37 @@ flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow) return 1; } +/** + * Release port ID action resource. + * + * @param flow + * Pointer to mlx5_flow. + * + * @return + * 1 while a reference on it exists, 0 when freed. + */ +static int +flow_dv_port_id_action_resource_release(struct mlx5_flow *flow) +{ + struct mlx5_flow_dv_port_id_action_resource *cache_resource = + flow->dv.port_id_action; + + assert(cache_resource->action); + DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { + claim_zero(mlx5_glue->destroy_flow_action + (cache_resource->action)); + LIST_REMOVE(cache_resource, next); + rte_free(cache_resource); + DRV_LOG(DEBUG, "port id action resource %p: removed", + (void *)cache_resource); + return 0; + } + return 1; +} + /** * Remove the flow from the NIC but keeps it in memory. * @@ -4115,7 +5427,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) return; flow_dv_remove(dev, flow); if (flow->counter) { - flow_dv_counter_release(flow->counter); + flow_dv_counter_release(dev, flow->counter); flow->counter = NULL; } if (flow->tag_resource) { @@ -4133,6 +5445,8 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_modify_hdr_resource_release(dev_flow); if (dev_flow->dv.jump) flow_dv_jump_tbl_resource_release(dev_flow); + if (dev_flow->dv.port_id_action) + flow_dv_port_id_action_resource_release(dev_flow); rte_free(dev_flow); } } @@ -4158,9 +5472,6 @@ flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_query_count *qc = data; - uint64_t pkts = 0; - uint64_t bytes = 0; - int err; if (!priv->config.devx) return rte_flow_error_set(error, ENOTSUP, @@ -4168,15 +5479,14 @@ flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, NULL, "counters are not supported"); if (flow->counter) { - err = mlx5_devx_cmd_flow_counter_query - (flow->counter->dcs, - qc->reset, &pkts, &bytes); + uint64_t pkts, bytes; + int err = _flow_dv_query_count(dev, flow->counter, &pkts, + &bytes); + if (err) - return rte_flow_error_set - (error, err, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "cannot read counters"); + return rte_flow_error_set(error, -err, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot read counters"); qc->hits_set = 1; qc->bytes_set = 1; qc->hits = pkts - flow->counter->hits;