X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=b7a56ebdc946286795353257c77c28c79713fb7f;hb=be11774d453ba9fee825c1055b6602cef4f0afb9;hp=d9e2ac9677be2284d331d4666c5742b5649609ad;hpb=cbb66daa3c859a14d4e5f993fc71bf7026f2ec8a;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index d9e2ac9677..b7a56ebdc9 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -29,15 +28,27 @@ #include "mlx5.h" #include "mlx5_defs.h" -#include "mlx5_prm.h" #include "mlx5_glue.h" #include "mlx5_flow.h" +#include "mlx5_prm.h" +#include "mlx5_rxtx.h" + #ifdef HAVE_IBV_FLOW_DV_SUPPORT #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 #endif +#ifndef HAVE_MLX5DV_DR_ESWITCH +#ifndef MLX5DV_FLOW_TABLE_TYPE_FDB +#define MLX5DV_FLOW_TABLE_TYPE_FDB 0 +#endif +#endif + +#ifndef HAVE_MLX5DV_DR +#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1 +#endif + union flow_dv_attr { struct { uint32_t valid:1; @@ -128,6 +139,45 @@ struct field_modify_info modify_tcp[] = { {0, 0, 0}, }; +/** + * Acquire the synchronizing object to protect multithreaded access + * to shared dv context. Lock occurs only if context is actually + * shared, i.e. we have multiport IB device and representors are + * created. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + */ +static void +flow_d_shared_lock(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + + if (sh->dv_refcnt > 1) { + int ret; + + ret = pthread_mutex_lock(&sh->dv_mutex); + assert(!ret); + (void)ret; + } +} + +static void +flow_d_shared_unlock(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + + if (sh->dv_refcnt > 1) { + int ret; + + ret = pthread_mutex_unlock(&sh->dv_mutex); + assert(!ret); + (void)ret; + } +} + /** * Convert modify-header action to DV specification. * @@ -571,6 +621,89 @@ flow_dv_validate_item_meta(struct rte_eth_dev *dev, return 0; } +/** + * Validate vport item. + * + * @param[in] dev + * Pointer to the rte_eth_dev structure. + * @param[in] item + * Item specification. + * @param[in] attr + * Attributes of flow that includes this item. + * @param[in] item_flags + * Bit-fields that holds the items detected until now. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_item_port_id(struct rte_eth_dev *dev, + const struct rte_flow_item *item, + const struct rte_flow_attr *attr, + uint64_t item_flags, + struct rte_flow_error *error) +{ + const struct rte_flow_item_port_id *spec = item->spec; + const struct rte_flow_item_port_id *mask = item->mask; + const struct rte_flow_item_port_id switch_mask = { + .id = 0xffffffff, + }; + uint16_t esw_domain_id; + uint16_t item_port_esw_domain_id; + int ret; + + if (!attr->transfer) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, + "match on port id is valid only" + " when transfer flag is enabled"); + if (item_flags & MLX5_FLOW_ITEM_PORT_ID) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "multiple source ports are not" + " supported"); + if (!mask) + mask = &switch_mask; + if (mask->id != 0xffffffff) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM_MASK, + mask, + "no support for partial mask on" + " \"id\" field"); + ret = mlx5_flow_item_acceptable + (item, (const uint8_t *)mask, + (const uint8_t *)&rte_flow_item_port_id_mask, + sizeof(struct rte_flow_item_port_id), + error); + if (ret) + return ret; + if (!spec) + return 0; + ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id, + NULL); + if (ret) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, + "failed to obtain E-Switch info for" + " port"); + ret = mlx5_port_to_eswitch_info(dev->data->port_id, + &esw_domain_id, NULL); + if (ret < 0) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "failed to obtain E-Switch info"); + if (item_port_esw_domain_id != esw_domain_id) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, + "cannot match on a port from a" + " different E-Switch"); + return 0; +} + /** * Validate count action. * @@ -635,7 +768,7 @@ flow_dv_validate_action_l2_encap(uint64_t action_flags, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "can only have a single encap or" " decap action in a flow"); - if (attr->ingress) + if (!attr->transfer && attr->ingress) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, NULL, @@ -720,7 +853,8 @@ flow_dv_validate_action_raw_encap(uint64_t action_flags, "can only have a single encap" " action in a flow"); /* encap without preceding decap is not supported for ingress */ - if (attr->ingress && !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP)) + if (!attr->transfer && attr->ingress && + !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP)) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, NULL, @@ -808,12 +942,24 @@ flow_dv_encap_decap_resource_register struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_encap_decap_resource *cache_resource; + struct rte_flow *flow = dev_flow->flow; + struct mlx5dv_dr_domain *domain; + + resource->flags = flow->group ? 0 : 1; + if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + domain = sh->fdb_domain; + else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) + domain = sh->rx_domain; + else + domain = sh->tx_domain; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &priv->encaps_decaps, next) { + LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) { if (resource->reformat_type == cache_resource->reformat_type && resource->ft_type == cache_resource->ft_type && + resource->flags == cache_resource->flags && resource->size == cache_resource->size && !memcmp((const void *)resource->buf, (const void *)cache_resource->buf, @@ -835,10 +981,10 @@ flow_dv_encap_decap_resource_register *cache_resource = *resource; cache_resource->verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat - (priv->sh->ctx, cache_resource->size, - (cache_resource->size ? cache_resource->buf : NULL), - cache_resource->reformat_type, - cache_resource->ft_type); + (sh->ctx, cache_resource->reformat_type, + cache_resource->ft_type, domain, cache_resource->flags, + cache_resource->size, + (cache_resource->size ? cache_resource->buf : NULL)); if (!cache_resource->verbs_action) { rte_free(cache_resource); return rte_flow_error_set(error, ENOMEM, @@ -847,7 +993,7 @@ flow_dv_encap_decap_resource_register } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&priv->encaps_decaps, cache_resource, next); + LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next); dev_flow->dv.encap_decap = cache_resource; DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", (void *)cache_resource, @@ -855,6 +1001,133 @@ flow_dv_encap_decap_resource_register return 0; } +/** + * Find existing table jump resource or create and register a new one. + * + * @param dev[in, out] + * Pointer to rte_eth_dev structure. + * @param[in, out] resource + * Pointer to jump table resource. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ +static int +flow_dv_jump_tbl_resource_register + (struct rte_eth_dev *dev, + struct mlx5_flow_dv_jump_tbl_resource *resource, + struct mlx5_flow *dev_flow, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_dv_jump_tbl_resource *cache_resource; + + /* Lookup a matching resource from cache. */ + LIST_FOREACH(cache_resource, &sh->jump_tbl, next) { + if (resource->tbl == cache_resource->tbl) { + DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + rte_atomic32_inc(&cache_resource->refcnt); + dev_flow->dv.jump = cache_resource; + return 0; + } + } + /* Register new jump table resource. */ + cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + if (!cache_resource) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + *cache_resource = *resource; + cache_resource->action = + mlx5_glue->dr_create_flow_action_dest_flow_tbl + (resource->tbl->obj); + if (!cache_resource->action) { + rte_free(cache_resource); + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create action"); + } + rte_atomic32_init(&cache_resource->refcnt); + rte_atomic32_inc(&cache_resource->refcnt); + LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next); + dev_flow->dv.jump = cache_resource; + DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + return 0; +} + +/** + * Find existing table port ID resource or create and register a new one. + * + * @param dev[in, out] + * Pointer to rte_eth_dev structure. + * @param[in, out] resource + * Pointer to port ID action resource. + * @parm[in, out] dev_flow + * Pointer to the dev_flow. + * @param[out] error + * pointer to error structure. + * + * @return + * 0 on success otherwise -errno and errno is set. + */ +static int +flow_dv_port_id_action_resource_register + (struct rte_eth_dev *dev, + struct mlx5_flow_dv_port_id_action_resource *resource, + struct mlx5_flow *dev_flow, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_dv_port_id_action_resource *cache_resource; + + /* Lookup a matching resource from cache. */ + LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) { + if (resource->port_id == cache_resource->port_id) { + DRV_LOG(DEBUG, "port id action resource resource %p: " + "refcnt %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + rte_atomic32_inc(&cache_resource->refcnt); + dev_flow->dv.port_id_action = cache_resource; + return 0; + } + } + /* Register new port id action resource. */ + cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); + if (!cache_resource) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "cannot allocate resource memory"); + *cache_resource = *resource; + cache_resource->action = + mlx5_glue->dr_create_flow_action_dest_vport + (priv->sh->fdb_domain, resource->port_id); + if (!cache_resource->action) { + rte_free(cache_resource); + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create action"); + } + rte_atomic32_init(&cache_resource->refcnt); + rte_atomic32_inc(&cache_resource->refcnt); + LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next); + dev_flow->dv.port_id_action = cache_resource; + DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + return 0; +} + /** * Get the size of specific rte_flow_item_type * @@ -938,14 +1211,14 @@ static int flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, size_t *size, struct rte_flow_error *error) { - struct ether_hdr *eth = NULL; - struct vlan_hdr *vlan = NULL; + struct rte_ether_hdr *eth = NULL; + struct rte_vlan_hdr *vlan = NULL; struct ipv4_hdr *ipv4 = NULL; struct ipv6_hdr *ipv6 = NULL; struct udp_hdr *udp = NULL; - struct vxlan_hdr *vxlan = NULL; - struct vxlan_gpe_hdr *vxlan_gpe = NULL; - struct gre_hdr *gre = NULL; + struct rte_vxlan_hdr *vxlan = NULL; + struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL; + struct rte_gre_hdr *gre = NULL; size_t len; size_t temp_size = 0; @@ -964,17 +1237,17 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, rte_memcpy((void *)&buf[temp_size], items->spec, len); switch (items->type) { case RTE_FLOW_ITEM_TYPE_ETH: - eth = (struct ether_hdr *)&buf[temp_size]; + eth = (struct rte_ether_hdr *)&buf[temp_size]; break; case RTE_FLOW_ITEM_TYPE_VLAN: - vlan = (struct vlan_hdr *)&buf[temp_size]; + vlan = (struct rte_vlan_hdr *)&buf[temp_size]; if (!eth) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, (void *)items->type, "eth header not found"); if (!eth->ether_type) - eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN); + eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN); break; case RTE_FLOW_ITEM_TYPE_IPV4: ipv4 = (struct ipv4_hdr *)&buf[temp_size]; @@ -985,9 +1258,9 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, "neither eth nor vlan" " header found"); if (vlan && !vlan->eth_proto) - vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4); + vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPv4); else if (eth && !eth->ether_type) - eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4); + eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPv4); if (!ipv4->version_ihl) ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION | MLX5_ENCAP_IPV4_IHL_MIN; @@ -1003,9 +1276,9 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, "neither eth nor vlan" " header found"); if (vlan && !vlan->eth_proto) - vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6); + vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPv6); else if (eth && !eth->ether_type) - eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6); + eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPv6); if (!ipv6->vtc_flow) ipv6->vtc_flow = RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW); @@ -1025,7 +1298,7 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, ipv6->proto = IPPROTO_UDP; break; case RTE_FLOW_ITEM_TYPE_VXLAN: - vxlan = (struct vxlan_hdr *)&buf[temp_size]; + vxlan = (struct rte_vxlan_hdr *)&buf[temp_size]; if (!udp) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1038,7 +1311,7 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS); break; case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: - vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size]; + vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size]; if (!udp) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1058,7 +1331,7 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, break; case RTE_FLOW_ITEM_TYPE_GRE: case RTE_FLOW_ITEM_TYPE_NVGRE: - gre = (struct gre_hdr *)&buf[temp_size]; + gre = (struct rte_gre_hdr *)&buf[temp_size]; if (!gre->proto) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1098,6 +1371,8 @@ flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, * Pointer to action structure. * @param[in, out] dev_flow * Pointer to the mlx5_flow. + * @param[in] transfer + * Mark if the flow is E-Switch flow. * @param[out] error * Pointer to the error structure. * @@ -1108,6 +1383,7 @@ static int flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, const struct rte_flow_action *action, struct mlx5_flow *dev_flow, + uint8_t transfer, struct rte_flow_error *error) { const struct rte_flow_item *encap_data; @@ -1115,7 +1391,8 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, struct mlx5_flow_dv_encap_decap_resource res = { .reformat_type = MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL, - .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX, + .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : + MLX5DV_FLOW_TABLE_TYPE_NIC_TX, }; if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { @@ -1150,6 +1427,8 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, * Pointer to rte_eth_dev structure. * @param[in, out] dev_flow * Pointer to the mlx5_flow. + * @param[in] transfer + * Mark if the flow is E-Switch flow. * @param[out] error * Pointer to the error structure. * @@ -1159,13 +1438,15 @@ flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, static int flow_dv_create_action_l2_decap(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, + uint8_t transfer, struct rte_flow_error *error) { struct mlx5_flow_dv_encap_decap_resource res = { .size = 0, .reformat_type = MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2, - .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX, + .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : + MLX5DV_FLOW_TABLE_TYPE_NIC_RX, }; if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) @@ -1208,8 +1489,11 @@ flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, res.reformat_type = attr->egress ? MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL : MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2; - res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : - MLX5DV_FLOW_TABLE_TYPE_NIC_RX; + if (attr->transfer) + res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; + else + res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : + MLX5DV_FLOW_TABLE_TYPE_NIC_RX; if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1417,6 +1701,108 @@ flow_dv_validate_action_modify_ttl(const uint64_t action_flags, return ret; } +/** + * Validate jump action. + * + * @param[in] action + * Pointer to the modify action. + * @param[in] group + * The group of the current flow. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_jump(const struct rte_flow_action *action, + uint32_t group, + struct rte_flow_error *error) +{ + if (action->type != RTE_FLOW_ACTION_TYPE_JUMP && !action->conf) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, "action configuration not set"); + if (group >= ((const struct rte_flow_action_jump *)action->conf)->group) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "target group must be higher then" + " the current flow group"); + return 0; +} + +/* + * Validate the port_id action. + * + * @param[in] dev + * Pointer to rte_eth_dev structure. + * @param[in] action_flags + * Bit-fields that holds the actions detected until now. + * @param[in] action + * Port_id RTE action structure. + * @param[in] attr + * Attributes of flow that includes this action. + * @param[out] error + * Pointer to error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_validate_action_port_id(struct rte_eth_dev *dev, + uint64_t action_flags, + const struct rte_flow_action *action, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + const struct rte_flow_action_port_id *port_id; + uint16_t port; + uint16_t esw_domain_id; + uint16_t act_port_domain_id; + int ret; + + if (!attr->transfer) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "port id action is valid in transfer" + " mode only"); + if (!action || !action->conf) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, + "port id action parameters must be" + " specified"); + if (action_flags & (MLX5_FLOW_FATE_ACTIONS | + MLX5_FLOW_FATE_ESWITCH_ACTIONS)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "can have only one fate actions in" + " a flow"); + ret = mlx5_port_to_eswitch_info(dev->data->port_id, + &esw_domain_id, NULL); + if (ret < 0) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "failed to obtain E-Switch info"); + port_id = action->conf; + port = port_id->original ? dev->data->port_id : port_id->id; + ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL); + if (ret) + return rte_flow_error_set + (error, -ret, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id, + "failed to obtain E-Switch port id for port"); + if (act_port_domain_id != esw_domain_id) + return rte_flow_error_set + (error, -ret, + RTE_FLOW_ERROR_TYPE_ACTION, NULL, + "port does not belong to" + " E-Switch being configured"); + return 0; +} + /** * Find existing modify-header resource or create and register a new one. * @@ -1440,12 +1826,23 @@ flow_dv_modify_hdr_resource_register struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_modify_hdr_resource *cache_resource; - + struct mlx5dv_dr_domain *ns; + + if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) + ns = sh->fdb_domain; + else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) + ns = sh->tx_domain; + else + ns = sh->rx_domain; + resource->flags = + dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &priv->modify_cmds, next) { + LIST_FOREACH(cache_resource, &sh->modify_cmds, next) { if (resource->ft_type == cache_resource->ft_type && resource->actions_num == cache_resource->actions_num && + resource->flags == cache_resource->flags && !memcmp((const void *)resource->actions, (const void *)cache_resource->actions, (resource->actions_num * @@ -1467,11 +1864,11 @@ flow_dv_modify_hdr_resource_register *cache_resource = *resource; cache_resource->verbs_action = mlx5_glue->dv_create_flow_action_modify_header - (priv->sh->ctx, + (sh->ctx, cache_resource->ft_type, + ns, cache_resource->flags, cache_resource->actions_num * sizeof(cache_resource->actions[0]), - (uint64_t *)cache_resource->actions, - cache_resource->ft_type); + (uint64_t *)cache_resource->actions); if (!cache_resource->verbs_action) { rte_free(cache_resource); return rte_flow_error_set(error, ENOMEM, @@ -1480,7 +1877,7 @@ flow_dv_modify_hdr_resource_register } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&priv->modify_cmds, cache_resource, next); + LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next); dev_flow->dv.modify_hdr = cache_resource; DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", (void *)cache_resource, @@ -1596,22 +1993,42 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; uint32_t priority_max = priv->config.flow_prio - 1; +#ifndef HAVE_MLX5DV_DR if (attributes->group) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, "groups is not supported"); +#endif if (attributes->priority != MLX5_FLOW_PRIO_RSVD && attributes->priority >= priority_max) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, NULL, "priority out of range"); - if (attributes->transfer) - return rte_flow_error_set(error, ENOTSUP, - RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, - NULL, - "transfer is not supported"); + if (attributes->transfer) { + if (!priv->config.dv_esw_en) + return rte_flow_error_set + (error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "E-Switch dr is not supported"); + if (!(priv->representor || priv->master)) + return rte_flow_error_set + (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "E-Switch configurationd can only be" + " done by a master or a representor device"); + if (attributes->egress) + return rte_flow_error_set + (error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes, + "egress is not supported"); + if (attributes->group >= MLX5_MAX_TABLES_FDB) + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, + NULL, "group must be smaller than " + RTE_STR(MLX5_MAX_FDB_TABLES)); + } if (!(attributes->egress ^ attributes->ingress)) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR, NULL, @@ -1647,7 +2064,6 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, uint64_t action_flags = 0; uint64_t item_flags = 0; uint64_t last_item = 0; - int tunnel = 0; uint8_t next_protocol = 0xff; int actions_n = 0; @@ -1657,10 +2073,17 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (ret < 0) return ret; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { - tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); + int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); switch (items->type) { case RTE_FLOW_ITEM_TYPE_VOID: break; + case RTE_FLOW_ITEM_TYPE_PORT_ID: + ret = flow_dv_validate_item_port_id + (dev, items, attr, item_flags, error); + if (ret < 0) + return ret; + last_item |= MLX5_FLOW_ITEM_PORT_ID; + break; case RTE_FLOW_ITEM_TYPE_ETH: ret = mlx5_flow_validate_item_eth(items, item_flags, error); @@ -1792,6 +2215,17 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_VOID: break; + case RTE_FLOW_ACTION_TYPE_PORT_ID: + ret = flow_dv_validate_action_port_id(dev, + action_flags, + actions, + attr, + error); + if (ret) + return ret; + action_flags |= MLX5_FLOW_ACTION_PORT_ID; + ++actions_n; + break; case RTE_FLOW_ACTION_TYPE_FLAG: ret = mlx5_flow_validate_action_flag(action_flags, attr, error); @@ -1829,7 +2263,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case RTE_FLOW_ACTION_TYPE_RSS: ret = mlx5_flow_validate_action_rss(actions, action_flags, dev, - attr, error); + attr, item_flags, + error); if (ret < 0) return ret; action_flags |= MLX5_FLOW_ACTION_RSS; @@ -1966,6 +2401,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, MLX5_FLOW_ACTION_SET_TTL : MLX5_FLOW_ACTION_DEC_TTL; break; + case RTE_FLOW_ACTION_TYPE_JUMP: + ret = flow_dv_validate_action_jump(actions, + attr->group, error); + if (ret) + return ret; + ++actions_n; + action_flags |= MLX5_FLOW_ACTION_JUMP; + break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1973,10 +2416,40 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, "action not supported"); } } - if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress) - return rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_ACTION, actions, - "no fate action is found"); + /* Eswitch has few restrictions on using items and actions */ + if (attr->transfer) { + if (action_flags & MLX5_FLOW_ACTION_FLAG) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported action FLAG"); + if (action_flags & MLX5_FLOW_ACTION_MARK) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported action MARK"); + if (action_flags & MLX5_FLOW_ACTION_QUEUE) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported action QUEUE"); + if (action_flags & MLX5_FLOW_ACTION_RSS) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "unsupported action RSS"); + if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "no fate action is found"); + } else { + if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "no fate action is found"); + } return 0; } @@ -2173,11 +2646,13 @@ flow_dv_translate_item_vlan(void *matcher, void *key, * Flow pattern to translate. * @param[in] inner * Item is inner pattern. + * @param[in] group + * The group to insert the rule. */ static void flow_dv_translate_item_ipv4(void *matcher, void *key, const struct rte_flow_item *item, - int inner) + int inner, uint32_t group) { const struct rte_flow_item_ipv4 *ipv4_m = item->mask; const struct rte_flow_item_ipv4 *ipv4_v = item->spec; @@ -2204,7 +2679,10 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, outer_headers); headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); } - MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); + if (group == 0) + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); + else + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4); if (!ipv4_v) return; @@ -2246,11 +2724,13 @@ flow_dv_translate_item_ipv4(void *matcher, void *key, * Flow pattern to translate. * @param[in] inner * Item is inner pattern. + * @param[in] group + * The group to insert the rule. */ static void flow_dv_translate_item_ipv6(void *matcher, void *key, const struct rte_flow_item *item, - int inner) + int inner, uint32_t group) { const struct rte_flow_item_ipv6 *ipv6_m = item->mask; const struct rte_flow_item_ipv6 *ipv6_v = item->spec; @@ -2287,7 +2767,10 @@ flow_dv_translate_item_ipv6(void *matcher, void *key, outer_headers); headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); } - MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); + if (group == 0) + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); + else + MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6); MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6); if (!ipv6_v) return; @@ -2611,7 +3094,7 @@ flow_dv_translate_item_mpls(void *matcher, void *key, case MLX5_FLOW_LAYER_GRE: MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff); MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, - ETHER_TYPE_MPLS); + RTE_ETHER_TYPE_MPLS); break; default: MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); @@ -2695,6 +3178,62 @@ flow_dv_translate_item_meta(void *matcher, void *key, } } +/** + * Add source vport match to the specified matcher. + * + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] port + * Source vport value to match + * @param[in] mask + * Mask + */ +static void +flow_dv_translate_item_source_vport(void *matcher, void *key, + int16_t port, uint16_t mask) +{ + void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); + void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); + + MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); + MLX5_SET(fte_match_set_misc, misc_v, source_port, port); +} + +/** + * Translate port-id item to eswitch match on port-id. + * + * @param[in] dev + * The devich to configure through. + * @param[in, out] matcher + * Flow matcher. + * @param[in, out] key + * Flow matcher value. + * @param[in] item + * Flow pattern to translate. + * + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, + void *key, const struct rte_flow_item *item) +{ + const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; + const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; + uint16_t mask, val, id; + int ret; + + mask = pid_m ? pid_m->id : 0xffff; + id = pid_v ? pid_v->id : dev->data->port_id; + ret = mlx5_port_to_eswitch_info(id, NULL, &val); + if (ret) + return ret; + flow_dv_translate_item_source_vport(matcher, key, val, mask); + return 0; +} + static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; #define HEADER_IS_ZERO(match_criteria, headers) \ @@ -2727,10 +3266,101 @@ flow_dv_matcher_enable(uint32_t *match_criteria) match_criteria_enable |= (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - +#ifdef HAVE_MLX5DV_DR + match_criteria_enable |= + (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) << + MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT; +#endif return match_criteria_enable; } + +/** + * Get a flow table. + * + * @param dev[in, out] + * Pointer to rte_eth_dev structure. + * @param[in] table_id + * Table id to use. + * @param[in] egress + * Direction of the table. + * @param[in] transfer + * E-Switch or NIC flow. + * @param[out] error + * pointer to error structure. + * + * @return + * Returns tables resource based on the index, NULL in case of failed. + */ +static struct mlx5_flow_tbl_resource * +flow_dv_tbl_resource_get(struct rte_eth_dev *dev, + uint32_t table_id, uint8_t egress, + uint8_t transfer, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_tbl_resource *tbl; + +#ifdef HAVE_MLX5DV_DR + if (transfer) { + tbl = &sh->fdb_tbl[table_id]; + if (!tbl->obj) + tbl->obj = mlx5_glue->dr_create_flow_tbl + (sh->fdb_domain, table_id); + } else if (egress) { + tbl = &sh->tx_tbl[table_id]; + if (!tbl->obj) + tbl->obj = mlx5_glue->dr_create_flow_tbl + (sh->tx_domain, table_id); + } else { + tbl = &sh->rx_tbl[table_id]; + if (!tbl->obj) + tbl->obj = mlx5_glue->dr_create_flow_tbl + (sh->rx_domain, table_id); + } + if (!tbl->obj) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create table"); + return NULL; + } + rte_atomic32_inc(&tbl->refcnt); + return tbl; +#else + (void)error; + (void)tbl; + if (transfer) + return &sh->fdb_tbl[table_id]; + else if (egress) + return &sh->tx_tbl[table_id]; + else + return &sh->rx_tbl[table_id]; +#endif +} + +/** + * Release a flow table. + * + * @param[in] tbl + * Table resource to be released. + * + * @return + * Returns 0 if table was released, else return 1; + */ +static int +flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl) +{ + if (!tbl) + return 0; + if (rte_atomic32_dec_and_test(&tbl->refcnt)) { + mlx5_glue->dr_destroy_flow_tbl(tbl->obj); + tbl->obj = NULL; + return 0; + } + return 1; +} + /** * Register the flow matcher. * @@ -2753,17 +3383,21 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_matcher *cache_matcher; struct mlx5dv_flow_matcher_attr dv_attr = { .type = IBV_FLOW_ATTR_NORMAL, .match_mask = (void *)&matcher->mask, }; + struct mlx5_flow_tbl_resource *tbl = NULL; /* Lookup from cache. */ - LIST_FOREACH(cache_matcher, &priv->matchers, next) { + LIST_FOREACH(cache_matcher, &sh->matchers, next) { if (matcher->crc == cache_matcher->crc && matcher->priority == cache_matcher->priority && matcher->egress == cache_matcher->egress && + matcher->group == cache_matcher->group && + matcher->transfer == cache_matcher->transfer && !memcmp((const void *)matcher->mask.buf, (const void *)cache_matcher->mask.buf, cache_matcher->mask.size)) { @@ -2784,6 +3418,15 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate matcher memory"); + tbl = flow_dv_tbl_resource_get(dev, matcher->group * MLX5_GROUP_FACTOR, + matcher->egress, matcher->transfer, + error); + if (!tbl) { + rte_free(cache_matcher); + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create table"); + } *cache_matcher = *matcher; dv_attr.match_criteria_enable = flow_dv_matcher_enable(cache_matcher->mask.buf); @@ -2791,46 +3434,27 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, if (matcher->egress) dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; cache_matcher->matcher_object = - mlx5_glue->dv_create_flow_matcher(priv->sh->ctx, &dv_attr); + mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj); if (!cache_matcher->matcher_object) { rte_free(cache_matcher); +#ifdef HAVE_MLX5DV_DR + flow_dv_tbl_resource_release(tbl); +#endif return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create matcher"); } rte_atomic32_inc(&cache_matcher->refcnt); - LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next); + LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next); dev_flow->dv.matcher = cache_matcher; DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d", cache_matcher->priority, cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher, rte_atomic32_read(&cache_matcher->refcnt)); + rte_atomic32_inc(&tbl->refcnt); return 0; } -/** - * Add source vport match to the specified matcher. - * - * @param[in, out] matcher - * Flow matcher. - * @param[in, out] key - * Flow matcher value. - * @param[in] port - * Source vport value to match - * @param[in] mask - * Mask - */ -static void -flow_dv_translate_source_vport(void *matcher, void *key, - int16_t port, uint16_t mask) -{ - void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); - void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); - - MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); - MLX5_SET(fte_match_set_misc, misc_v, source_port, port); -} - /** * Find existing tag resource or create and register a new one. * @@ -2854,10 +3478,11 @@ flow_dv_tag_resource_register struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; struct mlx5_flow_dv_tag_resource *cache_resource; /* Lookup a matching resource from cache. */ - LIST_FOREACH(cache_resource, &priv->tags, next) { + LIST_FOREACH(cache_resource, &sh->tags, next) { if (resource->tag == cache_resource->tag) { DRV_LOG(DEBUG, "tag resource %p: refcnt %d++", (void *)cache_resource, @@ -2884,7 +3509,7 @@ flow_dv_tag_resource_register } rte_atomic32_init(&cache_resource->refcnt); rte_atomic32_inc(&cache_resource->refcnt); - LIST_INSERT_HEAD(&priv->tags, cache_resource, next); + LIST_INSERT_HEAD(&sh->tags, cache_resource, next); dev_flow->flow->tag_resource = cache_resource; DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++", (void *)cache_resource, @@ -2922,6 +3547,44 @@ flow_dv_tag_release(struct rte_eth_dev *dev, return 1; } +/** + * Translate port ID action to vport. + * + * @param[in] dev + * Pointer to rte_eth_dev structure. + * @param[in] action + * Pointer to the port ID action. + * @param[out] dst_port_id + * The target port ID. + * @param[out] error + * Pointer to the error structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_translate_action_port_id(struct rte_eth_dev *dev, + const struct rte_flow_action *action, + uint32_t *dst_port_id, + struct rte_flow_error *error) +{ + uint32_t port; + uint16_t port_id; + int ret; + const struct rte_flow_action_port_id *conf = + (const struct rte_flow_action_port_id *)action->conf; + + port = conf->original ? dev->data->port_id : conf->id; + ret = mlx5_port_to_eswitch_info(port, NULL, &port_id); + if (ret) + return rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "No eswitch info was found for port"); + *dst_port_id = port_id; + return 0; +} + /** * Fill the flow with DV spec. * @@ -2968,7 +3631,13 @@ flow_dv_translate(struct rte_eth_dev *dev, }; union flow_dv_attr flow_attr = { .attr = 0 }; struct mlx5_flow_dv_tag_resource tag_resource; + uint32_t modify_action_position = UINT32_MAX; + void *match_mask = matcher.mask.buf; + void *match_value = dev_flow->dv.value.buf; + flow->group = attr->group; + if (attr->transfer) + res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; if (priority == MLX5_FLOW_PRIO_RSVD) priority = priv->config.flow_prio - 1; for (; !actions_end ; actions++) { @@ -2977,10 +3646,27 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action *action = actions; const struct rte_flow_action_count *count = action->conf; const uint8_t *rss_key; + const struct rte_flow_action_jump *jump_data; + struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource; + struct mlx5_flow_tbl_resource *tbl; + uint32_t port_id = 0; + struct mlx5_flow_dv_port_id_action_resource port_id_resource; switch (actions->type) { case RTE_FLOW_ACTION_TYPE_VOID: break; + case RTE_FLOW_ACTION_TYPE_PORT_ID: + if (flow_dv_translate_action_port_id(dev, action, + &port_id, error)) + return -rte_errno; + port_id_resource.port_id = port_id; + if (flow_dv_port_id_action_resource_register + (dev, &port_id_resource, dev_flow, error)) + return -rte_errno; + dev_flow->dv.actions[actions_n++] = + dev_flow->dv.port_id_action->action; + action_flags |= MLX5_FLOW_ACTION_PORT_ID; + break; case RTE_FLOW_ACTION_TYPE_FLAG: tag_resource.tag = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); @@ -3057,7 +3743,9 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: if (flow_dv_create_action_l2_encap(dev, actions, - dev_flow, error)) + dev_flow, + attr->transfer, + error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = dev_flow->dv.encap_decap->verbs_action; @@ -3069,6 +3757,7 @@ cnt_err: case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: if (flow_dv_create_action_l2_decap(dev, dev_flow, + attr->transfer, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = @@ -3088,9 +3777,9 @@ cnt_err: dev_flow->dv.encap_decap->verbs_action; } else { /* Handle encap without preceding decap. */ - if (flow_dv_create_action_l2_encap(dev, actions, - dev_flow, - error)) + if (flow_dv_create_action_l2_encap + (dev, actions, dev_flow, attr->transfer, + error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = dev_flow->dv.encap_decap->verbs_action; @@ -3105,9 +3794,8 @@ cnt_err: } /* Handle decap only if it isn't followed by encap. */ if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { - if (flow_dv_create_action_l2_decap(dev, - dev_flow, - error)) + if (flow_dv_create_action_l2_decap + (dev, dev_flow, attr->transfer, error)) return -rte_errno; dev_flow->dv.actions[actions_n++] = dev_flow->dv.encap_decap->verbs_action; @@ -3115,6 +3803,32 @@ cnt_err: /* If decap is followed by encap, handle it at encap. */ action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; break; + case RTE_FLOW_ACTION_TYPE_JUMP: + jump_data = action->conf; + tbl = flow_dv_tbl_resource_get(dev, jump_data->group * + MLX5_GROUP_FACTOR, + attr->egress, + attr->transfer, error); + if (!tbl) + return rte_flow_error_set + (error, errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot create jump action."); + jump_tbl_resource.tbl = tbl; + if (flow_dv_jump_tbl_resource_register + (dev, &jump_tbl_resource, dev_flow, error)) { + flow_dv_tbl_resource_release(tbl); + return rte_flow_error_set + (error, errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, + "cannot create jump action."); + } + dev_flow->dv.actions[actions_n++] = + dev_flow->dv.jump->action; + action_flags |= MLX5_FLOW_ACTION_JUMP; + break; case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: if (flow_dv_convert_action_modify_mac(&res, actions, @@ -3179,35 +3893,28 @@ cnt_err: dev_flow, error)) return -rte_errno; - dev_flow->dv.actions[actions_n++] = + dev_flow->dv.actions[modify_action_position] = dev_flow->dv.modify_hdr->verbs_action; } break; default: break; } + if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) && + modify_action_position == UINT32_MAX) + modify_action_position = actions_n++; } dev_flow->dv.actions_n = actions_n; flow->actions = action_flags; - if (attr->ingress && !attr->transfer && - (priv->representor || priv->master)) { - /* It was validated - we support unidirection flows only. */ - assert(!attr->egress); - /* - * Add matching on source vport index only - * for ingress rules in E-Switch configurations. - */ - flow_dv_translate_source_vport(matcher.mask.buf, - dev_flow->dv.value.buf, - priv->vport_id, - 0xffff); - } for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); - void *match_mask = matcher.mask.buf; - void *match_value = dev_flow->dv.value.buf; switch (items->type) { + case RTE_FLOW_ITEM_TYPE_PORT_ID: + flow_dv_translate_item_port_id(dev, match_mask, + match_value, items); + last_item = MLX5_FLOW_ITEM_PORT_ID; + break; case RTE_FLOW_ITEM_TYPE_ETH: flow_dv_translate_item_eth(match_mask, match_value, items, tunnel); @@ -3226,7 +3933,7 @@ cnt_err: break; case RTE_FLOW_ITEM_TYPE_IPV4: flow_dv_translate_item_ipv4(match_mask, match_value, - items, tunnel); + items, tunnel, attr->group); matcher.priority = MLX5_PRIORITY_MAP_L3; dev_flow->dv.hash_fields |= mlx5_flow_hashfields_adjust @@ -3238,7 +3945,7 @@ cnt_err: break; case RTE_FLOW_ITEM_TYPE_IPV6: flow_dv_translate_item_ipv6(match_mask, match_value, - items, tunnel); + items, tunnel, attr->group); matcher.priority = MLX5_PRIORITY_MAP_L3; dev_flow->dv.hash_fields |= mlx5_flow_hashfields_adjust @@ -3307,6 +4014,19 @@ cnt_err: } item_flags |= last_item; } + /* + * In case of ingress traffic when E-Switch mode is enabled, + * we have two cases where we need to set the source port manually. + * The first one, is in case of Nic steering rule, and the second is + * E-Switch rule where no port_id item was found. In both cases + * the source port is set according the current port in use. + */ + if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) && + (priv->representor || priv->master)) { + if (flow_dv_translate_item_port_id(dev, match_mask, + match_value, NULL)) + return -rte_errno; + } assert(!flow_dv_check_valid_spec(matcher.mask.buf, dev_flow->dv.value.buf)); dev_flow->layers = item_flags; @@ -3316,6 +4036,8 @@ cnt_err: matcher.priority = mlx5_flow_adjust_priority(dev, priority, matcher.priority); matcher.egress = attr->egress; + matcher.group = attr->group; + matcher.transfer = attr->transfer; if (flow_dv_matcher_register(dev, &matcher, dev_flow, error)) return -rte_errno; return 0; @@ -3340,6 +4062,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, { struct mlx5_flow_dv *dv; struct mlx5_flow *dev_flow; + struct mlx5_priv *priv = dev->data->dev_private; int n; int err; @@ -3347,17 +4070,20 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, dv = &dev_flow->dv; n = dv->actions_n; if (flow->actions & MLX5_FLOW_ACTION_DROP) { - dv->hrxq = mlx5_hrxq_drop_new(dev); - if (!dv->hrxq) { - rte_flow_error_set - (error, errno, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot get drop hash queue"); - goto error; + if (flow->transfer) { + dv->actions[n++] = priv->sh->esw_drop_action; + } else { + dv->hrxq = mlx5_hrxq_drop_new(dev); + if (!dv->hrxq) { + rte_flow_error_set + (error, errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot get drop hash queue"); + goto error; + } + dv->actions[n++] = dv->hrxq->action; } - dv->actions[n++] = - mlx5_glue->dv_create_flow_action_dest_ibv_qp - (dv->hrxq->qp); } else if (flow->actions & (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) { struct mlx5_hrxq *hrxq; @@ -3382,9 +4108,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, goto error; } dv->hrxq = hrxq; - dv->actions[n++] = - mlx5_glue->dv_create_flow_action_dest_ibv_qp - (dv->hrxq->qp); + dv->actions[n++] = dv->hrxq->action; } dv->flow = mlx5_glue->dv_create_flow(dv->matcher->matcher_object, @@ -3431,6 +4155,9 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, struct mlx5_flow *flow) { struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_ibv_shared *sh = priv->sh; + struct mlx5_flow_tbl_resource *tbl; assert(matcher->matcher_object); DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", @@ -3440,6 +4167,11 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, claim_zero(mlx5_glue->dv_destroy_flow_matcher (matcher->matcher_object)); LIST_REMOVE(matcher, next); + if (matcher->egress) + tbl = &sh->tx_tbl[matcher->group]; + else + tbl = &sh->rx_tbl[matcher->group]; + flow_dv_tbl_resource_release(tbl); rte_free(matcher); DRV_LOG(DEBUG, "port %u matcher %p: removed", dev->data->port_id, (void *)matcher); @@ -3479,6 +4211,38 @@ flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) return 1; } +/** + * Release an jump to table action resource. + * + * @param flow + * Pointer to mlx5_flow. + * + * @return + * 1 while a reference on it exists, 0 when freed. + */ +static int +flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow) +{ + struct mlx5_flow_dv_jump_tbl_resource *cache_resource = + flow->dv.jump; + + assert(cache_resource->action); + DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { + claim_zero(mlx5_glue->destroy_flow_action + (cache_resource->action)); + LIST_REMOVE(cache_resource, next); + flow_dv_tbl_resource_release(cache_resource->tbl); + rte_free(cache_resource); + DRV_LOG(DEBUG, "jump table resource %p: removed", + (void *)cache_resource); + return 0; + } + return 1; +} + /** * Release a modify-header resource. * @@ -3510,6 +4274,37 @@ flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow) return 1; } +/** + * Release port ID action resource. + * + * @param flow + * Pointer to mlx5_flow. + * + * @return + * 1 while a reference on it exists, 0 when freed. + */ +static int +flow_dv_port_id_action_resource_release(struct mlx5_flow *flow) +{ + struct mlx5_flow_dv_port_id_action_resource *cache_resource = + flow->dv.port_id_action; + + assert(cache_resource->action); + DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", + (void *)cache_resource, + rte_atomic32_read(&cache_resource->refcnt)); + if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { + claim_zero(mlx5_glue->destroy_flow_action + (cache_resource->action)); + LIST_REMOVE(cache_resource, next); + rte_free(cache_resource); + DRV_LOG(DEBUG, "port id action resource %p: removed", + (void *)cache_resource); + return 0; + } + return 1; +} + /** * Remove the flow from the NIC but keeps it in memory. * @@ -3529,7 +4324,7 @@ flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) LIST_FOREACH(dev_flow, &flow->dev_flows, next) { dv = &dev_flow->dv; if (dv->flow) { - claim_zero(mlx5_glue->destroy_flow(dv->flow)); + claim_zero(mlx5_glue->dv_destroy_flow(dv->flow)); dv->flow = NULL; } if (dv->hrxq) { @@ -3575,6 +4370,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_encap_decap_resource_release(dev_flow); if (dev_flow->dv.modify_hdr) flow_dv_modify_hdr_resource_release(dev_flow); + if (dev_flow->dv.jump) + flow_dv_jump_tbl_resource_release(dev_flow); + if (dev_flow->dv.port_id_action) + flow_dv_port_id_action_resource_release(dev_flow); rte_free(dev_flow); } } @@ -3667,14 +4466,70 @@ flow_dv_query(struct rte_eth_dev *dev, return ret; } +/* + * Mutex-protected thunk to flow_dv_translate(). + */ +static int +flow_d_translate(struct rte_eth_dev *dev, + struct mlx5_flow *dev_flow, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + int ret; + + flow_d_shared_lock(dev); + ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error); + flow_d_shared_unlock(dev); + return ret; +} + +/* + * Mutex-protected thunk to flow_dv_apply(). + */ +static int +flow_d_apply(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + int ret; + + flow_d_shared_lock(dev); + ret = flow_dv_apply(dev, flow, error); + flow_d_shared_unlock(dev); + return ret; +} + +/* + * Mutex-protected thunk to flow_dv_remove(). + */ +static void +flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow) +{ + flow_d_shared_lock(dev); + flow_dv_remove(dev, flow); + flow_d_shared_unlock(dev); +} + +/* + * Mutex-protected thunk to flow_dv_destroy(). + */ +static void +flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) +{ + flow_d_shared_lock(dev); + flow_dv_destroy(dev, flow); + flow_d_shared_unlock(dev); +} const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .validate = flow_dv_validate, .prepare = flow_dv_prepare, - .translate = flow_dv_translate, - .apply = flow_dv_apply, - .remove = flow_dv_remove, - .destroy = flow_dv_destroy, + .translate = flow_d_translate, + .apply = flow_d_apply, + .remove = flow_d_remove, + .destroy = flow_d_destroy, .query = flow_dv_query, };