X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=inline;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow.c;h=2af0ea9def29f2ca12a86698712872741a0d8a6b;hb=7b2423cd2e84c61d3052147cea7643d15d49d1e9;hp=e0775172cc7f7e0c1dea86c4b94590f4f70074f5;hpb=b365799b3cd440437367b4539f1792eb65c86cc7;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index e0775172cc..2af0ea9def 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2016 6WIND S.A. - * Copyright 2016 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2016 6WIND S.A. + * Copyright 2016 Mellanox. */ #include @@ -44,14 +16,16 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -#include +#include #include #include #include +#include #include "mlx5.h" #include "mlx5_defs.h" #include "mlx5_prm.h" +#include "mlx5_glue.h" /* Define minimal priority for control plane flows. */ #define MLX5_CTRL_FLOW_PRIORITY 4 @@ -61,22 +35,9 @@ #define MLX5_IPV6 6 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT -struct ibv_counter_set_init_attr { - int dummy; -}; struct ibv_flow_spec_counter_action { int dummy; }; -struct ibv_counter_set { - int dummy; -}; - -static inline int -ibv_destroy_counter_set(struct ibv_counter_set *cs) -{ - (void)cs; - return -ENOTSUP; -} #endif /* Dev ops structure defined in mlx5.c */ @@ -430,7 +391,6 @@ static const struct mlx5_flow_items mlx5_flow_items[] = { /** Structure to pass to the conversion function. */ struct mlx5_flow_parse { uint32_t inner; /**< Set once VXLAN is encountered. */ - uint32_t allmulti:1; /**< Set once allmulti dst MAC is encountered. */ uint32_t create:1; /**< Whether resources should remain after a validate. */ uint32_t drop:1; /**< Target is a drop queue. */ @@ -528,7 +488,7 @@ mlx5_flow_item_validate(const struct rte_flow_item *item, } if (item->mask) { unsigned int i; - const uint8_t *spec = item->mask; + const uint8_t *spec = item->spec; for (i = 0; i < size; ++i) if ((spec[i] | mask[i]) != mask[i]) @@ -552,7 +512,8 @@ mlx5_flow_item_validate(const struct rte_flow_item *item, } /** - * Copy the RSS configuration from the user ones. + * Copy the RSS configuration from the user ones, of the rss_conf is null, + * uses the driver one. * * @param priv * Pointer to private structure. @@ -565,25 +526,28 @@ mlx5_flow_item_validate(const struct rte_flow_item *item, * 0 on success, errno value on failure. */ static int -priv_flow_convert_rss_conf(struct priv *priv, +priv_flow_convert_rss_conf(struct priv *priv __rte_unused, struct mlx5_flow_parse *parser, const struct rte_eth_rss_conf *rss_conf) { - const struct rte_eth_rss_conf *rss; - + /* + * This function is also called at the beginning of + * priv_flow_convert_actions() to initialize the parser with the + * device default RSS configuration. + */ if (rss_conf) { if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) return EINVAL; - rss = rss_conf; - } else { - rss = &priv->rss_conf; + if (rss_conf->rss_key_len != 40) + return EINVAL; + if (rss_conf->rss_key_len && rss_conf->rss_key) { + parser->rss_conf.rss_key_len = rss_conf->rss_key_len; + memcpy(parser->rss_key, rss_conf->rss_key, + rss_conf->rss_key_len); + parser->rss_conf.rss_key = parser->rss_key; + } + parser->rss_conf.rss_hf = rss_conf->rss_hf; } - if (rss->rss_key_len > 40) - return EINVAL; - parser->rss_conf.rss_key_len = rss->rss_key_len; - parser->rss_conf.rss_hf = rss->rss_hf; - memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len); - parser->rss_conf.rss_key = parser->rss_key; return 0; } @@ -603,13 +567,11 @@ priv_flow_convert_rss_conf(struct priv *priv, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_attributes(struct priv *priv, +priv_flow_convert_attributes(struct priv *priv __rte_unused, const struct rte_flow_attr *attr, struct rte_flow_error *error, - struct mlx5_flow_parse *parser) + struct mlx5_flow_parse *parser __rte_unused) { - (void)priv; - (void)parser; if (attr->group) { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, @@ -778,7 +740,7 @@ priv_flow_convert_actions(struct priv *priv, } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { parser->mark = 1; } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && - priv->counter_set_supported) { + priv->config.flow_counter_en) { parser->count = 1; } else { goto exit_action_not_supported; @@ -814,7 +776,7 @@ exit_action_not_supported: * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_items_validate(struct priv *priv, +priv_flow_convert_items_validate(struct priv *priv __rte_unused, const struct rte_flow_item items[], struct rte_flow_error *error, struct mlx5_flow_parse *parser) @@ -822,7 +784,6 @@ priv_flow_convert_items_validate(struct priv *priv, const struct mlx5_flow_items *cur_item = mlx5_flow_items; unsigned int i; - (void)priv; /* Initialise the offsets to start after verbs attribute. */ for (i = 0; i != hash_rxq_init_n; ++i) parser->queue[i].offset = sizeof(struct ibv_flow_attr); @@ -861,13 +822,17 @@ priv_flow_convert_items_validate(struct priv *priv, } parser->inner = IBV_FLOW_SPEC_INNER; } - if (parser->drop || parser->queues_n == 1) { + if (parser->drop) { parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; } else { for (n = 0; n != hash_rxq_init_n; ++n) parser->queue[n].offset += cur_item->dst_sz; } } + if (parser->drop) { + parser->queue[HASH_RXQ_ETH].offset += + sizeof(struct ibv_flow_spec_action_drop); + } if (parser->mark) { for (i = 0; i != hash_rxq_init_n; ++i) parser->queue[i].offset += @@ -901,15 +866,14 @@ exit_item_not_supported: * @return * A verbs flow attribute on success, NULL otherwise. */ -static struct ibv_flow_attr* -priv_flow_convert_allocate(struct priv *priv, +static struct ibv_flow_attr * +priv_flow_convert_allocate(struct priv *priv __rte_unused, unsigned int priority, unsigned int size, struct rte_flow_error *error) { struct ibv_flow_attr *ibv_attr; - (void)priv; ibv_attr = rte_calloc(__func__, 1, size, 0); if (!ibv_attr) { rte_flow_error_set(error, ENOMEM, @@ -931,7 +895,8 @@ priv_flow_convert_allocate(struct priv *priv, * Internal parser structure. */ static void -priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) +priv_flow_convert_finalise(struct priv *priv __rte_unused, + struct mlx5_flow_parse *parser) { const unsigned int ipv4 = hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; @@ -942,7 +907,16 @@ priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; unsigned int i; - (void)priv; + /* Remove any other flow not matching the pattern. */ + if (parser->queues_n == 1) { + for (i = 0; i != hash_rxq_init_n; ++i) { + if (i == HASH_RXQ_ETH) + continue; + rte_free(parser->queue[i].ibv_attr); + parser->queue[i].ibv_attr = NULL; + } + return; + } if (parser->layer == HASH_RXQ_ETH) { goto fill; } else { @@ -1089,7 +1063,7 @@ priv_flow_convert(struct priv *priv, * Second step. * Allocate the memory space to store verbs specifications. */ - if (parser->drop || parser->queues_n == 1) { + if (parser->drop) { unsigned int priority = attr->priority + hash_rxq_init[HASH_RXQ_ETH].flow_priority; @@ -1151,14 +1125,9 @@ priv_flow_convert(struct priv *priv, * Last step. Complete missing specification to reach the RSS * configuration. */ - if (parser->queues_n > 1) { + if (!parser->drop) { priv_flow_convert_finalise(priv, parser); } else { - /* - * Action queue have their priority overridden with - * Ethernet priority, this priority needs to be adjusted to - * their most specific layer priority. - */ parser->queue[HASH_RXQ_ETH].ibv_attr->priority = attr->priority + hash_rxq_init[parser->layer].flow_priority; @@ -1173,17 +1142,6 @@ exit_free: } } } - if (parser->allmulti && - parser->layer == HASH_RXQ_ETH) { - for (i = 0; i != hash_rxq_init_n; ++i) { - if (!parser->queue[i].ibv_attr) - continue; - if (parser->queue[i].ibv_attr->num_of_specs != 1) - break; - parser->queue[i].ibv_attr->type = - IBV_FLOW_ATTR_MC_DEFAULT; - } - } return ret; exit_enomem: for (i = 0; i != hash_rxq_init_n; ++i) { @@ -1281,7 +1239,6 @@ mlx5_flow_create_eth(const struct rte_flow_item *item, eth.val.ether_type &= eth.mask.ether_type; } mlx5_flow_create_copy(parser, ð, eth_size); - parser->allmulti = eth.val.dst_mac[0] & 1; return 0; } @@ -1406,6 +1363,8 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item, parser->layer = HASH_RXQ_IPV6; if (spec) { unsigned int i; + uint32_t vtc_flow_val; + uint32_t vtc_flow_mask; if (!mask) mask = default_mask; @@ -1417,7 +1376,20 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item, RTE_DIM(ipv6.mask.src_ip)); memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, RTE_DIM(ipv6.mask.dst_ip)); - ipv6.mask.flow_label = mask->hdr.vtc_flow; + vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); + vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); + ipv6.val.flow_label = + rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >> + IPV6_HDR_FL_SHIFT); + ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >> + IPV6_HDR_TC_SHIFT; + ipv6.val.next_hdr = spec->hdr.proto; + ipv6.val.hop_limit = spec->hdr.hop_limits; + ipv6.mask.flow_label = + rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >> + IPV6_HDR_FL_SHIFT); + ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >> + IPV6_HDR_TC_SHIFT; ipv6.mask.next_hdr = mask->hdr.proto; ipv6.mask.hop_limit = mask->hdr.hop_limits; /* Remove unwanted bits from values. */ @@ -1426,6 +1398,7 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item, ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; } ipv6.val.flow_label &= ipv6.mask.flow_label; + ipv6.val.traffic_class &= ipv6.mask.traffic_class; ipv6.val.next_hdr &= ipv6.mask.next_hdr; ipv6.val.hop_limit &= ipv6.mask.hop_limit; } @@ -1627,7 +1600,7 @@ mlx5_flow_create_count(struct priv *priv __rte_unused, }; init_attr.counter_set_id = 0; - parser->cs = ibv_create_counter_set(priv->ctx, &init_attr); + parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr); if (!parser->cs) return EINVAL; counter.counter_set_handle = parser->cs->handle; @@ -1680,8 +1653,8 @@ priv_flow_create_action_queue_drop(struct priv *priv, return 0; parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; flow->frxq[HASH_RXQ_ETH].ibv_flow = - ibv_create_flow(priv->flow_drop_queue->qp, - flow->frxq[HASH_RXQ_ETH].ibv_attr); + mlx5_glue->create_flow(priv->flow_drop_queue->qp, + flow->frxq[HASH_RXQ_ETH].ibv_attr); if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "flow rule creation failure"); @@ -1692,7 +1665,8 @@ priv_flow_create_action_queue_drop(struct priv *priv, error: assert(flow); if (flow->frxq[HASH_RXQ_ETH].ibv_flow) { - claim_zero(ibv_destroy_flow(flow->frxq[HASH_RXQ_ETH].ibv_flow)); + claim_zero(mlx5_glue->destroy_flow + (flow->frxq[HASH_RXQ_ETH].ibv_flow)); flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; } if (flow->frxq[HASH_RXQ_ETH].ibv_attr) { @@ -1700,7 +1674,7 @@ error: flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL; } if (flow->cs) { - claim_zero(ibv_destroy_counter_set(flow->cs)); + claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); flow->cs = NULL; parser->cs = NULL; } @@ -1789,6 +1763,7 @@ priv_flow_create_action_queue(struct priv *priv, { int err = 0; unsigned int i; + unsigned int flows_n = 0; assert(priv->pd); assert(priv->ctx); @@ -1804,8 +1779,8 @@ priv_flow_create_action_queue(struct priv *priv, if (!flow->frxq[i].hrxq) continue; flow->frxq[i].ibv_flow = - ibv_create_flow(flow->frxq[i].hrxq->qp, - flow->frxq[i].ibv_attr); + mlx5_glue->create_flow(flow->frxq[i].hrxq->qp, + flow->frxq[i].ibv_attr); if (!flow->frxq[i].ibv_flow) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, @@ -1813,11 +1788,17 @@ priv_flow_create_action_queue(struct priv *priv, err = ENOMEM; goto error; } + ++flows_n; DEBUG("%p type %d QP %p ibv_flow %p", (void *)flow, i, (void *)flow->frxq[i].hrxq, (void *)flow->frxq[i].ibv_flow); } + if (!flows_n) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "internal error in flow creation"); + goto error; + } for (i = 0; i != parser->queues_n; ++i) { struct mlx5_rxq_data *q = (*priv->rxqs)[parser->queues[i]]; @@ -1831,7 +1812,7 @@ error: if (flow->frxq[i].ibv_flow) { struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; - claim_zero(ibv_destroy_flow(ibv_flow)); + claim_zero(mlx5_glue->destroy_flow(ibv_flow)); } if (flow->frxq[i].hrxq) mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); @@ -1839,7 +1820,7 @@ error: rte_free(flow->frxq[i].ibv_attr); } if (flow->cs) { - claim_zero(ibv_destroy_counter_set(flow->cs)); + claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); flow->cs = NULL; parser->cs = NULL; } @@ -1912,6 +1893,7 @@ priv_flow_create(struct priv *priv, DEBUG("Flow created %p", (void *)flow); return flow; exit: + ERROR("flow creation error: %s", error->message); for (i = 0; i != hash_rxq_init_n; ++i) { if (parser.queue[i].ibv_attr) rte_free(parser.queue[i].ibv_attr); @@ -1937,9 +1919,7 @@ mlx5_flow_validate(struct rte_eth_dev *dev, int ret; struct mlx5_flow_parse parser = { .create = 0, }; - priv_lock(priv); ret = priv_flow_convert(priv, attr, items, actions, error, &parser); - priv_unlock(priv); return ret; } @@ -1959,10 +1939,8 @@ mlx5_flow_create(struct rte_eth_dev *dev, struct priv *priv = dev->data->dev_private; struct rte_flow *flow; - priv_lock(priv); flow = priv_flow_create(priv, &priv->flows, attr, items, actions, error); - priv_unlock(priv); return flow; } @@ -2017,7 +1995,7 @@ priv_flow_destroy(struct priv *priv, free: if (flow->drop) { if (flow->frxq[HASH_RXQ_ETH].ibv_flow) - claim_zero(ibv_destroy_flow + claim_zero(mlx5_glue->destroy_flow (flow->frxq[HASH_RXQ_ETH].ibv_flow)); rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); } else { @@ -2025,7 +2003,8 @@ free: struct mlx5_flow *frxq = &flow->frxq[i]; if (frxq->ibv_flow) - claim_zero(ibv_destroy_flow(frxq->ibv_flow)); + claim_zero(mlx5_glue->destroy_flow + (frxq->ibv_flow)); if (frxq->hrxq) mlx5_priv_hrxq_release(priv, frxq->hrxq); if (frxq->ibv_attr) @@ -2033,7 +2012,7 @@ free: } } if (flow->cs) { - claim_zero(ibv_destroy_counter_set(flow->cs)); + claim_zero(mlx5_glue->destroy_counter_set(flow->cs)); flow->cs = NULL; } TAILQ_REMOVE(list, flow, next); @@ -2081,35 +2060,38 @@ priv_flow_create_drop_queue(struct priv *priv) WARN("cannot allocate memory for drop queue"); goto error; } - fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); + fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0); if (!fdq->cq) { WARN("cannot allocate CQ for drop queue"); goto error; } - fdq->wq = ibv_create_wq(priv->ctx, - &(struct ibv_wq_init_attr){ + fdq->wq = mlx5_glue->create_wq + (priv->ctx, + &(struct ibv_wq_init_attr){ .wq_type = IBV_WQT_RQ, .max_wr = 1, .max_sge = 1, .pd = priv->pd, .cq = fdq->cq, - }); + }); if (!fdq->wq) { WARN("cannot allocate WQ for drop queue"); goto error; } - fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx, - &(struct ibv_rwq_ind_table_init_attr){ + fdq->ind_table = mlx5_glue->create_rwq_ind_table + (priv->ctx, + &(struct ibv_rwq_ind_table_init_attr){ .log_ind_tbl_size = 0, .ind_tbl = &fdq->wq, .comp_mask = 0, - }); + }); if (!fdq->ind_table) { WARN("cannot allocate indirection table for drop queue"); goto error; } - fdq->qp = ibv_create_qp_ex(priv->ctx, - &(struct ibv_qp_init_attr_ex){ + fdq->qp = mlx5_glue->create_qp_ex + (priv->ctx, + &(struct ibv_qp_init_attr_ex){ .qp_type = IBV_QPT_RAW_PACKET, .comp_mask = IBV_QP_INIT_ATTR_PD | @@ -2124,7 +2106,7 @@ priv_flow_create_drop_queue(struct priv *priv) }, .rwq_ind_tbl = fdq->ind_table, .pd = priv->pd - }); + }); if (!fdq->qp) { WARN("cannot allocate QP for drop queue"); goto error; @@ -2133,13 +2115,13 @@ priv_flow_create_drop_queue(struct priv *priv) return 0; error: if (fdq->qp) - claim_zero(ibv_destroy_qp(fdq->qp)); + claim_zero(mlx5_glue->destroy_qp(fdq->qp)); if (fdq->ind_table) - claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); + claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table)); if (fdq->wq) - claim_zero(ibv_destroy_wq(fdq->wq)); + claim_zero(mlx5_glue->destroy_wq(fdq->wq)); if (fdq->cq) - claim_zero(ibv_destroy_cq(fdq->cq)); + claim_zero(mlx5_glue->destroy_cq(fdq->cq)); if (fdq) rte_free(fdq); priv->flow_drop_queue = NULL; @@ -2160,13 +2142,13 @@ priv_flow_delete_drop_queue(struct priv *priv) if (!fdq) return; if (fdq->qp) - claim_zero(ibv_destroy_qp(fdq->qp)); + claim_zero(mlx5_glue->destroy_qp(fdq->qp)); if (fdq->ind_table) - claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); + claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table)); if (fdq->wq) - claim_zero(ibv_destroy_wq(fdq->wq)); + claim_zero(mlx5_glue->destroy_wq(fdq->wq)); if (fdq->cq) - claim_zero(ibv_destroy_cq(fdq->cq)); + claim_zero(mlx5_glue->destroy_cq(fdq->cq)); rte_free(fdq); priv->flow_drop_queue = NULL; } @@ -2186,24 +2168,34 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list) TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { unsigned int i; + struct mlx5_ind_table_ibv *ind_tbl = NULL; if (flow->drop) { if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) continue; - claim_zero(ibv_destroy_flow + claim_zero(mlx5_glue->destroy_flow (flow->frxq[HASH_RXQ_ETH].ibv_flow)); flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; + DEBUG("Flow %p removed", (void *)flow); /* Next flow. */ continue; } + /* Verify the flow has not already been cleaned. */ + for (i = 0; i != hash_rxq_init_n; ++i) { + if (!flow->frxq[i].ibv_flow) + continue; + /* + * Indirection table may be necessary to remove the + * flags in the Rx queues. + * This helps to speed-up the process by avoiding + * another loop. + */ + ind_tbl = flow->frxq[i].hrxq->ind_table; + break; + } + if (i == hash_rxq_init_n) + return; if (flow->mark) { - struct mlx5_ind_table_ibv *ind_tbl = NULL; - - for (i = 0; i != hash_rxq_init_n; ++i) { - if (!flow->frxq[i].hrxq) - continue; - ind_tbl = flow->frxq[i].hrxq->ind_table; - } assert(ind_tbl); for (i = 0; i != ind_tbl->queues_n; ++i) (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; @@ -2211,7 +2203,8 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list) for (i = 0; i != hash_rxq_init_n; ++i) { if (!flow->frxq[i].ibv_flow) continue; - claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow)); + claim_zero(mlx5_glue->destroy_flow + (flow->frxq[i].ibv_flow)); flow->frxq[i].ibv_flow = NULL; mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); flow->frxq[i].hrxq = NULL; @@ -2241,7 +2234,7 @@ priv_flow_start(struct priv *priv, struct mlx5_flows *list) if (flow->drop) { flow->frxq[HASH_RXQ_ETH].ibv_flow = - ibv_create_flow + mlx5_glue->create_flow (priv->flow_drop_queue->qp, flow->frxq[HASH_RXQ_ETH].ibv_attr); if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { @@ -2279,8 +2272,8 @@ priv_flow_start(struct priv *priv, struct mlx5_flows *list) } flow_create: flow->frxq[i].ibv_flow = - ibv_create_flow(flow->frxq[i].hrxq->qp, - flow->frxq[i].ibv_attr); + mlx5_glue->create_flow(flow->frxq[i].hrxq->qp, + flow->frxq[i].ibv_attr); if (!flow->frxq[i].ibv_flow) { DEBUG("Flow %p cannot be applied", (void *)flow); @@ -2430,14 +2423,11 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev, int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, - struct rte_flow_error *error) + struct rte_flow_error *error __rte_unused) { struct priv *priv = dev->data->dev_private; - (void)error; - priv_lock(priv); priv_flow_destroy(priv, &priv->flows, flow); - priv_unlock(priv); return 0; } @@ -2449,14 +2439,11 @@ mlx5_flow_destroy(struct rte_eth_dev *dev, */ int mlx5_flow_flush(struct rte_eth_dev *dev, - struct rte_flow_error *error) + struct rte_flow_error *error __rte_unused) { struct priv *priv = dev->data->dev_private; - (void)error; - priv_lock(priv); priv_flow_flush(priv, &priv->flows); - priv_unlock(priv); return 0; } @@ -2487,7 +2474,7 @@ priv_flow_query_count(struct ibv_counter_set *cs, .out = counters, .outlen = 2 * sizeof(uint64_t), }; - int res = ibv_query_counter_set(&query_cs_attr, &query_out); + int res = mlx5_glue->query_counter_set(&query_cs_attr, &query_out); if (res) { rte_flow_error_set(error, -res, @@ -2514,16 +2501,14 @@ priv_flow_query_count(struct ibv_counter_set *cs, * @see rte_flow_ops */ int -mlx5_flow_query(struct rte_eth_dev *dev, +mlx5_flow_query(struct rte_eth_dev *dev __rte_unused, struct rte_flow *flow, enum rte_flow_action_type action __rte_unused, void *data, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; int res = EINVAL; - priv_lock(priv); if (flow->cs) { res = priv_flow_query_count(flow->cs, &flow->counter_stats, @@ -2535,7 +2520,6 @@ mlx5_flow_query(struct rte_eth_dev *dev, NULL, "no counter found for flow"); } - priv_unlock(priv); return -res; } #endif @@ -2553,13 +2537,11 @@ mlx5_flow_isolate(struct rte_eth_dev *dev, { struct priv *priv = dev->data->dev_private; - priv_lock(priv); if (dev->data->dev_started) { rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "port must be stopped first"); - priv_unlock(priv); return -rte_errno; } priv->isolated = !!enable; @@ -2567,7 +2549,6 @@ mlx5_flow_isolate(struct rte_eth_dev *dev, priv->dev->dev_ops = &mlx5_dev_ops_isolate; else priv->dev->dev_ops = &mlx5_dev_ops; - priv_unlock(priv); return 0; } @@ -2635,10 +2616,12 @@ priv_fdir_filter_convert(struct priv *priv, attributes->items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_IPV4, .spec = &attributes->l3, + .mask = &attributes->l3, }; attributes->items[2] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_UDP, .spec = &attributes->l4, + .mask = &attributes->l4, }; break; case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: @@ -2656,10 +2639,12 @@ priv_fdir_filter_convert(struct priv *priv, attributes->items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_IPV4, .spec = &attributes->l3, + .mask = &attributes->l3, }; attributes->items[2] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_TCP, .spec = &attributes->l4, + .mask = &attributes->l4, }; break; case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: @@ -2673,6 +2658,7 @@ priv_fdir_filter_convert(struct priv *priv, attributes->items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_IPV4, .spec = &attributes->l3, + .mask = &attributes->l3, }; break; case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: @@ -2693,10 +2679,12 @@ priv_fdir_filter_convert(struct priv *priv, attributes->items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_IPV6, .spec = &attributes->l3, + .mask = &attributes->l3, }; attributes->items[2] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_UDP, .spec = &attributes->l4, + .mask = &attributes->l4, }; break; case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: @@ -2717,10 +2705,12 @@ priv_fdir_filter_convert(struct priv *priv, attributes->items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_IPV6, .spec = &attributes->l3, + .mask = &attributes->l3, }; attributes->items[2] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_TCP, .spec = &attributes->l4, + .mask = &attributes->l4, }; break; case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: @@ -2737,6 +2727,7 @@ priv_fdir_filter_convert(struct priv *priv, attributes->items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_IPV6, .spec = &attributes->l3, + .mask = &attributes->l3, }; break; default: @@ -3039,9 +3030,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, *(const void **)arg = &mlx5_flow_ops; return 0; case RTE_ETH_FILTER_FDIR: - priv_lock(priv); ret = priv_fdir_ctrl_func(priv, filter_op, arg); - priv_unlock(priv); break; default: ERROR("%p: filter type (%d) not supported",