From e9e36e52eeb64dadce057dfd004a281fb8f8a0c2 Mon Sep 17 00:00:00 2001 From: Bing Zhao Date: Fri, 8 Nov 2019 17:23:10 +0200 Subject: [PATCH] net/mlx5: reorganize flow matcher resources Matchers are created on the specific table. If a single linked list is used to store these, then the finding process might be the bottleneck when there are a lot of different flow matchers on a huge amount of tables. The matchers could be move into the table data resource structure in order to reduce the comparison times when finding. Signed-off-by: Bing Zhao Acked-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.h | 1 - drivers/net/mlx5/mlx5_flow.h | 13 +++--- drivers/net/mlx5/mlx5_flow_dv.c | 83 ++++++++++++++++----------------- 3 files changed, 47 insertions(+), 50 deletions(-) diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 86d0b4a079..27242f2d6c 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -669,7 +669,6 @@ struct mlx5_ibv_shared { /* Direct Rules tables for FDB, NIC TX+RX */ void *esw_drop_action; /* Pointer to DR E-Switch drop action. */ void *pop_vlan_action; /* Pointer to DR pop VLAN action. */ - LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers; LIST_HEAD(encap_decap, mlx5_flow_dv_encap_decap_resource) encaps_decaps; LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds; LIST_HEAD(tag, mlx5_flow_dv_tag_resource) tags; diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 7bcdd5f489..26c6848be3 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -333,14 +333,13 @@ struct mlx5_flow_dv_match_params { /* Matcher structure. */ struct mlx5_flow_dv_matcher { LIST_ENTRY(mlx5_flow_dv_matcher) next; - /* Pointer to the next element. */ + /**< Pointer to the next element. */ + struct mlx5_flow_tbl_resource *tbl; + /**< Pointer to the table(group) the matcher associated with. */ rte_atomic32_t refcnt; /**< Reference counter. */ void *matcher_object; /**< Pointer to DV matcher */ uint16_t crc; /**< CRC of key. */ uint16_t priority; /**< Priority of matcher. */ - uint8_t egress; /**< Egress matcher. */ - uint8_t transfer; /**< 1 if the flow is E-Switch flow. */ - uint32_t group; /**< The matcher group. */ struct mlx5_flow_dv_match_params mask; /**< Matcher mask. */ }; @@ -437,9 +436,11 @@ struct mlx5_flow_mreg_copy_resource { /* Table data structure of the hash organization. */ struct mlx5_flow_tbl_data_entry { struct mlx5_hlist_entry entry; - /**< flow table resource, better to locate at the beginning. */ + /**< hash list entry, 64-bits key inside. */ struct mlx5_flow_tbl_resource tbl; - /**< flow table resource, better to locate at the beginning. */ + /**< flow table resource. */ + LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers; + /**< matchers' header associated with the flow table. */ struct mlx5_flow_dv_jump_tbl_resource jump; /**< jump resource, at most one for each table created. */ }; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index d3faac3fa9..3ba1e525dd 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -6298,6 +6298,8 @@ flow_dv_tbl_resource_release(struct rte_eth_dev *dev, * Pointer to rte_eth_dev structure. * @param[in, out] matcher * Pointer to flow matcher. + * @param[in, out] key + * Pointer to flow table key. * @parm[in, out] dev_flow * Pointer to the dev_flow. * @param[out] error @@ -6309,6 +6311,7 @@ flow_dv_tbl_resource_release(struct rte_eth_dev *dev, static int flow_dv_matcher_register(struct rte_eth_dev *dev, struct mlx5_flow_dv_matcher *matcher, + union mlx5_flow_tbl_key *key, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { @@ -6319,49 +6322,53 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, .type = IBV_FLOW_ATTR_NORMAL, .match_mask = (void *)&matcher->mask, }; - struct mlx5_flow_tbl_resource *tbl = NULL; + struct mlx5_flow_tbl_resource *tbl; + struct mlx5_flow_tbl_data_entry *tbl_data; + tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, + key->domain, error); + if (!tbl) + return -rte_errno; /* No need to refill the error info */ + tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); /* Lookup from cache. */ - LIST_FOREACH(cache_matcher, &sh->matchers, next) { + LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) { if (matcher->crc == cache_matcher->crc && matcher->priority == cache_matcher->priority && - matcher->egress == cache_matcher->egress && - matcher->group == cache_matcher->group && - matcher->transfer == cache_matcher->transfer && !memcmp((const void *)matcher->mask.buf, (const void *)cache_matcher->mask.buf, cache_matcher->mask.size)) { DRV_LOG(DEBUG, - "priority %hd use %s matcher %p: refcnt %d++", + "%s group %u priority %hd use %s " + "matcher %p: refcnt %d++", + key->domain ? "FDB" : "NIC", key->table_id, cache_matcher->priority, - cache_matcher->egress ? "tx" : "rx", + key->direction ? "tx" : "rx", (void *)cache_matcher, rte_atomic32_read(&cache_matcher->refcnt)); rte_atomic32_inc(&cache_matcher->refcnt); dev_flow->dv.matcher = cache_matcher; + /* old matcher should not make the table ref++. */ +#ifdef HAVE_MLX5DV_DR + flow_dv_tbl_resource_release(dev, tbl); +#endif return 0; } } /* Register new matcher. */ cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0); - if (!cache_matcher) + if (!cache_matcher) { +#ifdef HAVE_MLX5DV_DR + flow_dv_tbl_resource_release(dev, tbl); +#endif return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot allocate matcher memory"); - tbl = flow_dv_tbl_resource_get(dev, matcher->group, - matcher->egress, matcher->transfer, - error); - if (!tbl) { - rte_free(cache_matcher); - return rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create table"); } *cache_matcher = *matcher; dv_attr.match_criteria_enable = flow_dv_matcher_enable(cache_matcher->mask.buf); dv_attr.priority = matcher->priority; - if (matcher->egress) + if (key->direction) dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; cache_matcher->matcher_object = mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj); @@ -6374,14 +6381,18 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "cannot create matcher"); } + /* Save the table information */ + cache_matcher->tbl = tbl; + rte_atomic32_init(&cache_matcher->refcnt); + /* only matcher ref++, table ref++ already done above in get API. */ rte_atomic32_inc(&cache_matcher->refcnt); - LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next); + LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); dev_flow->dv.matcher = cache_matcher; - DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d", + DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", + key->domain ? "FDB" : "NIC", key->table_id, cache_matcher->priority, - cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher, + key->direction ? "tx" : "rx", (void *)cache_matcher, rte_atomic32_read(&cache_matcher->refcnt)); - rte_atomic32_inc(&tbl->refcnt); return 0; } @@ -6610,6 +6621,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, }; union flow_dv_attr flow_attr = { .attr = 0 }; struct mlx5_flow_dv_tag_resource tag_resource; + union mlx5_flow_tbl_key tbl_key; uint32_t modify_action_position = UINT32_MAX; void *match_mask = matcher.mask.buf; void *match_value = dev_flow->dv.value.buf; @@ -7242,10 +7254,11 @@ cnt_err: matcher.mask.size); matcher.priority = mlx5_flow_adjust_priority(dev, priority, matcher.priority); - matcher.egress = attr->egress; - matcher.group = dev_flow->group; - matcher.transfer = attr->transfer; - if (flow_dv_matcher_register(dev, &matcher, dev_flow, error)) + /* reserved field no needs to be set to 0 here. */ + tbl_key.domain = attr->transfer; + tbl_key.direction = attr->egress; + tbl_key.table_id = dev_flow->group; + if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error)) return -rte_errno; return 0; } @@ -7381,33 +7394,17 @@ flow_dv_matcher_release(struct rte_eth_dev *dev, struct mlx5_flow *flow) { struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher; - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_ibv_shared *sh = priv->sh; - struct mlx5_flow_tbl_data_entry *tbl_data; assert(matcher->matcher_object); DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", dev->data->port_id, (void *)matcher, rte_atomic32_read(&matcher->refcnt)); if (rte_atomic32_dec_and_test(&matcher->refcnt)) { - struct mlx5_hlist_entry *pos; - union mlx5_flow_tbl_key table_key = { - { - .table_id = matcher->group, - .reserved = 0, - .domain = !!matcher->transfer, - .direction = !!matcher->egress, - } - }; claim_zero(mlx5_glue->dv_destroy_flow_matcher (matcher->matcher_object)); LIST_REMOVE(matcher, next); - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64); - if (pos) { - tbl_data = container_of(pos, - struct mlx5_flow_tbl_data_entry, entry); - flow_dv_tbl_resource_release(dev, &tbl_data->tbl); - } + /* table ref-- in release interface. */ + flow_dv_tbl_resource_release(dev, matcher->tbl); rte_free(matcher); DRV_LOG(DEBUG, "port %u matcher %p: removed", dev->data->port_id, (void *)matcher); -- 2.20.1