From afd7a62514adca11a2f26e4e320cc7d3aff72952 Mon Sep 17 00:00:00 2001 From: Xueming Li Date: Wed, 28 Oct 2020 17:33:33 +0800 Subject: [PATCH] net/mlx5: make flow table cache thread safe To support multi-thread flow insertion/removal, this patch uses thread safe hash list API for flow table cache hash list. Signed-off-by: Xueming Li Acked-by: Matan Azrad --- drivers/net/mlx5/mlx5.c | 99 ++---------- drivers/net/mlx5/mlx5.h | 2 +- drivers/net/mlx5/mlx5_flow.c | 2 +- drivers/net/mlx5/mlx5_flow.h | 25 +++ drivers/net/mlx5/mlx5_flow_dv.c | 265 +++++++++++++++++--------------- 5 files changed, 181 insertions(+), 212 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index c74b55ecca..1d25a8ef5d 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -1012,7 +1012,7 @@ exit: } /** - * Destroy table hash list and all the root entries per domain. + * Destroy table hash list. * * @param[in] priv * Pointer to the private device data structure. @@ -1021,46 +1021,9 @@ void mlx5_free_table_hash_list(struct mlx5_priv *priv) { struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_tbl_data_entry *tbl_data; - union mlx5_flow_tbl_key table_key = { - { - .table_id = 0, - .reserved = 0, - .domain = 0, - .direction = 0, - } - }; - struct mlx5_hlist_entry *pos; if (!sh->flow_tbls) return; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - MLX5_ASSERT(tbl_data); - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_free(tbl_data); - } - table_key.direction = 1; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - MLX5_ASSERT(tbl_data); - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_free(tbl_data); - } - table_key.direction = 0; - table_key.domain = 1; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - MLX5_ASSERT(tbl_data); - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_free(tbl_data); - } mlx5_hlist_destroy(sh->flow_tbls); } @@ -1075,77 +1038,45 @@ mlx5_free_table_hash_list(struct mlx5_priv *priv) * Zero on success, positive error code otherwise. */ int -mlx5_alloc_table_hash_list(struct mlx5_priv *priv) +mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) { + int err = 0; + /* Tables are only used in DV and DR modes. */ +#ifdef HAVE_IBV_FLOW_DV_SUPPORT struct mlx5_dev_ctx_shared *sh = priv->sh; char s[MLX5_HLIST_NAMESIZE]; - int err = 0; MLX5_ASSERT(sh); snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, - 0, 0, NULL, NULL, NULL); + 0, 0, flow_dv_tbl_create_cb, NULL, + flow_dv_tbl_remove_cb); if (!sh->flow_tbls) { DRV_LOG(ERR, "flow tables with hash creation failed."); err = ENOMEM; return err; } + sh->flow_tbls->ctx = sh; #ifndef HAVE_MLX5DV_DR + struct rte_flow_error error; + struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; + /* * In case we have not DR support, the zero tables should be created * because DV expect to see them even if they cannot be created by * RDMA-CORE. */ - union mlx5_flow_tbl_key table_key = { - { - .table_id = 0, - .reserved = 0, - .domain = 0, - .direction = 0, - } - }; - struct mlx5_flow_tbl_data_entry *tbl_data = mlx5_malloc(MLX5_MEM_ZERO, - sizeof(*tbl_data), 0, - SOCKET_ID_ANY); - - if (!tbl_data) { - err = ENOMEM; - goto error; - } - tbl_data->entry.key = table_key.v64; - err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry); - if (err) - goto error; - __atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED); - table_key.direction = 1; - tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0, - SOCKET_ID_ANY); - if (!tbl_data) { + if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL, 0, 1, &error) || + !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, NULL, 0, 1, &error) || + !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, NULL, 0, 1, &error)) { err = ENOMEM; goto error; } - tbl_data->entry.key = table_key.v64; - err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry); - if (err) - goto error; - __atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED); - table_key.direction = 0; - table_key.domain = 1; - tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0, - SOCKET_ID_ANY); - if (!tbl_data) { - err = ENOMEM; - goto error; - } - tbl_data->entry.key = table_key.v64; - err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry); - if (err) - goto error; - __atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED); return err; error: mlx5_free_table_hash_list(priv); #endif /* HAVE_MLX5DV_DR */ +#endif return err; } diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 3ce890ab1f..81dba2fe80 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -499,7 +499,7 @@ union mlx5_flow_tbl_key { struct { /* Table ID should be at the lowest address. */ uint32_t table_id; /**< ID of the table. */ - uint16_t reserved; /**< must be zero for comparison. */ + uint16_t dummy; /**< Dummy table for DV API. */ uint8_t domain; /**< 1 - FDB, 0 - NIC TX/RX. */ uint8_t direction; /**< 1 - egress, 0 - ingress. */ }; diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index ac4a7845e8..bb4e7a05e4 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -7556,7 +7556,7 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) union mlx5_flow_tbl_key table_key = { { .table_id = tunnel_id_to_flow_tbl(mbits.table_id), - .reserved = 0, + .dummy = 0, .domain = !!mbits.transfer, .direction = 0, } diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 8253c8cd61..5fa8d6e759 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -384,6 +384,13 @@ enum mlx5_flow_fate_type { MLX5_FLOW_FATE_MAX, }; +/* Hash list callback context */ +struct mlx5_flow_cb_ctx { + struct rte_eth_dev *dev; + struct rte_flow_error *error; + void *data; +}; + /* Matcher PRM representation */ struct mlx5_flow_dv_match_params { size_t size; @@ -523,6 +530,13 @@ struct mlx5_flow_mreg_copy_resource { uint32_t rix_flow; /* Built flow for copy. */ }; +/* Table tunnel parameter. */ +struct mlx5_flow_tbl_tunnel_prm { + const struct mlx5_flow_tunnel *tunnel; + uint32_t group_id; + bool external; +}; + /* Table data structure of the hash organization. */ struct mlx5_flow_tbl_data_entry { struct mlx5_hlist_entry entry; @@ -1401,4 +1415,15 @@ struct rte_flow_shared_action *mlx5_flow_get_shared_rss(struct rte_flow *flow); int mlx5_shared_action_flush(struct rte_eth_dev *dev); void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id); int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh); + +/* Hash list callbacks for flow tables: */ +struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list, + uint64_t key, void *entry_ctx); +void flow_dv_tbl_remove_cb(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry); +struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev, + uint32_t table_id, uint8_t egress, uint8_t transfer, + bool external, const struct mlx5_flow_tunnel *tunnel, + uint32_t group_id, uint8_t dummy, struct rte_flow_error *error); + #endif /* RTE_PMD_MLX5_FLOW_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 0b4b7fff8e..4c9e45fbb3 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -7922,58 +7922,21 @@ flow_dv_matcher_enable(uint32_t *match_criteria) return match_criteria_enable; } - -/** - * Get a flow table. - * - * @param[in, out] dev - * Pointer to rte_eth_dev structure. - * @param[in] table_id - * Table id to use. - * @param[in] egress - * Direction of the table. - * @param[in] transfer - * E-Switch or NIC flow. - * @param[out] error - * pointer to error structure. - * - * @return - * Returns tables resource based on the index, NULL in case of failed. - */ -static struct mlx5_flow_tbl_resource * -flow_dv_tbl_resource_get(struct rte_eth_dev *dev, - uint32_t table_id, uint8_t egress, - uint8_t transfer, - bool external, - const struct mlx5_flow_tunnel *tunnel, - uint32_t group_id, - struct rte_flow_error *error) +struct mlx5_hlist_entry * +flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_tbl_resource *tbl; - union mlx5_flow_tbl_key table_key = { - { - .table_id = table_id, - .reserved = 0, - .domain = !!transfer, - .direction = !!egress, - } - }; - struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, - table_key.v64, NULL); + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct rte_eth_dev *dev = ctx->dev; struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data; + struct rte_flow_error *error = ctx->error; + union mlx5_flow_tbl_key key = { .v64 = key64 }; + struct mlx5_flow_tbl_resource *tbl; + void *domain; uint32_t idx = 0; int ret; - void *domain; - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - tbl = &tbl_data->tbl; - __atomic_fetch_add(&tbl->refcnt, 1, __ATOMIC_RELAXED); - return tbl; - } tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); if (!tbl_data) { rte_flow_error_set(error, ENOMEM, @@ -7983,19 +7946,20 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, return NULL; } tbl_data->idx = idx; - tbl_data->tunnel = tunnel; - tbl_data->group_id = group_id; - tbl_data->external = external; + tbl_data->tunnel = tt_prm->tunnel; + tbl_data->group_id = tt_prm->group_id; + tbl_data->external = tt_prm->external; tbl_data->tunnel_offload = is_tunnel_offload_active(dev); tbl = &tbl_data->tbl; - pos = &tbl_data->entry; - if (transfer) + if (key.dummy) + return &tbl_data->entry; + if (key.domain) domain = sh->fdb_domain; - else if (egress) + else if (key.direction) domain = sh->tx_domain; else domain = sh->rx_domain; - ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj); + ret = mlx5_flow_os_create_flow_tbl(domain, key.table_id, &tbl->obj); if (ret) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -8003,13 +7967,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); return NULL; } - /* - * No multi-threads now, but still better to initialize the reference - * count before insert it into the hash list. - */ - __atomic_store_n(&tbl->refcnt, 0, __ATOMIC_RELAXED); - - if (table_id) { + if (key.table_id) { ret = mlx5_flow_os_create_flow_action_dest_flow_tbl (tbl->obj, &tbl_data->jump.action); if (ret) { @@ -8022,17 +7980,118 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, return NULL; } } - pos->key = table_key.v64; - ret = !mlx5_hlist_insert(sh->flow_tbls, pos); - if (ret < 0) { - rte_flow_error_set(error, -ret, + return &tbl_data->entry; +} + +/** + * Get a flow table. + * + * @param[in, out] dev + * Pointer to rte_eth_dev structure. + * @param[in] table_id + * Table id to use. + * @param[in] egress + * Direction of the table. + * @param[in] transfer + * E-Switch or NIC flow. + * @param[in] dummy + * Dummy entry for dv API. + * @param[out] error + * pointer to error structure. + * + * @return + * Returns tables resource based on the index, NULL in case of failed. + */ +struct mlx5_flow_tbl_resource * +flow_dv_tbl_resource_get(struct rte_eth_dev *dev, + uint32_t table_id, uint8_t egress, + uint8_t transfer, + bool external, + const struct mlx5_flow_tunnel *tunnel, + uint32_t group_id, uint8_t dummy, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + union mlx5_flow_tbl_key table_key = { + { + .table_id = table_id, + .dummy = dummy, + .domain = !!transfer, + .direction = !!egress, + } + }; + struct mlx5_flow_tbl_tunnel_prm tt_prm = { + .tunnel = tunnel, + .group_id = group_id, + .external = external, + }; + struct mlx5_flow_cb_ctx ctx = { + .dev = dev, + .error = error, + .data = &tt_prm, + }; + struct mlx5_hlist_entry *entry; + struct mlx5_flow_tbl_data_entry *tbl_data; + + entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx); + if (!entry) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot insert flow table data entry"); - mlx5_flow_os_destroy_flow_tbl(tbl->obj); - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); + "cannot get table"); + return NULL; } - __atomic_fetch_add(&tbl->refcnt, 1, __ATOMIC_RELAXED); - return tbl; + tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + return &tbl_data->tbl; +} + +void +flow_dv_tbl_remove_cb(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + + MLX5_ASSERT(entry && sh); + if (tbl_data->jump.action) + mlx5_flow_os_destroy_flow_action(tbl_data->jump.action); + if (tbl_data->tbl.obj) + mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj); + if (tbl_data->tunnel_offload && tbl_data->external) { + struct mlx5_hlist_entry *he; + struct mlx5_hlist *tunnel_grp_hash; + struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; + union tunnel_tbl_key tunnel_key = { + .tunnel_id = tbl_data->tunnel ? + tbl_data->tunnel->tunnel_id : 0, + .group = tbl_data->group_id + }; + union mlx5_flow_tbl_key table_key = { + .v64 = entry->key + }; + uint32_t table_id = table_key.table_id; + + tunnel_grp_hash = tbl_data->tunnel ? + tbl_data->tunnel->groups : + thub->groups; + he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL); + if (he) { + struct tunnel_tbl_entry *tte; + tte = container_of(he, typeof(*tte), hash); + MLX5_ASSERT(tte->flow_table == table_id); + mlx5_hlist_remove(tunnel_grp_hash, he); + mlx5_free(tte); + } + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], + tunnel_flow_tbl_to_id(table_id)); + DRV_LOG(DEBUG, + "Table_id %#x tunnel %u group %u released.", + table_id, + tbl_data->tunnel ? + tbl_data->tunnel->tunnel_id : 0, + tbl_data->group_id); + } + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx); } /** @@ -8057,54 +8116,7 @@ flow_dv_tbl_resource_release(struct rte_eth_dev *dev, if (!tbl) return 0; - if (__atomic_sub_fetch(&tbl->refcnt, 1, __ATOMIC_RELAXED) == 0) { - struct mlx5_hlist_entry *pos = &tbl_data->entry; - - mlx5_flow_os_destroy_flow_tbl(tbl->obj); - tbl->obj = NULL; - if (tbl_data->tunnel_offload && tbl_data->external) { - struct mlx5_hlist_entry *he; - struct mlx5_hlist *tunnel_grp_hash; - struct mlx5_flow_tunnel_hub *thub = - mlx5_tunnel_hub(dev); - union tunnel_tbl_key tunnel_key = { - .tunnel_id = tbl_data->tunnel ? - tbl_data->tunnel->tunnel_id : 0, - .group = tbl_data->group_id - }; - union mlx5_flow_tbl_key table_key = { - .v64 = pos->key - }; - uint32_t table_id = table_key.table_id; - - tunnel_grp_hash = tbl_data->tunnel ? - tbl_data->tunnel->groups : - thub->groups; - he = mlx5_hlist_lookup(tunnel_grp_hash, - tunnel_key.val, NULL); - if (he) { - struct tunnel_tbl_entry *tte; - tte = container_of(he, typeof(*tte), hash); - MLX5_ASSERT(tte->flow_table == table_id); - mlx5_hlist_remove(tunnel_grp_hash, he); - mlx5_free(tte); - } - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID], - tunnel_flow_tbl_to_id(table_id)); - DRV_LOG(DEBUG, - "port %u release table_id %#x tunnel %u group %u", - dev->data->port_id, table_id, - tbl_data->tunnel ? - tbl_data->tunnel->tunnel_id : 0, - tbl_data->group_id); - } - /* remove the entry from the hash list and free memory. */ - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP], - tbl_data->idx); - return 0; - } - return 1; + return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry); } /** @@ -8143,7 +8155,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, int ret; tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, - key->domain, false, NULL, 0, error); + key->domain, false, NULL, 0, 0, error); if (!tbl) return -rte_errno; /* No need to refill the error info */ tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); @@ -8637,7 +8649,7 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev, /* Create normal path table level */ tbl = flow_dv_tbl_resource_get(dev, next_ft_id, attr->egress, attr->transfer, - dev_flow->external, NULL, 0, error); + dev_flow->external, NULL, 0, 0, error); if (!tbl) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -9317,7 +9329,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, attr->transfer, !!dev_flow->external, tunnel, - attr->group, error); + attr->group, 0, error); if (!tbl) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, @@ -9599,7 +9611,7 @@ __flow_dv_translate(struct rte_eth_dev *dev, tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, attr->transfer, !!dev_flow->external, - tunnel, jump_group, + tunnel, jump_group, 0, error); if (!tbl) return rte_flow_error_set @@ -11554,7 +11566,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, /* Create the meter table with METER level. */ dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, egress, transfer, false, NULL, 0, - &error); + 0, &error); if (!dtb->tbl) { DRV_LOG(ERR, "Failed to create meter policer table."); return -1; @@ -11563,7 +11575,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, dtb->sfx_tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_SUFFIX, egress, transfer, false, NULL, 0, - &error); + 0, &error); if (!dtb->sfx_tbl) { DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; @@ -11882,10 +11894,11 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev) void *flow = NULL; int i, ret = -1; - tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, NULL); + tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, 0, NULL); if (!tbl) goto err; - dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, false, NULL, 0, NULL); + dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, false, + NULL, 0, 0, NULL); if (!dest_tbl) goto err; dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); -- 2.20.1