From e69a59227db0f2dd6ccc9f618d0ff7e6f0dd03aa Mon Sep 17 00:00:00 2001 From: Xueming Li Date: Wed, 28 Oct 2020 17:33:31 +0800 Subject: [PATCH] net/mlx5: support concurrent access for hash list In order to support hash list concurrent access, adding next: 1. List level read/write lock. 2. Entry reference counter. 3. Entry create/match/remove callback. 4. Remove insert/lookup/remove function which are not thread safe. 5. Add register/unregister function to support entry reuse. For better performance, lookup function uses read lock to allow concurrent lookup from different thread, all other hash list modification functions uses write lock which blocks concurrent modification and lookups from other thread. The exact objects change will be applied in the next patches. Signed-off-by: Xueming Li Acked-by: Matan Azrad --- drivers/net/mlx5/linux/mlx5_os.c | 27 +++--- drivers/net/mlx5/mlx5.c | 13 +-- drivers/net/mlx5/mlx5_flow.c | 23 ++--- drivers/net/mlx5/mlx5_flow_dv.c | 9 +- drivers/net/mlx5/mlx5_utils.c | 154 +++++++++++++++++++++++++------ drivers/net/mlx5/mlx5_utils.h | 149 ++++++++++++++++++++++++------ 6 files changed, 287 insertions(+), 88 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index b12d1d5502..4db5d339d4 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -236,14 +236,16 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) return err; /* Create tags hash list table. */ snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name); - sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE); + sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0, + 0, NULL, NULL, NULL); if (!sh->tag_table) { DRV_LOG(ERR, "tags with hash creation failed."); err = ENOMEM; goto error; } snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name); - sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ); + sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ, + 0, 0, NULL, NULL, NULL); if (!sh->modify_cmds) { DRV_LOG(ERR, "hdr modify hash creation failed"); err = ENOMEM; @@ -251,7 +253,8 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) } snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name); sh->encaps_decaps = mlx5_hlist_create(s, - MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ); + MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ, + 0, 0, NULL, NULL, NULL); if (!sh->encaps_decaps) { DRV_LOG(ERR, "encap decap hash creation failed"); err = ENOMEM; @@ -333,16 +336,16 @@ error: sh->pop_vlan_action = NULL; } if (sh->encaps_decaps) { - mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL); + mlx5_hlist_destroy(sh->encaps_decaps); sh->encaps_decaps = NULL; } if (sh->modify_cmds) { - mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL); + mlx5_hlist_destroy(sh->modify_cmds); sh->modify_cmds = NULL; } if (sh->tag_table) { /* tags should be destroyed with flow before. */ - mlx5_hlist_destroy(sh->tag_table, NULL, NULL); + mlx5_hlist_destroy(sh->tag_table); sh->tag_table = NULL; } if (sh->tunnel_hub) { @@ -396,16 +399,16 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv) mlx5_glue->destroy_flow_action (sh->default_miss_action); if (sh->encaps_decaps) { - mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL); + mlx5_hlist_destroy(sh->encaps_decaps); sh->encaps_decaps = NULL; } if (sh->modify_cmds) { - mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL); + mlx5_hlist_destroy(sh->modify_cmds); sh->modify_cmds = NULL; } if (sh->tag_table) { /* tags should be destroyed with flow before. */ - mlx5_hlist_destroy(sh->tag_table, NULL, NULL); + mlx5_hlist_destroy(sh->tag_table); sh->tag_table = NULL; } if (sh->tunnel_hub) { @@ -1472,7 +1475,9 @@ err_secondary: mlx5_flow_ext_mreg_supported(eth_dev) && priv->sh->dv_regc0_mask) { priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME, - MLX5_FLOW_MREG_HTABLE_SZ); + MLX5_FLOW_MREG_HTABLE_SZ, + 0, 0, + NULL, NULL, NULL); if (!priv->mreg_cp_tbl) { err = ENOMEM; goto error; @@ -1483,7 +1488,7 @@ err_secondary: error: if (priv) { if (priv->mreg_cp_tbl) - mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL); + mlx5_hlist_destroy(priv->mreg_cp_tbl); if (priv->sh) mlx5_os_free_shared_dr(priv); if (priv->nl_socket_route >= 0) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 2ebc179864..c74b55ecca 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -1034,7 +1034,7 @@ mlx5_free_table_hash_list(struct mlx5_priv *priv) if (!sh->flow_tbls) return; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64); + pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); if (pos) { tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, entry); @@ -1043,7 +1043,7 @@ mlx5_free_table_hash_list(struct mlx5_priv *priv) mlx5_free(tbl_data); } table_key.direction = 1; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64); + pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); if (pos) { tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, entry); @@ -1053,7 +1053,7 @@ mlx5_free_table_hash_list(struct mlx5_priv *priv) } table_key.direction = 0; table_key.domain = 1; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64); + pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); if (pos) { tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, entry); @@ -1061,7 +1061,7 @@ mlx5_free_table_hash_list(struct mlx5_priv *priv) mlx5_hlist_remove(sh->flow_tbls, pos); mlx5_free(tbl_data); } - mlx5_hlist_destroy(sh->flow_tbls, NULL, NULL); + mlx5_hlist_destroy(sh->flow_tbls); } /** @@ -1083,7 +1083,8 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv) MLX5_ASSERT(sh); snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); - sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE); + sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, + 0, 0, NULL, NULL, NULL); if (!sh->flow_tbls) { DRV_LOG(ERR, "flow tables with hash creation failed."); err = ENOMEM; @@ -1311,7 +1312,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) if (priv->drop_queue.hrxq) mlx5_drop_action_destroy(dev); if (priv->mreg_cp_tbl) - mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL); + mlx5_hlist_destroy(priv->mreg_cp_tbl); mlx5_mprq_free_mp(dev); mlx5_os_free_shared_dr(priv); if (priv->rss_conf.rss_key != NULL) diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index dfc713a60a..ac4a7845e8 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -3747,7 +3747,7 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, cp_mreg.src = ret; /* Check if already registered. */ MLX5_ASSERT(priv->mreg_cp_tbl); - mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id); + mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, NULL); if (mcp_res) { /* For non-default rule. */ if (mark_id != MLX5_DEFAULT_COPY_ID) @@ -3824,8 +3824,7 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, goto error; mcp_res->refcnt++; mcp_res->hlist_ent.key = mark_id; - ret = mlx5_hlist_insert(priv->mreg_cp_tbl, - &mcp_res->hlist_ent); + ret = !mlx5_hlist_insert(priv->mreg_cp_tbl, &mcp_res->hlist_ent); MLX5_ASSERT(!ret); if (ret) goto error; @@ -3975,7 +3974,7 @@ flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) if (!priv->mreg_cp_tbl) return; mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, - MLX5_DEFAULT_COPY_ID); + MLX5_DEFAULT_COPY_ID, NULL); if (!mcp_res) return; MLX5_ASSERT(mcp_res->rix_flow); @@ -7562,7 +7561,7 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) .direction = 0, } }; - he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64); + he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); return he ? container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; } @@ -7584,7 +7583,7 @@ tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, struct mlx5_hlist *group_hash; group_hash = tunnel ? tunnel->groups : thub->groups; - he = mlx5_hlist_lookup(group_hash, key.val); + he = mlx5_hlist_lookup(group_hash, key.val, NULL); if (!he) { tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*tte), 0, @@ -8091,7 +8090,7 @@ mlx5_flow_tunnel_free(struct rte_eth_dev *dev, LIST_REMOVE(tunnel, chain); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID], tunnel->tunnel_id); - mlx5_hlist_destroy(tunnel->groups, NULL, NULL); + mlx5_hlist_destroy(tunnel->groups); mlx5_free(tunnel); } @@ -8139,7 +8138,8 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id); return NULL; } - tunnel->groups = mlx5_hlist_create("tunnel groups", 1024); + tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0, + NULL, NULL, NULL); if (!tunnel->groups) { mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id); @@ -8204,7 +8204,7 @@ void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) return; if (!LIST_EMPTY(&thub->tunnels)) DRV_LOG(WARNING, "port %u tunnels present\n", port_id); - mlx5_hlist_destroy(thub->groups, NULL, NULL); + mlx5_hlist_destroy(thub->groups); mlx5_free(thub); } @@ -8218,7 +8218,8 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) if (!thub) return -ENOMEM; LIST_INIT(&thub->tunnels); - thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES); + thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0, + 0, NULL, NULL, NULL); if (!thub->groups) { err = -rte_errno; goto err; @@ -8229,7 +8230,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) err: if (thub->groups) - mlx5_hlist_destroy(thub->groups, NULL, NULL); + mlx5_hlist_destroy(thub->groups); if (thub) mlx5_free(thub); return err; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 4322c59a83..13bc46d512 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -7961,7 +7961,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, } }; struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, - table_key.v64); + table_key.v64, NULL); struct mlx5_flow_tbl_data_entry *tbl_data; uint32_t idx = 0; int ret; @@ -8022,7 +8022,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev, } } pos->key = table_key.v64; - ret = mlx5_hlist_insert(sh->flow_tbls, pos); + ret = !mlx5_hlist_insert(sh->flow_tbls, pos); if (ret < 0) { rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -8079,7 +8079,8 @@ flow_dv_tbl_resource_release(struct rte_eth_dev *dev, tunnel_grp_hash = tbl_data->tunnel ? tbl_data->tunnel->groups : thub->groups; - he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val); + he = mlx5_hlist_lookup(tunnel_grp_hash, + tunnel_key.val, NULL); if (he) { struct tunnel_tbl_entry *tte; tte = container_of(he, typeof(*tte), hash); @@ -8238,7 +8239,7 @@ flow_dv_tag_resource_register int ret; /* Lookup a matching resource from cache. */ - entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24); + entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24, NULL); if (entry) { cache_resource = container_of (entry, struct mlx5_flow_dv_tag_resource, entry); diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c index 7a6b0c69ed..d041b07801 100644 --- a/drivers/net/mlx5/mlx5_utils.c +++ b/drivers/net/mlx5/mlx5_utils.c @@ -9,14 +9,40 @@ #include "mlx5_utils.h" +/********************* Hash List **********************/ + +static struct mlx5_hlist_entry * +mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused, + void *ctx __rte_unused) +{ + return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY); +} + +static void +mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused, + struct mlx5_hlist_entry *entry) +{ + mlx5_free(entry); +} + +static int +mlx5_hlist_default_match_cb(struct mlx5_hlist *h __rte_unused, + struct mlx5_hlist_entry *entry, + uint64_t key, void *ctx __rte_unused) +{ + return entry->key != key; +} + struct mlx5_hlist * -mlx5_hlist_create(const char *name, uint32_t size) +mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size, + uint32_t flags, mlx5_hlist_create_cb cb_create, + mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove) { struct mlx5_hlist *h; uint32_t act_size; uint32_t alloc_size; - if (!size) + if (!size || (!cb_create ^ !cb_remove)) return NULL; /* Align to the next power of 2, 32bits integer is enough now. */ if (!rte_is_power_of_2(size)) { @@ -40,45 +66,108 @@ mlx5_hlist_create(const char *name, uint32_t size) snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name); h->table_sz = act_size; h->mask = act_size - 1; + h->entry_sz = entry_size; + h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY); + h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST); + h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb; + h->cb_match = cb_match ? cb_match : mlx5_hlist_default_match_cb; + h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb; + rte_rwlock_init(&h->lock); DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.", h->name, act_size); return h; } -struct mlx5_hlist_entry * -mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key) +static struct mlx5_hlist_entry * +__hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx, bool reuse) { uint32_t idx; struct mlx5_hlist_head *first; struct mlx5_hlist_entry *node; MLX5_ASSERT(h); - idx = rte_hash_crc_8byte(key, 0) & h->mask; + if (h->direct_key) + idx = (uint32_t)(key & h->mask); + else + idx = rte_hash_crc_8byte(key, 0) & h->mask; first = &h->heads[idx]; LIST_FOREACH(node, first, next) { - if (node->key == key) - return node; + if (!h->cb_match(h, node, key, ctx)) { + if (reuse) { + __atomic_add_fetch(&node->ref_cnt, 1, + __ATOMIC_RELAXED); + DRV_LOG(DEBUG, "Hash list %s entry %p " + "reuse: %u.", + h->name, (void *)node, node->ref_cnt); + } + break; + } } - return NULL; + return node; } -int -mlx5_hlist_insert(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry) +static struct mlx5_hlist_entry * +hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx, bool reuse) +{ + struct mlx5_hlist_entry *node; + + MLX5_ASSERT(h); + rte_rwlock_read_lock(&h->lock); + node = __hlist_lookup(h, key, ctx, reuse); + rte_rwlock_read_unlock(&h->lock); + return node; +} + +struct mlx5_hlist_entry * +mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx) +{ + return hlist_lookup(h, key, ctx, false); +} + +struct mlx5_hlist_entry* +mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx) { uint32_t idx; struct mlx5_hlist_head *first; - struct mlx5_hlist_entry *node; + struct mlx5_hlist_entry *entry; + uint32_t prev_gen_cnt = 0; MLX5_ASSERT(h && entry); - idx = rte_hash_crc_8byte(entry->key, 0) & h->mask; + /* Use write lock directly for write-most list. */ + if (!h->write_most) { + prev_gen_cnt = __atomic_load_n(&h->gen_cnt, __ATOMIC_ACQUIRE); + entry = hlist_lookup(h, key, ctx, true); + if (entry) + return entry; + } + rte_rwlock_write_lock(&h->lock); + /* Check if the list changed by other threads. */ + if (h->write_most || + prev_gen_cnt != __atomic_load_n(&h->gen_cnt, __ATOMIC_ACQUIRE)) { + entry = __hlist_lookup(h, key, ctx, true); + if (entry) + goto done; + } + if (h->direct_key) + idx = (uint32_t)(key & h->mask); + else + idx = rte_hash_crc_8byte(key, 0) & h->mask; first = &h->heads[idx]; - /* No need to reuse the lookup function. */ - LIST_FOREACH(node, first, next) { - if (node->key == entry->key) - return -EEXIST; + entry = h->cb_create(h, key, ctx); + if (!entry) { + rte_errno = ENOMEM; + DRV_LOG(ERR, "Can't allocate hash list %s entry.", h->name); + goto done; } + entry->key = key; + entry->ref_cnt = 1; LIST_INSERT_HEAD(first, entry, next); - return 0; + __atomic_add_fetch(&h->gen_cnt, 1, __ATOMIC_ACQ_REL); + DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.", + h->name, (void *)entry, entry->ref_cnt); +done: + rte_rwlock_write_unlock(&h->lock); + return entry; } struct mlx5_hlist_entry * @@ -119,26 +208,36 @@ mlx5_hlist_insert_ex(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry, return 0; } -void -mlx5_hlist_remove(struct mlx5_hlist *h __rte_unused, - struct mlx5_hlist_entry *entry) +int +mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry) { - MLX5_ASSERT(entry && entry->next.le_prev); + rte_rwlock_write_lock(&h->lock); + MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev); + DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.", + h->name, (void *)entry, entry->ref_cnt); + if (--entry->ref_cnt) { + rte_rwlock_write_unlock(&h->lock); + return 1; + } LIST_REMOVE(entry, next); /* Set to NULL to get rid of removing action for more than once. */ entry->next.le_prev = NULL; + h->cb_remove(h, entry); + rte_rwlock_write_unlock(&h->lock); + DRV_LOG(DEBUG, "Hash list %s entry %p removed.", + h->name, (void *)entry); + return 0; } void -mlx5_hlist_destroy(struct mlx5_hlist *h, - mlx5_hlist_destroy_callback_fn cb, void *ctx) +mlx5_hlist_destroy(struct mlx5_hlist *h) { uint32_t idx; struct mlx5_hlist_entry *entry; MLX5_ASSERT(h); for (idx = 0; idx < h->table_sz; ++idx) { - /* no LIST_FOREACH_SAFE, using while instead */ + /* No LIST_FOREACH_SAFE, using while instead. */ while (!LIST_EMPTY(&h->heads[idx])) { entry = LIST_FIRST(&h->heads[idx]); LIST_REMOVE(entry, next); @@ -150,15 +249,14 @@ mlx5_hlist_destroy(struct mlx5_hlist *h, * the beginning). Or else the default free function * will be used. */ - if (cb) - cb(entry, ctx); - else - mlx5_free(entry); + h->cb_remove(h, entry); } } mlx5_free(h); } +/********************* Indexed pool **********************/ + static inline void mlx5_ipool_lock(struct mlx5_indexed_pool *pool) { diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h index ca9bb76cf0..c665558b0a 100644 --- a/drivers/net/mlx5/mlx5_utils.h +++ b/drivers/net/mlx5/mlx5_utils.h @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -20,6 +21,11 @@ #include "mlx5_defs.h" +#define mlx5_hlist_remove(h, e) \ + mlx5_hlist_unregister(h, e) + +#define mlx5_hlist_insert(h, e) \ + mlx5_hlist_register(h, 0, e) /* Convert a bit number to the corresponding 64-bit mask */ #define MLX5_BITSHIFT(v) (UINT64_C(1) << (v)) @@ -259,9 +265,14 @@ log2above(unsigned int v) return l + r; } +#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */ +#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */ + /** Maximum size of string for naming the hlist table. */ #define MLX5_HLIST_NAMESIZE 32 +struct mlx5_hlist; + /** * Structure of the entry in the hash list, user should define its own struct * that contains this in order to store the data. The 'key' is 64-bits right @@ -270,6 +281,7 @@ log2above(unsigned int v) struct mlx5_hlist_entry { LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */ uint64_t key; /* user defined 'key', could be the hash signature. */ + uint32_t ref_cnt; /* Reference count. */ }; /** Structure for hash head. */ @@ -292,13 +304,77 @@ typedef void (*mlx5_hlist_destroy_callback_fn)(void *p, void *ctx); typedef int (*mlx5_hlist_match_callback_fn)(struct mlx5_hlist_entry *entry, void *ctx); -/** hash list table structure */ +/** + * Type of callback function for entry removal. + * + * @param list + * The hash list. + * @param entry + * The entry in the list. + */ +typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry); + +/** + * Type of function for user defined matching. + * + * @param list + * The hash list. + * @param entry + * The entry in the list. + * @param key + * The new entry key. + * @param ctx + * The pointer to new entry context. + * + * @return + * 0 if matching, non-zero number otherwise. + */ +typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry, + uint64_t key, void *ctx); + +/** + * Type of function for user defined hash list entry creation. + * + * @param list + * The hash list. + * @param key + * The key of the new entry. + * @param ctx + * The pointer to new entry context. + * + * @return + * Pointer to allocated entry on success, NULL otherwise. + */ +typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb) + (struct mlx5_hlist *list, + uint64_t key, void *ctx); + +/** + * Hash list table structure + * + * Entry in hash list could be reused if entry already exists, reference + * count will increase and the existing entry returns. + * + * When destroy an entry from list, decrease reference count and only + * destroy when no further reference. + */ struct mlx5_hlist { char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */ /**< number of heads, need to be power of 2. */ uint32_t table_sz; + uint32_t entry_sz; /**< Size of entry, used to allocate entry. */ /**< mask to get the index of the list heads. */ uint32_t mask; + rte_rwlock_t lock; + uint32_t gen_cnt; /* List modification will update generation count. */ + bool direct_key; /* Use the new entry key directly as hash index. */ + bool write_most; /* List mostly used for append new or destroy. */ + void *ctx; + mlx5_hlist_create_cb cb_create; /**< entry create callback. */ + mlx5_hlist_match_cb cb_match; /**< entry match callback. */ + mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */ struct mlx5_hlist_head heads[]; /**< list head arrays. */ }; @@ -314,40 +390,43 @@ struct mlx5_hlist { * Name of the hash list(optional). * @param size * Heads array size of the hash list. - * + * @param entry_size + * Entry size to allocate if cb_create not specified. + * @param flags + * The hash list attribute flags. + * @param cb_create + * Callback function for entry create. + * @param cb_match + * Callback function for entry match. + * @param cb_destroy + * Callback function for entry destroy. * @return * Pointer of the hash list table created, NULL on failure. */ -struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size); +struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size, + uint32_t entry_size, uint32_t flags, + mlx5_hlist_create_cb cb_create, + mlx5_hlist_match_cb cb_match, + mlx5_hlist_remove_cb cb_destroy); /** * Search an entry matching the key. * + * Result returned might be destroyed by other thread, must use + * this function only in main thread. + * * @param h * Pointer to the hast list table. * @param key * Key for the searching entry. + * @param ctx + * Common context parameter used by entry callback function. * * @return * Pointer of the hlist entry if found, NULL otherwise. */ -struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key); - -/** - * Insert an entry to the hash list table, the entry is only part of whole data - * element and a 64B key is used for matching. User should construct the key or - * give a calculated hash signature and guarantee there is no collision. - * - * @param h - * Pointer to the hast list table. - * @param entry - * Entry to be inserted into the hash list table. - * - * @return - * - zero for success. - * - -EEXIST if the entry is already inserted. - */ -int mlx5_hlist_insert(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry); +struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, + void *ctx); /** * Extended routine to search an entry matching the context with @@ -392,6 +471,24 @@ struct mlx5_hlist_entry *mlx5_hlist_lookup_ex(struct mlx5_hlist *h, int mlx5_hlist_insert_ex(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry, mlx5_hlist_match_callback_fn cb, void *ctx); +/** + * Insert an entry to the hash list table, the entry is only part of whole data + * element and a 64B key is used for matching. User should construct the key or + * give a calculated hash signature and guarantee there is no collision. + * + * @param h + * Pointer to the hast list table. + * @param entry + * Entry to be inserted into the hash list table. + * @param ctx + * Common context parameter used by callback function. + * + * @return + * registered entry on success, NULL otherwise + */ +struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, + void *ctx); + /** * Remove an entry from the hash list table. User should guarantee the validity * of the entry. @@ -400,9 +497,10 @@ int mlx5_hlist_insert_ex(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry, * Pointer to the hast list table. (not used) * @param entry * Entry to be removed from the hash list table. + * @return + * 0 on entry removed, 1 on entry still referenced. */ -void mlx5_hlist_remove(struct mlx5_hlist *h __rte_unused, - struct mlx5_hlist_entry *entry); +int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry); /** * Destroy the hash list table, all the entries already inserted into the lists @@ -411,13 +509,8 @@ void mlx5_hlist_remove(struct mlx5_hlist *h __rte_unused, * * @param h * Pointer to the hast list table. - * @param cb - * Callback function for each inserted entry when destroying the hash list. - * @param ctx - * Common context parameter used by callback function for each entry. */ -void mlx5_hlist_destroy(struct mlx5_hlist *h, - mlx5_hlist_destroy_callback_fn cb, void *ctx); +void mlx5_hlist_destroy(struct mlx5_hlist *h); /** * This function allocates non-initialized memory entry from pool. -- 2.20.1