From: Shiri Kuzin Date: Tue, 4 May 2021 17:54:55 +0000 (+0300) Subject: common/mlx5: share hash list tool X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=25245d5dc9ecfa8bc9964c69a756beca6ee1ca72;p=dpdk.git common/mlx5: share hash list tool In order to use the hash list defined in net in other drivers, the hash list is moved to common utilities. In addition, the log definition was moved from the common utilities to a dedicated new log file in common in order to prevent a conflict. Signed-off-by: Shiri Kuzin Acked-by: Matan Azrad --- diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c index 5cf9576921..fba8245b8b 100644 --- a/drivers/common/mlx5/linux/mlx5_common_os.c +++ b/drivers/common/mlx5/linux/mlx5_common_os.c @@ -15,7 +15,7 @@ #include #include "mlx5_common.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_glue.h" #ifdef MLX5_GLUE diff --git a/drivers/common/mlx5/linux/mlx5_nl.c b/drivers/common/mlx5/linux/mlx5_nl.c index 752c57b33d..f0d04f9473 100644 --- a/drivers/common/mlx5/linux/mlx5_nl.c +++ b/drivers/common/mlx5/linux/mlx5_nl.c @@ -20,7 +20,7 @@ #include #include "mlx5_nl.h" -#include "mlx5_common_utils.h" +#include "../mlx5_common_log.h" #include "mlx5_malloc.h" #ifdef HAVE_DEVLINK #include diff --git a/drivers/common/mlx5/meson.build b/drivers/common/mlx5/meson.build index e78b4f47bc..fa2b8b9834 100644 --- a/drivers/common/mlx5/meson.build +++ b/drivers/common/mlx5/meson.build @@ -16,6 +16,7 @@ sources += files( 'mlx5_malloc.c', 'mlx5_common_pci.c', 'mlx5_common_devx.c', + 'mlx5_common_utils.c', ) cflags_options = [ diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c index f92f05bda5..d397459a3d 100644 --- a/drivers/common/mlx5/mlx5_common.c +++ b/drivers/common/mlx5/mlx5_common.c @@ -11,7 +11,7 @@ #include "mlx5_common.h" #include "mlx5_common_os.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_common_pci.h" uint8_t haswell_broadwell_cpu; diff --git a/drivers/common/mlx5/mlx5_common_devx.c b/drivers/common/mlx5/mlx5_common_devx.c index d19be122bd..22c8d356c4 100644 --- a/drivers/common/mlx5/mlx5_common_devx.c +++ b/drivers/common/mlx5/mlx5_common_devx.c @@ -12,7 +12,7 @@ #include "mlx5_prm.h" #include "mlx5_devx_cmds.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_malloc.h" #include "mlx5_common.h" #include "mlx5_common_devx.h" diff --git a/drivers/common/mlx5/mlx5_common_log.h b/drivers/common/mlx5/mlx5_common_log.h new file mode 100644 index 0000000000..26b13fedaf --- /dev/null +++ b/drivers/common/mlx5/mlx5_common_log.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 Mellanox Technologies, Ltd + */ + +#ifndef RTE_PMD_MLX5_COMMON_LOG_H_ +#define RTE_PMD_MLX5_COMMON_LOG_H_ + +#include "mlx5_common.h" + + +extern int mlx5_common_logtype; + +#define MLX5_COMMON_LOG_PREFIX "mlx5_common" +/* Generic printf()-like logging macro with automatic line feed. */ +#define DRV_LOG(level, ...) \ + PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \ + __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \ + PMD_DRV_LOG_CPAREN) + +#endif /* RTE_PMD_MLX5_COMMON_LOG_H_ */ + diff --git a/drivers/common/mlx5/mlx5_common_mp.c b/drivers/common/mlx5/mlx5_common_mp.c index 40e3956e45..673a7c31de 100644 --- a/drivers/common/mlx5/mlx5_common_mp.c +++ b/drivers/common/mlx5/mlx5_common_mp.c @@ -10,7 +10,7 @@ #include #include "mlx5_common_mp.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_malloc.h" /** diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c index e1ed0caf3a..afb5b3d0a7 100644 --- a/drivers/common/mlx5/mlx5_common_mr.c +++ b/drivers/common/mlx5/mlx5_common_mr.c @@ -11,7 +11,7 @@ #include "mlx5_glue.h" #include "mlx5_common_mp.h" #include "mlx5_common_mr.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_malloc.h" struct mr_find_contig_memsegs_data { diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c index 5a73ffa60a..3f16cd21cf 100644 --- a/drivers/common/mlx5/mlx5_common_pci.c +++ b/drivers/common/mlx5/mlx5_common_pci.c @@ -4,7 +4,9 @@ #include #include -#include "mlx5_common_utils.h" +#include + +#include "mlx5_common_log.h" #include "mlx5_common_pci.h" struct mlx5_pci_device { diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c new file mode 100644 index 0000000000..ad2011e858 --- /dev/null +++ b/drivers/common/mlx5/mlx5_common_utils.c @@ -0,0 +1,221 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 Mellanox Technologies, Ltd + */ + +#include +#include +#include + +#include + +#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" + +/********************* Hash List **********************/ + +static struct mlx5_hlist_entry * +mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused, + void *ctx __rte_unused) +{ + return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY); +} + +static void +mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused, + struct mlx5_hlist_entry *entry) +{ + mlx5_free(entry); +} + +struct mlx5_hlist * +mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size, + uint32_t flags, mlx5_hlist_create_cb cb_create, + mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove) +{ + struct mlx5_hlist *h; + uint32_t act_size; + uint32_t alloc_size; + uint32_t i; + + if (!size || !cb_match || (!cb_create ^ !cb_remove)) + return NULL; + /* Align to the next power of 2, 32bits integer is enough now. */ + if (!rte_is_power_of_2(size)) { + act_size = rte_align32pow2(size); + DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, " + "will be aligned to 0x%" PRIX32 ".", size, act_size); + } else { + act_size = size; + } + alloc_size = sizeof(struct mlx5_hlist) + + sizeof(struct mlx5_hlist_bucket) * act_size; + /* Using zmalloc, then no need to initialize the heads. */ + h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE, + SOCKET_ID_ANY); + if (!h) { + DRV_LOG(ERR, "No memory for hash list %s creation", + name ? name : "None"); + return NULL; + } + if (name) + snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name); + h->table_sz = act_size; + h->mask = act_size - 1; + h->entry_sz = entry_size; + h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY); + h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST); + h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb; + h->cb_match = cb_match; + h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb; + for (i = 0; i < act_size; i++) + rte_rwlock_init(&h->buckets[i].lock); + DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.", + h->name, act_size); + return h; +} + +static struct mlx5_hlist_entry * +__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx, + void *ctx, bool reuse) +{ + struct mlx5_hlist_head *first; + struct mlx5_hlist_entry *node; + + MLX5_ASSERT(h); + first = &h->buckets[idx].head; + LIST_FOREACH(node, first, next) { + if (!h->cb_match(h, node, key, ctx)) { + if (reuse) { + __atomic_add_fetch(&node->ref_cnt, 1, + __ATOMIC_RELAXED); + DRV_LOG(DEBUG, "Hash list %s entry %p " + "reuse: %u.", + h->name, (void *)node, node->ref_cnt); + } + break; + } + } + return node; +} + +static struct mlx5_hlist_entry * +hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx, + void *ctx, bool reuse) +{ + struct mlx5_hlist_entry *node; + + MLX5_ASSERT(h); + rte_rwlock_read_lock(&h->buckets[idx].lock); + node = __hlist_lookup(h, key, idx, ctx, reuse); + rte_rwlock_read_unlock(&h->buckets[idx].lock); + return node; +} + +struct mlx5_hlist_entry * +mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx) +{ + uint32_t idx; + + if (h->direct_key) + idx = (uint32_t)(key & h->mask); + else + idx = rte_hash_crc_8byte(key, 0) & h->mask; + return hlist_lookup(h, key, idx, ctx, false); +} + +struct mlx5_hlist_entry* +mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx) +{ + uint32_t idx; + struct mlx5_hlist_head *first; + struct mlx5_hlist_bucket *b; + struct mlx5_hlist_entry *entry; + uint32_t prev_gen_cnt = 0; + + if (h->direct_key) + idx = (uint32_t)(key & h->mask); + else + idx = rte_hash_crc_8byte(key, 0) & h->mask; + MLX5_ASSERT(h); + b = &h->buckets[idx]; + /* Use write lock directly for write-most list. */ + if (!h->write_most) { + prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE); + entry = hlist_lookup(h, key, idx, ctx, true); + if (entry) + return entry; + } + rte_rwlock_write_lock(&b->lock); + /* Check if the list changed by other threads. */ + if (h->write_most || + prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) { + entry = __hlist_lookup(h, key, idx, ctx, true); + if (entry) + goto done; + } + first = &b->head; + entry = h->cb_create(h, key, ctx); + if (!entry) { + rte_errno = ENOMEM; + DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name); + goto done; + } + entry->idx = idx; + entry->ref_cnt = 1; + LIST_INSERT_HEAD(first, entry, next); + __atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL); + DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.", + h->name, (void *)entry, entry->ref_cnt); +done: + rte_rwlock_write_unlock(&b->lock); + return entry; +} + +int +mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry) +{ + uint32_t idx = entry->idx; + + rte_rwlock_write_lock(&h->buckets[idx].lock); + MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev); + DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.", + h->name, (void *)entry, entry->ref_cnt); + if (--entry->ref_cnt) { + rte_rwlock_write_unlock(&h->buckets[idx].lock); + return 1; + } + LIST_REMOVE(entry, next); + /* Set to NULL to get rid of removing action for more than once. */ + entry->next.le_prev = NULL; + h->cb_remove(h, entry); + rte_rwlock_write_unlock(&h->buckets[idx].lock); + DRV_LOG(DEBUG, "Hash list %s entry %p removed.", + h->name, (void *)entry); + return 0; +} + +void +mlx5_hlist_destroy(struct mlx5_hlist *h) +{ + uint32_t idx; + struct mlx5_hlist_entry *entry; + + MLX5_ASSERT(h); + for (idx = 0; idx < h->table_sz; ++idx) { + /* No LIST_FOREACH_SAFE, using while instead. */ + while (!LIST_EMPTY(&h->buckets[idx].head)) { + entry = LIST_FIRST(&h->buckets[idx].head); + LIST_REMOVE(entry, next); + /* + * The owner of whole element which contains data entry + * is the user, so it's the user's duty to do the clean + * up and the free work because someone may not put the + * hlist entry at the beginning(suggested to locate at + * the beginning). Or else the default free function + * will be used. + */ + h->cb_remove(h, entry); + } + } + mlx5_free(h); +} diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h index 6cba39c8cc..ed378ce9bd 100644 --- a/drivers/common/mlx5/mlx5_common_utils.h +++ b/drivers/common/mlx5/mlx5_common_utils.h @@ -7,14 +7,202 @@ #include "mlx5_common.h" +#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */ +#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */ -extern int mlx5_common_logtype; +/** Maximum size of string for naming the hlist table. */ +#define MLX5_HLIST_NAMESIZE 32 -#define MLX5_COMMON_LOG_PREFIX "mlx5_common" -/* Generic printf()-like logging macro with automatic line feed. */ -#define DRV_LOG(level, ...) \ - PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \ - __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \ - PMD_DRV_LOG_CPAREN) +struct mlx5_hlist; + +/** + * Structure of the entry in the hash list, user should define its own struct + * that contains this in order to store the data. The 'key' is 64-bits right + * now and its user's responsibility to guarantee there is no collision. + */ +struct mlx5_hlist_entry { + LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */ + uint32_t idx; /* Bucket index the entry belongs to. */ + uint32_t ref_cnt; /* Reference count. */ +}; + +/** Structure for hash head. */ +LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry); + +/** + * Type of callback function for entry removal. + * + * @param list + * The hash list. + * @param entry + * The entry in the list. + */ +typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry); + +/** + * Type of function for user defined matching. + * + * @param list + * The hash list. + * @param entry + * The entry in the list. + * @param key + * The new entry key. + * @param ctx + * The pointer to new entry context. + * + * @return + * 0 if matching, non-zero number otherwise. + */ +typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry, + uint64_t key, void *ctx); + +/** + * Type of function for user defined hash list entry creation. + * + * @param list + * The hash list. + * @param key + * The key of the new entry. + * @param ctx + * The pointer to new entry context. + * + * @return + * Pointer to allocated entry on success, NULL otherwise. + */ +typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb) + (struct mlx5_hlist *list, + uint64_t key, void *ctx); + +/* Hash list bucket head. */ +struct mlx5_hlist_bucket { + struct mlx5_hlist_head head; /* List head. */ + rte_rwlock_t lock; /* Bucket lock. */ + uint32_t gen_cnt; /* List modification will update generation count. */ +} __rte_cache_aligned; + +/** + * Hash list table structure + * + * Entry in hash list could be reused if entry already exists, reference + * count will increase and the existing entry returns. + * + * When destroy an entry from list, decrease reference count and only + * destroy when no further reference. + */ +struct mlx5_hlist { + char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */ + /**< number of heads, need to be power of 2. */ + uint32_t table_sz; + uint32_t entry_sz; /**< Size of entry, used to allocate entry. */ + /**< mask to get the index of the list heads. */ + uint32_t mask; + bool direct_key; /* Use the new entry key directly as hash index. */ + bool write_most; /* List mostly used for append new or destroy. */ + void *ctx; + mlx5_hlist_create_cb cb_create; /**< entry create callback. */ + mlx5_hlist_match_cb cb_match; /**< entry match callback. */ + mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */ + struct mlx5_hlist_bucket buckets[] __rte_cache_aligned; + /**< list bucket arrays. */ +}; + +/** + * Create a hash list table, the user can specify the list heads array size + * of the table, now the size should be a power of 2 in order to get better + * distribution for the entries. Each entry is a part of the whole data element + * and the caller should be responsible for the data element's allocation and + * cleanup / free. Key of each entry will be calculated with CRC in order to + * generate a little fairer distribution. + * + * @param name + * Name of the hash list(optional). + * @param size + * Heads array size of the hash list. + * @param entry_size + * Entry size to allocate if cb_create not specified. + * @param flags + * The hash list attribute flags. + * @param cb_create + * Callback function for entry create. + * @param cb_match + * Callback function for entry match. + * @param cb_destroy + * Callback function for entry destroy. + * @return + * Pointer of the hash list table created, NULL on failure. + */ +__rte_internal +struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size, + uint32_t entry_size, uint32_t flags, + mlx5_hlist_create_cb cb_create, + mlx5_hlist_match_cb cb_match, + mlx5_hlist_remove_cb cb_destroy); + +/** + * Search an entry matching the key. + * + * Result returned might be destroyed by other thread, must use + * this function only in main thread. + * + * @param h + * Pointer to the hast list table. + * @param key + * Key for the searching entry. + * @param ctx + * Common context parameter used by entry callback function. + * + * @return + * Pointer of the hlist entry if found, NULL otherwise. + */ +__rte_internal +struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, + void *ctx); + +/** + * Insert an entry to the hash list table, the entry is only part of whole data + * element and a 64B key is used for matching. User should construct the key or + * give a calculated hash signature and guarantee there is no collision. + * + * @param h + * Pointer to the hast list table. + * @param entry + * Entry to be inserted into the hash list table. + * @param ctx + * Common context parameter used by callback function. + * + * @return + * registered entry on success, NULL otherwise + */ +__rte_internal +struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, + void *ctx); + +/** + * Remove an entry from the hash list table. User should guarantee the validity + * of the entry. + * + * @param h + * Pointer to the hast list table. (not used) + * @param entry + * Entry to be removed from the hash list table. + * @return + * 0 on entry removed, 1 on entry still referenced. + */ +__rte_internal +int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry); + +/** + * Destroy the hash list table, all the entries already inserted into the lists + * will be handled by the callback function provided by the user (including + * free if needed) before the table is freed. + * + * @param h + * Pointer to the hast list table. + */ +__rte_internal +void mlx5_hlist_destroy(struct mlx5_hlist *h); #endif /* RTE_PMD_MLX5_COMMON_UTILS_H_ */ diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index 703b18cc32..6b7684481f 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -9,7 +9,7 @@ #include "mlx5_prm.h" #include "mlx5_devx_cmds.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_malloc.h" diff --git a/drivers/common/mlx5/mlx5_malloc.c b/drivers/common/mlx5/mlx5_malloc.c index 9d30cedbaa..b19501e1bc 100644 --- a/drivers/common/mlx5/mlx5_malloc.c +++ b/drivers/common/mlx5/mlx5_malloc.c @@ -8,7 +8,7 @@ #include #include -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_common_os.h" #include "mlx5_malloc.h" diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map index 89f0ee04cb..40b0d713ac 100644 --- a/drivers/common/mlx5/version.map +++ b/drivers/common/mlx5/version.map @@ -70,6 +70,12 @@ INTERNAL { mlx5_glue; + mlx5_hlist_create; + mlx5_hlist_lookup; + mlx5_hlist_register; + mlx5_hlist_unregister; + mlx5_hlist_destroy; + mlx5_malloc; mlx5_malloc_mem_select; diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c index f2d781a965..2e6e172a96 100644 --- a/drivers/common/mlx5/windows/mlx5_common_os.c +++ b/drivers/common/mlx5/windows/mlx5_common_os.c @@ -11,7 +11,7 @@ #include #include "mlx5_devx_cmds.h" -#include "mlx5_common_utils.h" +#include "../mlx5_common_log.h" #include "mlx5_common.h" #include "mlx5_common_os.h" #include "mlx5_malloc.h" diff --git a/drivers/common/mlx5/windows/mlx5_glue.c b/drivers/common/mlx5/windows/mlx5_glue.c index aef6d3b5f4..535487a8d4 100644 --- a/drivers/common/mlx5/windows/mlx5_glue.c +++ b/drivers/common/mlx5/windows/mlx5_glue.c @@ -12,7 +12,7 @@ #include #include "mlx5_glue.h" -#include "mlx5_common_utils.h" +#include "../mlx5_common_log.h" #include "mlx5_win_ext.h" /* diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c index a39b5edddc..18fe23e4fb 100644 --- a/drivers/net/mlx5/mlx5_utils.c +++ b/drivers/net/mlx5/mlx5_utils.c @@ -3,220 +3,11 @@ */ #include -#include #include #include "mlx5_utils.h" -/********************* Hash List **********************/ - -static struct mlx5_hlist_entry * -mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused, - void *ctx __rte_unused) -{ - return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY); -} - -static void -mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused, - struct mlx5_hlist_entry *entry) -{ - mlx5_free(entry); -} - -struct mlx5_hlist * -mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size, - uint32_t flags, mlx5_hlist_create_cb cb_create, - mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove) -{ - struct mlx5_hlist *h; - uint32_t act_size; - uint32_t alloc_size; - uint32_t i; - - if (!size || !cb_match || (!cb_create ^ !cb_remove)) - return NULL; - /* Align to the next power of 2, 32bits integer is enough now. */ - if (!rte_is_power_of_2(size)) { - act_size = rte_align32pow2(size); - DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, " - "will be aligned to 0x%" PRIX32 ".", size, act_size); - } else { - act_size = size; - } - alloc_size = sizeof(struct mlx5_hlist) + - sizeof(struct mlx5_hlist_bucket) * act_size; - /* Using zmalloc, then no need to initialize the heads. */ - h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE, - SOCKET_ID_ANY); - if (!h) { - DRV_LOG(ERR, "No memory for hash list %s creation", - name ? name : "None"); - return NULL; - } - if (name) - snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name); - h->table_sz = act_size; - h->mask = act_size - 1; - h->entry_sz = entry_size; - h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY); - h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST); - h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb; - h->cb_match = cb_match; - h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb; - for (i = 0; i < act_size; i++) - rte_rwlock_init(&h->buckets[i].lock); - DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.", - h->name, act_size); - return h; -} - -static struct mlx5_hlist_entry * -__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx, - void *ctx, bool reuse) -{ - struct mlx5_hlist_head *first; - struct mlx5_hlist_entry *node; - - MLX5_ASSERT(h); - first = &h->buckets[idx].head; - LIST_FOREACH(node, first, next) { - if (!h->cb_match(h, node, key, ctx)) { - if (reuse) { - __atomic_add_fetch(&node->ref_cnt, 1, - __ATOMIC_RELAXED); - DRV_LOG(DEBUG, "Hash list %s entry %p " - "reuse: %u.", - h->name, (void *)node, node->ref_cnt); - } - break; - } - } - return node; -} - -static struct mlx5_hlist_entry * -hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx, - void *ctx, bool reuse) -{ - struct mlx5_hlist_entry *node; - - MLX5_ASSERT(h); - rte_rwlock_read_lock(&h->buckets[idx].lock); - node = __hlist_lookup(h, key, idx, ctx, reuse); - rte_rwlock_read_unlock(&h->buckets[idx].lock); - return node; -} - -struct mlx5_hlist_entry * -mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx) -{ - uint32_t idx; - - if (h->direct_key) - idx = (uint32_t)(key & h->mask); - else - idx = rte_hash_crc_8byte(key, 0) & h->mask; - return hlist_lookup(h, key, idx, ctx, false); -} - -struct mlx5_hlist_entry* -mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx) -{ - uint32_t idx; - struct mlx5_hlist_head *first; - struct mlx5_hlist_bucket *b; - struct mlx5_hlist_entry *entry; - uint32_t prev_gen_cnt = 0; - - if (h->direct_key) - idx = (uint32_t)(key & h->mask); - else - idx = rte_hash_crc_8byte(key, 0) & h->mask; - MLX5_ASSERT(h); - b = &h->buckets[idx]; - /* Use write lock directly for write-most list. */ - if (!h->write_most) { - prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE); - entry = hlist_lookup(h, key, idx, ctx, true); - if (entry) - return entry; - } - rte_rwlock_write_lock(&b->lock); - /* Check if the list changed by other threads. */ - if (h->write_most || - prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) { - entry = __hlist_lookup(h, key, idx, ctx, true); - if (entry) - goto done; - } - first = &b->head; - entry = h->cb_create(h, key, ctx); - if (!entry) { - rte_errno = ENOMEM; - DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name); - goto done; - } - entry->idx = idx; - entry->ref_cnt = 1; - LIST_INSERT_HEAD(first, entry, next); - __atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL); - DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.", - h->name, (void *)entry, entry->ref_cnt); -done: - rte_rwlock_write_unlock(&b->lock); - return entry; -} - -int -mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry) -{ - uint32_t idx = entry->idx; - - rte_rwlock_write_lock(&h->buckets[idx].lock); - MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev); - DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.", - h->name, (void *)entry, entry->ref_cnt); - if (--entry->ref_cnt) { - rte_rwlock_write_unlock(&h->buckets[idx].lock); - return 1; - } - LIST_REMOVE(entry, next); - /* Set to NULL to get rid of removing action for more than once. */ - entry->next.le_prev = NULL; - h->cb_remove(h, entry); - rte_rwlock_write_unlock(&h->buckets[idx].lock); - DRV_LOG(DEBUG, "Hash list %s entry %p removed.", - h->name, (void *)entry); - return 0; -} - -void -mlx5_hlist_destroy(struct mlx5_hlist *h) -{ - uint32_t idx; - struct mlx5_hlist_entry *entry; - - MLX5_ASSERT(h); - for (idx = 0; idx < h->table_sz; ++idx) { - /* No LIST_FOREACH_SAFE, using while instead. */ - while (!LIST_EMPTY(&h->buckets[idx].head)) { - entry = LIST_FIRST(&h->buckets[idx].head); - LIST_REMOVE(entry, next); - /* - * The owner of whole element which contains data entry - * is the user, so it's the user's duty to do the clean - * up and the free work because someone may not put the - * hlist entry at the beginning(suggested to locate at - * the beginning). Or else the default free function - * will be used. - */ - h->cb_remove(h, entry); - } - } - mlx5_free(h); -} /********************* Cache list ************************/ diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h index 289941cebc..b54517c6df 100644 --- a/drivers/net/mlx5/mlx5_utils.h +++ b/drivers/net/mlx5/mlx5_utils.h @@ -18,6 +18,7 @@ #include #include +#include #include "mlx5_defs.h" @@ -261,199 +262,6 @@ log2above(unsigned int v) return l + r; } -#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */ -#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */ - -/** Maximum size of string for naming the hlist table. */ -#define MLX5_HLIST_NAMESIZE 32 - -struct mlx5_hlist; - -/** - * Structure of the entry in the hash list, user should define its own struct - * that contains this in order to store the data. The 'key' is 64-bits right - * now and its user's responsibility to guarantee there is no collision. - */ -struct mlx5_hlist_entry { - LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */ - uint32_t idx; /* Bucket index the entry belongs to. */ - uint32_t ref_cnt; /* Reference count. */ -}; - -/** Structure for hash head. */ -LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry); - -/** - * Type of callback function for entry removal. - * - * @param list - * The hash list. - * @param entry - * The entry in the list. - */ -typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list, - struct mlx5_hlist_entry *entry); - -/** - * Type of function for user defined matching. - * - * @param list - * The hash list. - * @param entry - * The entry in the list. - * @param key - * The new entry key. - * @param ctx - * The pointer to new entry context. - * - * @return - * 0 if matching, non-zero number otherwise. - */ -typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list, - struct mlx5_hlist_entry *entry, - uint64_t key, void *ctx); - -/** - * Type of function for user defined hash list entry creation. - * - * @param list - * The hash list. - * @param key - * The key of the new entry. - * @param ctx - * The pointer to new entry context. - * - * @return - * Pointer to allocated entry on success, NULL otherwise. - */ -typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb) - (struct mlx5_hlist *list, - uint64_t key, void *ctx); - -/* Hash list bucket head. */ -struct mlx5_hlist_bucket { - struct mlx5_hlist_head head; /* List head. */ - rte_rwlock_t lock; /* Bucket lock. */ - uint32_t gen_cnt; /* List modification will update generation count. */ -} __rte_cache_aligned; - -/** - * Hash list table structure - * - * Entry in hash list could be reused if entry already exists, reference - * count will increase and the existing entry returns. - * - * When destroy an entry from list, decrease reference count and only - * destroy when no further reference. - */ -struct mlx5_hlist { - char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */ - /**< number of heads, need to be power of 2. */ - uint32_t table_sz; - uint32_t entry_sz; /**< Size of entry, used to allocate entry. */ - /**< mask to get the index of the list heads. */ - uint32_t mask; - bool direct_key; /* Use the new entry key directly as hash index. */ - bool write_most; /* List mostly used for append new or destroy. */ - void *ctx; - mlx5_hlist_create_cb cb_create; /**< entry create callback. */ - mlx5_hlist_match_cb cb_match; /**< entry match callback. */ - mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */ - struct mlx5_hlist_bucket buckets[] __rte_cache_aligned; - /**< list bucket arrays. */ -}; - -/** - * Create a hash list table, the user can specify the list heads array size - * of the table, now the size should be a power of 2 in order to get better - * distribution for the entries. Each entry is a part of the whole data element - * and the caller should be responsible for the data element's allocation and - * cleanup / free. Key of each entry will be calculated with CRC in order to - * generate a little fairer distribution. - * - * @param name - * Name of the hash list(optional). - * @param size - * Heads array size of the hash list. - * @param entry_size - * Entry size to allocate if cb_create not specified. - * @param flags - * The hash list attribute flags. - * @param cb_create - * Callback function for entry create. - * @param cb_match - * Callback function for entry match. - * @param cb_destroy - * Callback function for entry destroy. - * @return - * Pointer of the hash list table created, NULL on failure. - */ -struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size, - uint32_t entry_size, uint32_t flags, - mlx5_hlist_create_cb cb_create, - mlx5_hlist_match_cb cb_match, - mlx5_hlist_remove_cb cb_destroy); - -/** - * Search an entry matching the key. - * - * Result returned might be destroyed by other thread, must use - * this function only in main thread. - * - * @param h - * Pointer to the hast list table. - * @param key - * Key for the searching entry. - * @param ctx - * Common context parameter used by entry callback function. - * - * @return - * Pointer of the hlist entry if found, NULL otherwise. - */ -struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, - void *ctx); - -/** - * Insert an entry to the hash list table, the entry is only part of whole data - * element and a 64B key is used for matching. User should construct the key or - * give a calculated hash signature and guarantee there is no collision. - * - * @param h - * Pointer to the hast list table. - * @param entry - * Entry to be inserted into the hash list table. - * @param ctx - * Common context parameter used by callback function. - * - * @return - * registered entry on success, NULL otherwise - */ -struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, - void *ctx); - -/** - * Remove an entry from the hash list table. User should guarantee the validity - * of the entry. - * - * @param h - * Pointer to the hast list table. (not used) - * @param entry - * Entry to be removed from the hash list table. - * @return - * 0 on entry removed, 1 on entry still referenced. - */ -int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry); - -/** - * Destroy the hash list table, all the entries already inserted into the lists - * will be handled by the callback function provided by the user (including - * free if needed) before the table is freed. - * - * @param h - * Pointer to the hast list table. - */ -void mlx5_hlist_destroy(struct mlx5_hlist *h); - /************************ cache list *****************************/ /** Maximum size of string for naming. */