#include <rte_string_fns.h>
#include "mlx5_common.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
#include "mlx5_glue.h"
#ifdef MLX5_GLUE
#include <rte_errno.h>
#include "mlx5_nl.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
#include "mlx5_malloc.h"
#ifdef HAVE_DEVLINK
#include <linux/devlink.h>
'mlx5_malloc.c',
'mlx5_common_pci.c',
'mlx5_common_devx.c',
+ 'mlx5_common_utils.c',
)
cflags_options = [
#include "mlx5_common.h"
#include "mlx5_common_os.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
#include "mlx5_common_pci.h"
uint8_t haswell_broadwell_cpu;
#include "mlx5_prm.h"
#include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
#include "mlx5_malloc.h"
#include "mlx5_common.h"
#include "mlx5_common_devx.h"
--- /dev/null
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMMON_LOG_H_
+#define RTE_PMD_MLX5_COMMON_LOG_H_
+
+#include "mlx5_common.h"
+
+
+extern int mlx5_common_logtype;
+
+#define MLX5_COMMON_LOG_PREFIX "mlx5_common"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+ PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \
+ __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+ PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_COMMON_LOG_H_ */
+
#include <rte_errno.h>
#include "mlx5_common_mp.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
#include "mlx5_malloc.h"
/**
#include "mlx5_glue.h"
#include "mlx5_common_mp.h"
#include "mlx5_common_mr.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
#include "mlx5_malloc.h"
struct mr_find_contig_memsegs_data {
#include <stdlib.h>
#include <rte_malloc.h>
-#include "mlx5_common_utils.h"
+#include <rte_class.h>
+
+#include "mlx5_common_log.h"
#include "mlx5_common_pci.h"
struct mlx5_pci_device {
--- /dev/null
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_hash_crc.h>
+#include <rte_errno.h>
+
+#include <mlx5_malloc.h>
+
+#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
+
+/********************* Hash List **********************/
+
+static struct mlx5_hlist_entry *
+mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
+ void *ctx __rte_unused)
+{
+ return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
+}
+
+static void
+mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
+ struct mlx5_hlist_entry *entry)
+{
+ mlx5_free(entry);
+}
+
+struct mlx5_hlist *
+mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
+ uint32_t flags, mlx5_hlist_create_cb cb_create,
+ mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+{
+ struct mlx5_hlist *h;
+ uint32_t act_size;
+ uint32_t alloc_size;
+ uint32_t i;
+
+ if (!size || !cb_match || (!cb_create ^ !cb_remove))
+ return NULL;
+ /* Align to the next power of 2, 32bits integer is enough now. */
+ if (!rte_is_power_of_2(size)) {
+ act_size = rte_align32pow2(size);
+ DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
+ "will be aligned to 0x%" PRIX32 ".", size, act_size);
+ } else {
+ act_size = size;
+ }
+ alloc_size = sizeof(struct mlx5_hlist) +
+ sizeof(struct mlx5_hlist_bucket) * act_size;
+ /* Using zmalloc, then no need to initialize the heads. */
+ h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
+ SOCKET_ID_ANY);
+ if (!h) {
+ DRV_LOG(ERR, "No memory for hash list %s creation",
+ name ? name : "None");
+ return NULL;
+ }
+ if (name)
+ snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
+ h->table_sz = act_size;
+ h->mask = act_size - 1;
+ h->entry_sz = entry_size;
+ h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
+ h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
+ h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
+ h->cb_match = cb_match;
+ h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
+ for (i = 0; i < act_size; i++)
+ rte_rwlock_init(&h->buckets[i].lock);
+ DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
+ h->name, act_size);
+ return h;
+}
+
+static struct mlx5_hlist_entry *
+__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
+ void *ctx, bool reuse)
+{
+ struct mlx5_hlist_head *first;
+ struct mlx5_hlist_entry *node;
+
+ MLX5_ASSERT(h);
+ first = &h->buckets[idx].head;
+ LIST_FOREACH(node, first, next) {
+ if (!h->cb_match(h, node, key, ctx)) {
+ if (reuse) {
+ __atomic_add_fetch(&node->ref_cnt, 1,
+ __ATOMIC_RELAXED);
+ DRV_LOG(DEBUG, "Hash list %s entry %p "
+ "reuse: %u.",
+ h->name, (void *)node, node->ref_cnt);
+ }
+ break;
+ }
+ }
+ return node;
+}
+
+static struct mlx5_hlist_entry *
+hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
+ void *ctx, bool reuse)
+{
+ struct mlx5_hlist_entry *node;
+
+ MLX5_ASSERT(h);
+ rte_rwlock_read_lock(&h->buckets[idx].lock);
+ node = __hlist_lookup(h, key, idx, ctx, reuse);
+ rte_rwlock_read_unlock(&h->buckets[idx].lock);
+ return node;
+}
+
+struct mlx5_hlist_entry *
+mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
+{
+ uint32_t idx;
+
+ if (h->direct_key)
+ idx = (uint32_t)(key & h->mask);
+ else
+ idx = rte_hash_crc_8byte(key, 0) & h->mask;
+ return hlist_lookup(h, key, idx, ctx, false);
+}
+
+struct mlx5_hlist_entry*
+mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
+{
+ uint32_t idx;
+ struct mlx5_hlist_head *first;
+ struct mlx5_hlist_bucket *b;
+ struct mlx5_hlist_entry *entry;
+ uint32_t prev_gen_cnt = 0;
+
+ if (h->direct_key)
+ idx = (uint32_t)(key & h->mask);
+ else
+ idx = rte_hash_crc_8byte(key, 0) & h->mask;
+ MLX5_ASSERT(h);
+ b = &h->buckets[idx];
+ /* Use write lock directly for write-most list. */
+ if (!h->write_most) {
+ prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
+ entry = hlist_lookup(h, key, idx, ctx, true);
+ if (entry)
+ return entry;
+ }
+ rte_rwlock_write_lock(&b->lock);
+ /* Check if the list changed by other threads. */
+ if (h->write_most ||
+ prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
+ entry = __hlist_lookup(h, key, idx, ctx, true);
+ if (entry)
+ goto done;
+ }
+ first = &b->head;
+ entry = h->cb_create(h, key, ctx);
+ if (!entry) {
+ rte_errno = ENOMEM;
+ DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
+ goto done;
+ }
+ entry->idx = idx;
+ entry->ref_cnt = 1;
+ LIST_INSERT_HEAD(first, entry, next);
+ __atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
+ DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
+ h->name, (void *)entry, entry->ref_cnt);
+done:
+ rte_rwlock_write_unlock(&b->lock);
+ return entry;
+}
+
+int
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+{
+ uint32_t idx = entry->idx;
+
+ rte_rwlock_write_lock(&h->buckets[idx].lock);
+ MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
+ DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
+ h->name, (void *)entry, entry->ref_cnt);
+ if (--entry->ref_cnt) {
+ rte_rwlock_write_unlock(&h->buckets[idx].lock);
+ return 1;
+ }
+ LIST_REMOVE(entry, next);
+ /* Set to NULL to get rid of removing action for more than once. */
+ entry->next.le_prev = NULL;
+ h->cb_remove(h, entry);
+ rte_rwlock_write_unlock(&h->buckets[idx].lock);
+ DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
+ h->name, (void *)entry);
+ return 0;
+}
+
+void
+mlx5_hlist_destroy(struct mlx5_hlist *h)
+{
+ uint32_t idx;
+ struct mlx5_hlist_entry *entry;
+
+ MLX5_ASSERT(h);
+ for (idx = 0; idx < h->table_sz; ++idx) {
+ /* No LIST_FOREACH_SAFE, using while instead. */
+ while (!LIST_EMPTY(&h->buckets[idx].head)) {
+ entry = LIST_FIRST(&h->buckets[idx].head);
+ LIST_REMOVE(entry, next);
+ /*
+ * The owner of whole element which contains data entry
+ * is the user, so it's the user's duty to do the clean
+ * up and the free work because someone may not put the
+ * hlist entry at the beginning(suggested to locate at
+ * the beginning). Or else the default free function
+ * will be used.
+ */
+ h->cb_remove(h, entry);
+ }
+ }
+ mlx5_free(h);
+}
#include "mlx5_common.h"
+#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
+#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
-extern int mlx5_common_logtype;
+/** Maximum size of string for naming the hlist table. */
+#define MLX5_HLIST_NAMESIZE 32
-#define MLX5_COMMON_LOG_PREFIX "mlx5_common"
-/* Generic printf()-like logging macro with automatic line feed. */
-#define DRV_LOG(level, ...) \
- PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \
- __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
- PMD_DRV_LOG_CPAREN)
+struct mlx5_hlist;
+
+/**
+ * Structure of the entry in the hash list, user should define its own struct
+ * that contains this in order to store the data. The 'key' is 64-bits right
+ * now and its user's responsibility to guarantee there is no collision.
+ */
+struct mlx5_hlist_entry {
+ LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
+ uint32_t idx; /* Bucket index the entry belongs to. */
+ uint32_t ref_cnt; /* Reference count. */
+};
+
+/** Structure for hash head. */
+LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
+
+/**
+ * Type of callback function for entry removal.
+ *
+ * @param list
+ * The hash list.
+ * @param entry
+ * The entry in the list.
+ */
+typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
+ struct mlx5_hlist_entry *entry);
+
+/**
+ * Type of function for user defined matching.
+ *
+ * @param list
+ * The hash list.
+ * @param entry
+ * The entry in the list.
+ * @param key
+ * The new entry key.
+ * @param ctx
+ * The pointer to new entry context.
+ *
+ * @return
+ * 0 if matching, non-zero number otherwise.
+ */
+typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
+ struct mlx5_hlist_entry *entry,
+ uint64_t key, void *ctx);
+
+/**
+ * Type of function for user defined hash list entry creation.
+ *
+ * @param list
+ * The hash list.
+ * @param key
+ * The key of the new entry.
+ * @param ctx
+ * The pointer to new entry context.
+ *
+ * @return
+ * Pointer to allocated entry on success, NULL otherwise.
+ */
+typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
+ (struct mlx5_hlist *list,
+ uint64_t key, void *ctx);
+
+/* Hash list bucket head. */
+struct mlx5_hlist_bucket {
+ struct mlx5_hlist_head head; /* List head. */
+ rte_rwlock_t lock; /* Bucket lock. */
+ uint32_t gen_cnt; /* List modification will update generation count. */
+} __rte_cache_aligned;
+
+/**
+ * Hash list table structure
+ *
+ * Entry in hash list could be reused if entry already exists, reference
+ * count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ */
+struct mlx5_hlist {
+ char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
+ /**< number of heads, need to be power of 2. */
+ uint32_t table_sz;
+ uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
+ /**< mask to get the index of the list heads. */
+ uint32_t mask;
+ bool direct_key; /* Use the new entry key directly as hash index. */
+ bool write_most; /* List mostly used for append new or destroy. */
+ void *ctx;
+ mlx5_hlist_create_cb cb_create; /**< entry create callback. */
+ mlx5_hlist_match_cb cb_match; /**< entry match callback. */
+ mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+ struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
+ /**< list bucket arrays. */
+};
+
+/**
+ * Create a hash list table, the user can specify the list heads array size
+ * of the table, now the size should be a power of 2 in order to get better
+ * distribution for the entries. Each entry is a part of the whole data element
+ * and the caller should be responsible for the data element's allocation and
+ * cleanup / free. Key of each entry will be calculated with CRC in order to
+ * generate a little fairer distribution.
+ *
+ * @param name
+ * Name of the hash list(optional).
+ * @param size
+ * Heads array size of the hash list.
+ * @param entry_size
+ * Entry size to allocate if cb_create not specified.
+ * @param flags
+ * The hash list attribute flags.
+ * @param cb_create
+ * Callback function for entry create.
+ * @param cb_match
+ * Callback function for entry match.
+ * @param cb_destroy
+ * Callback function for entry destroy.
+ * @return
+ * Pointer of the hash list table created, NULL on failure.
+ */
+__rte_internal
+struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
+ uint32_t entry_size, uint32_t flags,
+ mlx5_hlist_create_cb cb_create,
+ mlx5_hlist_match_cb cb_match,
+ mlx5_hlist_remove_cb cb_destroy);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param h
+ * Pointer to the hast list table.
+ * @param key
+ * Key for the searching entry.
+ * @param ctx
+ * Common context parameter used by entry callback function.
+ *
+ * @return
+ * Pointer of the hlist entry if found, NULL otherwise.
+ */
+__rte_internal
+struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+ void *ctx);
+
+/**
+ * Insert an entry to the hash list table, the entry is only part of whole data
+ * element and a 64B key is used for matching. User should construct the key or
+ * give a calculated hash signature and guarantee there is no collision.
+ *
+ * @param h
+ * Pointer to the hast list table.
+ * @param entry
+ * Entry to be inserted into the hash list table.
+ * @param ctx
+ * Common context parameter used by callback function.
+ *
+ * @return
+ * registered entry on success, NULL otherwise
+ */
+__rte_internal
+struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+ void *ctx);
+
+/**
+ * Remove an entry from the hash list table. User should guarantee the validity
+ * of the entry.
+ *
+ * @param h
+ * Pointer to the hast list table. (not used)
+ * @param entry
+ * Entry to be removed from the hash list table.
+ * @return
+ * 0 on entry removed, 1 on entry still referenced.
+ */
+__rte_internal
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+
+/**
+ * Destroy the hash list table, all the entries already inserted into the lists
+ * will be handled by the callback function provided by the user (including
+ * free if needed) before the table is freed.
+ *
+ * @param h
+ * Pointer to the hast list table.
+ */
+__rte_internal
+void mlx5_hlist_destroy(struct mlx5_hlist *h);
#endif /* RTE_PMD_MLX5_COMMON_UTILS_H_ */
#include "mlx5_prm.h"
#include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
#include "mlx5_malloc.h"
#include <stdbool.h>
#include <string.h>
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
#include "mlx5_common_os.h"
#include "mlx5_malloc.h"
mlx5_glue;
+ mlx5_hlist_create;
+ mlx5_hlist_lookup;
+ mlx5_hlist_register;
+ mlx5_hlist_unregister;
+ mlx5_hlist_destroy;
+
mlx5_malloc;
mlx5_malloc_mem_select;
#include <rte_errno.h>
#include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
#include "mlx5_common.h"
#include "mlx5_common_os.h"
#include "mlx5_malloc.h"
#include <rte_malloc.h>
#include "mlx5_glue.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
#include "mlx5_win_ext.h"
/*
*/
#include <rte_malloc.h>
-#include <rte_hash_crc.h>
#include <mlx5_malloc.h>
#include "mlx5_utils.h"
-/********************* Hash List **********************/
-
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
- void *ctx __rte_unused)
-{
- return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
- struct mlx5_hlist_entry *entry)
-{
- mlx5_free(entry);
-}
-
-struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
- uint32_t flags, mlx5_hlist_create_cb cb_create,
- mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
-{
- struct mlx5_hlist *h;
- uint32_t act_size;
- uint32_t alloc_size;
- uint32_t i;
-
- if (!size || !cb_match || (!cb_create ^ !cb_remove))
- return NULL;
- /* Align to the next power of 2, 32bits integer is enough now. */
- if (!rte_is_power_of_2(size)) {
- act_size = rte_align32pow2(size);
- DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
- "will be aligned to 0x%" PRIX32 ".", size, act_size);
- } else {
- act_size = size;
- }
- alloc_size = sizeof(struct mlx5_hlist) +
- sizeof(struct mlx5_hlist_bucket) * act_size;
- /* Using zmalloc, then no need to initialize the heads. */
- h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
- SOCKET_ID_ANY);
- if (!h) {
- DRV_LOG(ERR, "No memory for hash list %s creation",
- name ? name : "None");
- return NULL;
- }
- if (name)
- snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
- h->table_sz = act_size;
- h->mask = act_size - 1;
- h->entry_sz = entry_size;
- h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
- h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
- h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
- h->cb_match = cb_match;
- h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
- for (i = 0; i < act_size; i++)
- rte_rwlock_init(&h->buckets[i].lock);
- DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
- h->name, act_size);
- return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
- void *ctx, bool reuse)
-{
- struct mlx5_hlist_head *first;
- struct mlx5_hlist_entry *node;
-
- MLX5_ASSERT(h);
- first = &h->buckets[idx].head;
- LIST_FOREACH(node, first, next) {
- if (!h->cb_match(h, node, key, ctx)) {
- if (reuse) {
- __atomic_add_fetch(&node->ref_cnt, 1,
- __ATOMIC_RELAXED);
- DRV_LOG(DEBUG, "Hash list %s entry %p "
- "reuse: %u.",
- h->name, (void *)node, node->ref_cnt);
- }
- break;
- }
- }
- return node;
-}
-
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
- void *ctx, bool reuse)
-{
- struct mlx5_hlist_entry *node;
-
- MLX5_ASSERT(h);
- rte_rwlock_read_lock(&h->buckets[idx].lock);
- node = __hlist_lookup(h, key, idx, ctx, reuse);
- rte_rwlock_read_unlock(&h->buckets[idx].lock);
- return node;
-}
-
-struct mlx5_hlist_entry *
-mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
-{
- uint32_t idx;
-
- if (h->direct_key)
- idx = (uint32_t)(key & h->mask);
- else
- idx = rte_hash_crc_8byte(key, 0) & h->mask;
- return hlist_lookup(h, key, idx, ctx, false);
-}
-
-struct mlx5_hlist_entry*
-mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
-{
- uint32_t idx;
- struct mlx5_hlist_head *first;
- struct mlx5_hlist_bucket *b;
- struct mlx5_hlist_entry *entry;
- uint32_t prev_gen_cnt = 0;
-
- if (h->direct_key)
- idx = (uint32_t)(key & h->mask);
- else
- idx = rte_hash_crc_8byte(key, 0) & h->mask;
- MLX5_ASSERT(h);
- b = &h->buckets[idx];
- /* Use write lock directly for write-most list. */
- if (!h->write_most) {
- prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
- entry = hlist_lookup(h, key, idx, ctx, true);
- if (entry)
- return entry;
- }
- rte_rwlock_write_lock(&b->lock);
- /* Check if the list changed by other threads. */
- if (h->write_most ||
- prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
- entry = __hlist_lookup(h, key, idx, ctx, true);
- if (entry)
- goto done;
- }
- first = &b->head;
- entry = h->cb_create(h, key, ctx);
- if (!entry) {
- rte_errno = ENOMEM;
- DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
- goto done;
- }
- entry->idx = idx;
- entry->ref_cnt = 1;
- LIST_INSERT_HEAD(first, entry, next);
- __atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
- DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
- h->name, (void *)entry, entry->ref_cnt);
-done:
- rte_rwlock_write_unlock(&b->lock);
- return entry;
-}
-
-int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
-{
- uint32_t idx = entry->idx;
-
- rte_rwlock_write_lock(&h->buckets[idx].lock);
- MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
- DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
- h->name, (void *)entry, entry->ref_cnt);
- if (--entry->ref_cnt) {
- rte_rwlock_write_unlock(&h->buckets[idx].lock);
- return 1;
- }
- LIST_REMOVE(entry, next);
- /* Set to NULL to get rid of removing action for more than once. */
- entry->next.le_prev = NULL;
- h->cb_remove(h, entry);
- rte_rwlock_write_unlock(&h->buckets[idx].lock);
- DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
- h->name, (void *)entry);
- return 0;
-}
-
-void
-mlx5_hlist_destroy(struct mlx5_hlist *h)
-{
- uint32_t idx;
- struct mlx5_hlist_entry *entry;
-
- MLX5_ASSERT(h);
- for (idx = 0; idx < h->table_sz; ++idx) {
- /* No LIST_FOREACH_SAFE, using while instead. */
- while (!LIST_EMPTY(&h->buckets[idx].head)) {
- entry = LIST_FIRST(&h->buckets[idx].head);
- LIST_REMOVE(entry, next);
- /*
- * The owner of whole element which contains data entry
- * is the user, so it's the user's duty to do the clean
- * up and the free work because someone may not put the
- * hlist entry at the beginning(suggested to locate at
- * the beginning). Or else the default free function
- * will be used.
- */
- h->cb_remove(h, entry);
- }
- }
- mlx5_free(h);
-}
/********************* Cache list ************************/
#include <rte_bitmap.h>
#include <mlx5_common.h>
+#include <mlx5_common_utils.h>
#include "mlx5_defs.h"
return l + r;
}
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
-
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE 32
-
-struct mlx5_hlist;
-
-/**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
- */
-struct mlx5_hlist_entry {
- LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
- uint32_t idx; /* Bucket index the entry belongs to. */
- uint32_t ref_cnt; /* Reference count. */
-};
-
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- * The hash list.
- * @param entry
- * The entry in the list.
- */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
- struct mlx5_hlist_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- * The hash list.
- * @param entry
- * The entry in the list.
- * @param key
- * The new entry key.
- * @param ctx
- * The pointer to new entry context.
- *
- * @return
- * 0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
- struct mlx5_hlist_entry *entry,
- uint64_t key, void *ctx);
-
-/**
- * Type of function for user defined hash list entry creation.
- *
- * @param list
- * The hash list.
- * @param key
- * The key of the new entry.
- * @param ctx
- * The pointer to new entry context.
- *
- * @return
- * Pointer to allocated entry on success, NULL otherwise.
- */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
- (struct mlx5_hlist *list,
- uint64_t key, void *ctx);
-
-/* Hash list bucket head. */
-struct mlx5_hlist_bucket {
- struct mlx5_hlist_head head; /* List head. */
- rte_rwlock_t lock; /* Bucket lock. */
- uint32_t gen_cnt; /* List modification will update generation count. */
-} __rte_cache_aligned;
-
-/**
- * Hash list table structure
- *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- */
-struct mlx5_hlist {
- char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
- /**< number of heads, need to be power of 2. */
- uint32_t table_sz;
- uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
- /**< mask to get the index of the list heads. */
- uint32_t mask;
- bool direct_key; /* Use the new entry key directly as hash index. */
- bool write_most; /* List mostly used for append new or destroy. */
- void *ctx;
- mlx5_hlist_create_cb cb_create; /**< entry create callback. */
- mlx5_hlist_match_cb cb_match; /**< entry match callback. */
- mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
- struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
- /**< list bucket arrays. */
-};
-
-/**
- * Create a hash list table, the user can specify the list heads array size
- * of the table, now the size should be a power of 2 in order to get better
- * distribution for the entries. Each entry is a part of the whole data element
- * and the caller should be responsible for the data element's allocation and
- * cleanup / free. Key of each entry will be calculated with CRC in order to
- * generate a little fairer distribution.
- *
- * @param name
- * Name of the hash list(optional).
- * @param size
- * Heads array size of the hash list.
- * @param entry_size
- * Entry size to allocate if cb_create not specified.
- * @param flags
- * The hash list attribute flags.
- * @param cb_create
- * Callback function for entry create.
- * @param cb_match
- * Callback function for entry match.
- * @param cb_destroy
- * Callback function for entry destroy.
- * @return
- * Pointer of the hash list table created, NULL on failure.
- */
-struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
- uint32_t entry_size, uint32_t flags,
- mlx5_hlist_create_cb cb_create,
- mlx5_hlist_match_cb cb_match,
- mlx5_hlist_remove_cb cb_destroy);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param h
- * Pointer to the hast list table.
- * @param key
- * Key for the searching entry.
- * @param ctx
- * Common context parameter used by entry callback function.
- *
- * @return
- * Pointer of the hlist entry if found, NULL otherwise.
- */
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
- void *ctx);
-
-/**
- * Insert an entry to the hash list table, the entry is only part of whole data
- * element and a 64B key is used for matching. User should construct the key or
- * give a calculated hash signature and guarantee there is no collision.
- *
- * @param h
- * Pointer to the hast list table.
- * @param entry
- * Entry to be inserted into the hash list table.
- * @param ctx
- * Common context parameter used by callback function.
- *
- * @return
- * registered entry on success, NULL otherwise
- */
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
- void *ctx);
-
-/**
- * Remove an entry from the hash list table. User should guarantee the validity
- * of the entry.
- *
- * @param h
- * Pointer to the hast list table. (not used)
- * @param entry
- * Entry to be removed from the hash list table.
- * @return
- * 0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
-
-/**
- * Destroy the hash list table, all the entries already inserted into the lists
- * will be handled by the callback function provided by the user (including
- * free if needed) before the table is freed.
- *
- * @param h
- * Pointer to the hast list table.
- */
-void mlx5_hlist_destroy(struct mlx5_hlist *h);
-
/************************ cache list *****************************/
/** Maximum size of string for naming. */
git.droids-corp.org / dpdk.git / commitdiff