1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
7 #include <rte_common.h>
9 #include <rte_memory.h>
10 #include <rte_malloc.h>
14 #include "rte_table_hash.h"
16 #ifdef RTE_TABLE_STATS_COLLECT
18 #define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(table, val) \
19 (table->stats.n_pkts_in += val)
20 #define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(table, val) \
21 (table->stats.n_pkts_lookup_miss += val)
25 #define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(table, val)
26 #define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(table, val)
31 struct rte_table_hash {
32 struct rte_table_stats stats;
34 /* Input parameters */
38 rte_table_hash_op_hash f_hash;
42 /* cuckoo hash table object */
43 struct rte_hash *h_table;
46 uint8_t memory[0] __rte_cache_aligned;
50 check_params_create_hash_cuckoo(struct rte_table_hash_params *params)
53 RTE_LOG(ERR, TABLE, "NULL Input Parameters.\n");
57 if (params->name == NULL) {
58 RTE_LOG(ERR, TABLE, "Table name is NULL.\n");
62 if (params->key_size == 0) {
63 RTE_LOG(ERR, TABLE, "Invalid key_size.\n");
67 if (params->n_keys == 0) {
68 RTE_LOG(ERR, TABLE, "Invalid n_keys.\n");
72 if (params->f_hash == NULL) {
73 RTE_LOG(ERR, TABLE, "f_hash is NULL.\n");
81 rte_table_hash_cuckoo_create(void *params,
85 struct rte_table_hash_params *p = params;
86 struct rte_hash *h_table;
87 struct rte_table_hash *t;
90 /* Check input parameters */
91 if (check_params_create_hash_cuckoo(params))
94 /* Memory allocation */
95 total_size = sizeof(struct rte_table_hash) +
96 RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
98 t = rte_zmalloc_socket(p->name, total_size, RTE_CACHE_LINE_SIZE, socket_id);
101 "%s: Cannot allocate %u bytes for cuckoo hash table %s\n",
102 __func__, total_size, p->name);
106 /* Create cuckoo hash table */
107 struct rte_hash_parameters hash_cuckoo_params = {
108 .entries = p->n_keys,
109 .key_len = p->key_size,
110 .hash_func = (rte_hash_function)(p->f_hash),
111 .hash_func_init_val = p->seed,
112 .socket_id = socket_id,
116 h_table = rte_hash_find_existing(p->name);
117 if (h_table == NULL) {
118 h_table = rte_hash_create(&hash_cuckoo_params);
119 if (h_table == NULL) {
121 "%s: failed to create cuckoo hash table %s\n",
128 /* initialize the cuckoo hash parameters */
129 t->key_size = p->key_size;
130 t->entry_size = entry_size;
131 t->n_keys = p->n_keys;
132 t->f_hash = p->f_hash;
134 t->key_offset = p->key_offset;
135 t->h_table = h_table;
138 "%s: Cuckoo hash table %s memory footprint is %u bytes\n",
139 __func__, p->name, total_size);
144 rte_table_hash_cuckoo_free(void *table) {
145 struct rte_table_hash *t = table;
150 rte_hash_free(t->h_table);
157 rte_table_hash_cuckoo_entry_add(void *table, void *key, void *entry,
158 int *key_found, void **entry_ptr)
160 struct rte_table_hash *t = table;
163 /* Check input parameters */
164 if ((table == NULL) ||
167 (key_found == NULL) ||
171 /* Find Existing entries */
172 pos = rte_hash_lookup(t->h_table, key);
174 uint8_t *existing_entry;
177 existing_entry = &t->memory[pos * t->entry_size];
178 memcpy(existing_entry, entry, t->entry_size);
179 *entry_ptr = existing_entry;
184 if (pos == -ENOENT) {
185 /* Entry not found. Adding new entry */
188 pos = rte_hash_add_key(t->h_table, key);
192 new_entry = &t->memory[pos * t->entry_size];
193 memcpy(new_entry, entry, t->entry_size);
196 *entry_ptr = new_entry;
204 rte_table_hash_cuckoo_entry_delete(void *table, void *key,
205 int *key_found, void *entry)
207 struct rte_table_hash *t = table;
210 /* Check input parameters */
211 if ((table == NULL) ||
216 pos = rte_hash_del_key(t->h_table, key);
219 uint8_t *entry_ptr = &t->memory[pos * t->entry_size];
222 memcpy(entry, entry_ptr, t->entry_size);
224 memset(&t->memory[pos * t->entry_size], 0, t->entry_size);
233 rte_table_hash_cuckoo_lookup(void *table,
234 struct rte_mbuf **pkts,
236 uint64_t *lookup_hit_mask,
239 struct rte_table_hash *t = table;
240 uint64_t pkts_mask_out = 0;
243 __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
245 RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(t, n_pkts_in);
247 if ((pkts_mask & (pkts_mask + 1)) == 0) {
248 const uint8_t *keys[RTE_PORT_IN_BURST_SIZE_MAX];
249 int32_t positions[RTE_PORT_IN_BURST_SIZE_MAX], status;
251 /* Keys for bulk lookup */
252 for (i = 0; i < n_pkts_in; i++)
253 keys[i] = RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
257 status = rte_hash_lookup_bulk(t->h_table,
258 (const void **) keys,
262 for (i = 0; i < n_pkts_in; i++) {
263 if (likely(positions[i] >= 0)) {
264 uint64_t pkt_mask = 1LLU << i;
266 entries[i] = &t->memory[positions[i]
268 pkts_mask_out |= pkt_mask;
273 for (i = 0; i < (uint32_t)(RTE_PORT_IN_BURST_SIZE_MAX
274 - __builtin_clzll(pkts_mask)); i++) {
275 uint64_t pkt_mask = 1LLU << i;
277 if (pkt_mask & pkts_mask) {
278 struct rte_mbuf *pkt = pkts[i];
279 uint8_t *key = RTE_MBUF_METADATA_UINT8_PTR(pkt,
283 pos = rte_hash_lookup(t->h_table, key);
284 if (likely(pos >= 0)) {
285 entries[i] = &t->memory[pos
287 pkts_mask_out |= pkt_mask;
292 *lookup_hit_mask = pkts_mask_out;
293 RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(t,
294 n_pkts_in - __builtin_popcountll(pkts_mask_out));
301 rte_table_hash_cuckoo_stats_read(void *table, struct rte_table_stats *stats,
304 struct rte_table_hash *t = table;
307 memcpy(stats, &t->stats, sizeof(t->stats));
310 memset(&t->stats, 0, sizeof(t->stats));
315 struct rte_table_ops rte_table_hash_cuckoo_ops = {
316 .f_create = rte_table_hash_cuckoo_create,
317 .f_free = rte_table_hash_cuckoo_free,
318 .f_add = rte_table_hash_cuckoo_entry_add,
319 .f_delete = rte_table_hash_cuckoo_entry_delete,
321 .f_delete_bulk = NULL,
322 .f_lookup = rte_table_hash_cuckoo_lookup,
323 .f_stats = rte_table_hash_cuckoo_stats_read,