4 * Copyright(c) 2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include <rte_common.h>
38 #include <rte_memory.h>
39 #include <rte_malloc.h>
43 #include "rte_table_hash.h"
45 #ifdef RTE_TABLE_STATS_COLLECT
47 #define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(table, val) \
48 (table->stats.n_pkts_in += val)
49 #define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(table, val) \
50 (table->stats.n_pkts_lookup_miss += val)
54 #define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(table, val)
55 #define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(table, val)
60 struct rte_table_hash {
61 struct rte_table_stats stats;
63 /* Input parameters */
67 rte_table_hash_op_hash f_hash;
69 uint32_t signature_offset;
73 /* cuckoo hash table object */
74 struct rte_hash *h_table;
77 uint8_t memory[0] __rte_cache_aligned; };
80 check_params_create_hash_cuckoo(const struct
81 rte_table_hash_cuckoo_params *params) {
82 /* Check for valid parameters */
84 RTE_LOG(ERR, TABLE, "NULL Input Parameters.\n");
88 if (params->key_size == 0) {
89 RTE_LOG(ERR, TABLE, "Invalid key_size.\n");
93 if (params->n_keys == 0) {
94 RTE_LOG(ERR, TABLE, "Invalid n_keys.\n");
98 if (params->f_hash == NULL) {
99 RTE_LOG(ERR, TABLE, "f_hash is NULL.\n");
103 if (params->name == NULL) {
104 RTE_LOG(ERR, TABLE, "Table name is NULL.\n");
112 rte_table_hash_cuckoo_create(void *params,
116 struct rte_hash *rte_hash_handle;
117 struct rte_table_hash *t;
118 uint32_t total_size, total_cl_size;
120 /* Check input parameters */
121 struct rte_table_hash_cuckoo_params *p =
122 (struct rte_table_hash_cuckoo_params *) params;
124 if (check_params_create_hash_cuckoo(params))
127 /* Memory allocation */
129 (sizeof(struct rte_table_hash) +
130 RTE_CACHE_LINE_SIZE) / RTE_CACHE_LINE_SIZE;
131 total_cl_size += (p->n_keys * entry_size +
132 RTE_CACHE_LINE_SIZE) / RTE_CACHE_LINE_SIZE;
133 total_size = total_cl_size * RTE_CACHE_LINE_SIZE;
135 t = rte_zmalloc_socket("TABLE",
141 "%s: Cannot allocate %u bytes for Cuckoo hash table\n",
143 (uint32_t)sizeof(struct rte_table_hash));
147 /* Create cuckoo hash table */
148 struct rte_hash_parameters hash_cuckoo_params = {
149 .entries = p->n_keys,
150 .key_len = p->key_size,
151 .hash_func = (rte_hash_function)(p->f_hash),
152 .hash_func_init_val = p->seed,
153 .socket_id = socket_id,
157 rte_hash_handle = rte_hash_find_existing(p->name);
158 if (rte_hash_handle == NULL) {
159 rte_hash_handle = rte_hash_create(&hash_cuckoo_params);
160 if (NULL == rte_hash_handle) {
162 "%s: failed to create cuckoo hash table. keysize: %u",
163 __func__, hash_cuckoo_params.key_len);
169 /* initialize the cuckoo hash parameters */
170 t->key_size = p->key_size;
171 t->entry_size = entry_size;
172 t->n_keys = p->n_keys;
173 t->f_hash = p->f_hash;
175 t->signature_offset = p->signature_offset;
176 t->key_offset = p->key_offset;
178 t->h_table = rte_hash_handle;
181 "%s: Cuckoo Hash table memory footprint is %u bytes\n",
182 __func__, total_size);
187 rte_table_hash_cuckoo_free(void *table) {
189 RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
193 struct rte_table_hash *t = (struct rte_table_hash *)table;
195 rte_hash_free(t->h_table);
202 rte_table_hash_cuckoo_entry_add(void *table, void *key, void *entry,
203 int *key_found, void **entry_ptr) {
207 RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
212 RTE_LOG(ERR, TABLE, "%s: key parameter is NULL\n", __func__);
217 RTE_LOG(ERR, TABLE, "%s: entry parameter is NULL\n", __func__);
221 struct rte_table_hash *t = (struct rte_table_hash *)table;
223 /* Find Existing entries */
224 pos = rte_hash_lookup(t->h_table, key);
226 uint8_t *existing_entry;
229 existing_entry = &t->memory[pos * t->entry_size];
230 memcpy(existing_entry, entry, t->entry_size);
231 *entry_ptr = existing_entry;
234 } else if (pos == -ENOENT) {
235 /* Entry not found. Adding new entry */
238 pos = rte_hash_add_key(t->h_table, key);
241 "%s: Entry not added, status : %u\n",
246 new_entry = &t->memory[pos * t->entry_size];
247 memcpy(new_entry, entry, t->entry_size);
250 *entry_ptr = new_entry;
257 rte_table_hash_cuckoo_entry_delete(void *table, void *key,
258 int *key_found, __rte_unused void *entry) {
262 RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
267 RTE_LOG(ERR, TABLE, "%s: key parameter is NULL\n", __func__);
271 struct rte_table_hash *t = (struct rte_table_hash *)table;
273 pos = rte_hash_del_key(t->h_table, key);
276 uint8_t *entry_ptr = &t->memory[pos * t->entry_size];
279 memcpy(entry, entry_ptr, t->entry_size);
281 memset(&t->memory[pos * t->entry_size], 0, t->entry_size);
289 rte_table_hash_cuckoo_lookup_dosig(void *table,
290 struct rte_mbuf **pkts,
292 uint64_t *lookup_hit_mask,
295 struct rte_table_hash *t = (struct rte_table_hash *)table;
296 uint64_t pkts_mask_out = 0;
299 __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
301 RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(t, n_pkts_in);
303 if ((pkts_mask & (pkts_mask + 1)) == 0) {
304 const uint8_t *keys[64];
305 int32_t positions[64], status;
307 /* Keys for bulk lookup */
308 for (i = 0; i < n_pkts_in; i++)
309 keys[i] = RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
313 status = rte_hash_lookup_bulk(t->h_table,
314 (const void **) keys,
319 for (i = 0; i < n_pkts_in; i++) {
320 if (likely(positions[i] >= 0)) {
321 uint64_t pkt_mask = 1LLU << i;
323 entries[i] = &t->memory[positions[i]
325 pkts_mask_out |= pkt_mask;
330 for (i = 0; i < (uint32_t)(RTE_PORT_IN_BURST_SIZE_MAX
331 - __builtin_clzll(pkts_mask)); i++) {
332 uint64_t pkt_mask = 1LLU << i;
334 if (pkt_mask & pkts_mask) {
335 struct rte_mbuf *pkt = pkts[i];
336 uint8_t *key = RTE_MBUF_METADATA_UINT8_PTR(pkt,
340 pos = rte_hash_lookup(t->h_table, key);
341 if (likely(pos >= 0)) {
342 entries[i] = &t->memory[pos
344 pkts_mask_out |= pkt_mask;
350 *lookup_hit_mask = pkts_mask_out;
351 RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(t,
352 n_pkts_in - __builtin_popcountll(pkts_mask_out));
359 rte_table_hash_cuckoo_stats_read(void *table, struct rte_table_stats *stats,
362 struct rte_table_hash *t = (struct rte_table_hash *) table;
365 memcpy(stats, &t->stats, sizeof(t->stats));
368 memset(&t->stats, 0, sizeof(t->stats));
373 struct rte_table_ops rte_table_hash_cuckoo_dosig_ops = {
374 .f_create = rte_table_hash_cuckoo_create,
375 .f_free = rte_table_hash_cuckoo_free,
376 .f_add = rte_table_hash_cuckoo_entry_add,
377 .f_delete = rte_table_hash_cuckoo_entry_delete,
379 .f_delete_bulk = NULL,
380 .f_lookup = rte_table_hash_cuckoo_lookup_dosig,
381 .f_stats = rte_table_hash_cuckoo_stats_read,