From b5cde2cb8c81eb512555c640cd3d36f6ac89559f Mon Sep 17 00:00:00 2001 From: Cristian Dumitrescu Date: Wed, 18 Oct 2017 16:03:45 +0100 Subject: [PATCH] table: rework variable size key lru hash table Rework for the variable size key LRU hash table to use the mask-based hash function and the unified parameter structure. Signed-off-by: Cristian Dumitrescu --- lib/librte_table/rte_table_hash.h | 26 --- lib/librte_table/rte_table_hash_lru.c | 277 ++++++++++++++++---------- test/test-pipeline/pipeline_hash.c | 9 +- 3 files changed, 177 insertions(+), 135 deletions(-) diff --git a/lib/librte_table/rte_table_hash.h b/lib/librte_table/rte_table_hash.h index 0024b99933..fb5f4d7d21 100644 --- a/lib/librte_table/rte_table_hash.h +++ b/lib/librte_table/rte_table_hash.h @@ -137,32 +137,6 @@ typedef uint64_t (*rte_table_hash_op_hash_nomask)( extern struct rte_table_ops rte_table_hash_ext_ops; -/** LRU hash table parameters */ -struct rte_table_hash_lru_params { - /** Key size (number of bytes) */ - uint32_t key_size; - - /** Maximum number of keys */ - uint32_t n_keys; - - /** Number of hash table buckets. Each bucket stores up to 4 keys. */ - uint32_t n_buckets; - - /** Hash function */ - rte_table_hash_op_hash_nomask f_hash; - - /** Seed value for the hash function */ - uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - - /** Byte offset within packet meta-data where the key is located */ - uint32_t key_offset; -}; - extern struct rte_table_ops rte_table_hash_lru_ops; /** diff --git a/lib/librte_table/rte_table_hash_lru.c b/lib/librte_table/rte_table_hash_lru.c index 61050f3b37..eebc1cd43a 100644 --- a/lib/librte_table/rte_table_hash_lru.c +++ b/lib/librte_table/rte_table_hash_lru.c @@ -84,9 +84,8 @@ struct rte_table_hash { uint32_t entry_size; uint32_t n_keys; uint32_t n_buckets; - rte_table_hash_op_hash_nomask f_hash; + rte_table_hash_op_hash f_hash; uint64_t seed; - uint32_t signature_offset; uint32_t key_offset; /* Internal */ @@ -99,6 +98,7 @@ struct rte_table_hash { struct grinder grinders[RTE_PORT_IN_BURST_SIZE_MAX]; /* Tables */ + uint64_t *key_mask; struct bucket *buckets; uint8_t *key_mem; uint8_t *data_mem; @@ -109,12 +109,39 @@ struct rte_table_hash { }; static int -check_params_create(struct rte_table_hash_lru_params *params) +keycmp(void *a, void *b, void *b_mask, uint32_t n_bytes) { - uint32_t n_buckets_min; + uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask; + uint32_t i; + + for (i = 0; i < n_bytes / sizeof(uint64_t); i++) + if (a64[i] != (b64[i] & b_mask64[i])) + return 1; + + return 0; +} + +static void +keycpy(void *dst, void *src, void *src_mask, uint32_t n_bytes) +{ + uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask; + uint32_t i; + + for (i = 0; i < n_bytes / sizeof(uint64_t); i++) + dst64[i] = src64[i] & src_mask64[i]; +} + +static int +check_params_create(struct rte_table_hash_params *params) +{ + /* name */ + if (params->name == NULL) { + RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__); + return -EINVAL; + } /* key_size */ - if ((params->key_size == 0) || + if ((params->key_size < sizeof(uint64_t)) || (!rte_is_power_of_2(params->key_size))) { RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__); return -EINVAL; @@ -128,10 +155,8 @@ check_params_create(struct rte_table_hash_lru_params *params) } /* n_buckets */ - n_buckets_min = (params->n_keys + KEYS_PER_BUCKET - 1) / params->n_keys; if ((params->n_buckets == 0) || - (!rte_is_power_of_2(params->n_keys)) || - (params->n_buckets < n_buckets_min)) { + (!rte_is_power_of_2(params->n_keys))) { RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__); return -EINVAL; } @@ -148,13 +173,13 @@ check_params_create(struct rte_table_hash_lru_params *params) static void * rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_lru_params *p = - params; + struct rte_table_hash_params *p = params; struct rte_table_hash *t; - uint32_t total_size, table_meta_sz; - uint32_t bucket_sz, key_sz, key_stack_sz, data_sz; - uint32_t bucket_offset, key_offset, key_stack_offset, data_offset; - uint32_t i; + uint64_t table_meta_sz, key_mask_sz, bucket_sz, key_sz, key_stack_sz; + uint64_t data_sz, total_size; + uint64_t key_mask_offset, bucket_offset, key_offset, key_stack_offset; + uint64_t data_offset; + uint32_t n_buckets, i; /* Check input parameters */ if ((check_params_create(p) != 0) || @@ -164,33 +189,65 @@ rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size) return NULL; } + /* + * Table dimensioning + * + * Objective: Pick the number of buckets (n_buckets) so that there a chance + * to store n_keys keys in the table. + * + * Note: Since the buckets do not get extended, it is not possible to + * guarantee that n_keys keys can be stored in the table at any time. In the + * worst case scenario when all the n_keys fall into the same bucket, only + * a maximum of KEYS_PER_BUCKET keys will be stored in the table. This case + * defeats the purpose of the hash table. It indicates unsuitable f_hash or + * n_keys to n_buckets ratio. + * + * MIN(n_buckets) = (n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET + */ + n_buckets = rte_align32pow2( + (p->n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET); + n_buckets = RTE_MAX(n_buckets, p->n_buckets); + /* Memory allocation */ table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash)); - bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket)); + key_mask_sz = RTE_CACHE_LINE_ROUNDUP(p->key_size); + bucket_sz = RTE_CACHE_LINE_ROUNDUP(n_buckets * sizeof(struct bucket)); key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size); key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t)); data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size); - total_size = table_meta_sz + bucket_sz + key_sz + key_stack_sz + - data_sz; + total_size = table_meta_sz + key_mask_sz + bucket_sz + key_sz + + key_stack_sz + data_sz; + + if (total_size > SIZE_MAX) { + RTE_LOG(ERR, TABLE, + "%s: Cannot allocate %" PRIu64 " bytes for hash " + "table %s\n", + __func__, total_size, p->name); + return NULL; + } - t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); + t = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); if (t == NULL) { RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + "%s: Cannot allocate %" PRIu64 " bytes for hash " + "table %s\n", + __func__, total_size, p->name); return NULL; } - RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table memory footprint is " - "%u bytes\n", __func__, p->key_size, total_size); + RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table %s memory footprint" + " is %" PRIu64 " bytes\n", + __func__, p->key_size, p->name, total_size); /* Memory initialization */ t->key_size = p->key_size; t->entry_size = entry_size; t->n_keys = p->n_keys; - t->n_buckets = p->n_buckets; + t->n_buckets = n_buckets; t->f_hash = p->f_hash; t->seed = p->seed; - t->signature_offset = p->signature_offset; t->key_offset = p->key_offset; /* Internal */ @@ -199,16 +256,24 @@ rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size) t->data_size_shl = __builtin_ctzl(entry_size); /* Tables */ - bucket_offset = 0; + key_mask_offset = 0; + bucket_offset = key_mask_offset + key_mask_sz; key_offset = bucket_offset + bucket_sz; key_stack_offset = key_offset + key_sz; data_offset = key_stack_offset + key_stack_sz; + t->key_mask = (uint64_t *) &t->memory[key_mask_offset]; t->buckets = (struct bucket *) &t->memory[bucket_offset]; t->key_mem = &t->memory[key_offset]; t->key_stack = (uint32_t *) &t->memory[key_stack_offset]; t->data_mem = &t->memory[data_offset]; + /* Key mask */ + if (p->key_mask == NULL) + memset(t->key_mask, 0xFF, p->key_size); + else + memcpy(t->key_mask, p->key_mask, p->key_size); + /* Key stack */ for (i = 0; i < t->n_keys; i++) t->key_stack[i] = t->n_keys - 1 - i; @@ -246,7 +311,7 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry, uint64_t sig; uint32_t bkt_index, i; - sig = t->f_hash(key, t->key_size, t->seed); + sig = t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt = &t->buckets[bkt_index]; sig = (sig >> 16) | 1LLU; @@ -258,8 +323,8 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry, uint8_t *bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl]; - if ((sig == bkt_sig) && (memcmp(key, bkt_key, t->key_size) - == 0)) { + if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask, + t->key_size) == 0)) { uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; @@ -292,7 +357,7 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry, bkt->sig[i] = (uint16_t) sig; bkt->key_pos[i] = bkt_key_index; - memcpy(bkt_key, key, t->key_size); + keycpy(bkt_key, key, t->key_mask, t->key_size); memcpy(data, entry, t->entry_size); lru_update(bkt, i); @@ -311,7 +376,7 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry, uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; bkt->sig[pos] = (uint16_t) sig; - memcpy(bkt_key, key, t->key_size); + keycpy(bkt_key, key, t->key_mask, t->key_size); memcpy(data, entry, t->entry_size); lru_update(bkt, pos); @@ -330,7 +395,7 @@ rte_table_hash_lru_entry_delete(void *table, void *key, int *key_found, uint64_t sig; uint32_t bkt_index, i; - sig = t->f_hash(key, t->key_size, t->seed); + sig = t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt = &t->buckets[bkt_index]; sig = (sig >> 16) | 1LLU; @@ -343,14 +408,15 @@ rte_table_hash_lru_entry_delete(void *table, void *key, int *key_found, t->key_size_shl]; if ((sig == bkt_sig) && - (memcmp(key, bkt_key, t->key_size) == 0)) { + (keycmp(bkt_key, key, t->key_mask, t->key_size) == 0)) { uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; bkt->sig[i] = 0; t->key_stack[t->key_stack_tos++] = bkt_key_index; *key_found = 1; - memcpy(entry, data, t->entry_size); + if (entry) + memcpy(entry, data, t->entry_size); return 0; } } @@ -386,7 +452,7 @@ static int rte_table_hash_lru_lookup_unoptimized( pkt = pkts[pkt_index]; key = RTE_MBUF_METADATA_UINT8_PTR(pkt, t->key_offset); - sig = (uint64_t) t->f_hash(key, t->key_size, t->seed); + sig = (uint64_t) t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt = &t->buckets[bkt_index]; @@ -399,7 +465,7 @@ static int rte_table_hash_lru_lookup_unoptimized( uint8_t *bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl]; - if ((sig == bkt_sig) && (memcmp(key, bkt_key, + if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask, t->key_size) == 0)) { uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; @@ -497,74 +563,75 @@ static int rte_table_hash_lru_lookup_unoptimized( match_pos = (LUT_MATCH_POS >> (mask_all << 1)) & 3; \ } -#define lookup_cmp_key(mbuf, key, match_key, f) \ -{ \ +#define lookup_cmp_key(mbuf, key, match_key, f) \ +{ \ uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(mbuf, f->key_offset);\ - uint64_t *bkt_key = (uint64_t *) key; \ - \ - switch (f->key_size) { \ - case 8: \ - { \ - uint64_t xor = pkt_key[0] ^ bkt_key[0]; \ - match_key = 0; \ - if (xor == 0) \ - match_key = 1; \ - } \ - break; \ - \ - case 16: \ - { \ - uint64_t xor[2], or; \ - \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ - or = xor[0] | xor[1]; \ - match_key = 0; \ - if (or == 0) \ - match_key = 1; \ - } \ - break; \ - \ - case 32: \ - { \ - uint64_t xor[4], or; \ - \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ - xor[2] = pkt_key[2] ^ bkt_key[2]; \ - xor[3] = pkt_key[3] ^ bkt_key[3]; \ - or = xor[0] | xor[1] | xor[2] | xor[3]; \ - match_key = 0; \ - if (or == 0) \ - match_key = 1; \ - } \ - break; \ - \ - case 64: \ - { \ - uint64_t xor[8], or; \ - \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ - xor[2] = pkt_key[2] ^ bkt_key[2]; \ - xor[3] = pkt_key[3] ^ bkt_key[3]; \ - xor[4] = pkt_key[4] ^ bkt_key[4]; \ - xor[5] = pkt_key[5] ^ bkt_key[5]; \ - xor[6] = pkt_key[6] ^ bkt_key[6]; \ - xor[7] = pkt_key[7] ^ bkt_key[7]; \ - or = xor[0] | xor[1] | xor[2] | xor[3] | \ - xor[4] | xor[5] | xor[6] | xor[7]; \ - match_key = 0; \ - if (or == 0) \ - match_key = 1; \ - } \ - break; \ - \ - default: \ - match_key = 0; \ - if (memcmp(pkt_key, bkt_key, f->key_size) == 0) \ - match_key = 1; \ - } \ + uint64_t *bkt_key = (uint64_t *) key; \ + uint64_t *key_mask = f->key_mask; \ + \ + switch (f->key_size) { \ + case 8: \ + { \ + uint64_t xor = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + match_key = 0; \ + if (xor == 0) \ + match_key = 1; \ + } \ + break; \ + \ + case 16: \ + { \ + uint64_t xor[2], or; \ + \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ + or = xor[0] | xor[1]; \ + match_key = 0; \ + if (or == 0) \ + match_key = 1; \ + } \ + break; \ + \ + case 32: \ + { \ + uint64_t xor[4], or; \ + \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ + xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2]; \ + xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3]; \ + or = xor[0] | xor[1] | xor[2] | xor[3]; \ + match_key = 0; \ + if (or == 0) \ + match_key = 1; \ + } \ + break; \ + \ + case 64: \ + { \ + uint64_t xor[8], or; \ + \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ + xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2]; \ + xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3]; \ + xor[4] = (pkt_key[4] & key_mask[4]) ^ bkt_key[4]; \ + xor[5] = (pkt_key[5] & key_mask[5]) ^ bkt_key[5]; \ + xor[6] = (pkt_key[6] & key_mask[6]) ^ bkt_key[6]; \ + xor[7] = (pkt_key[7] & key_mask[7]) ^ bkt_key[7]; \ + or = xor[0] | xor[1] | xor[2] | xor[3] | \ + xor[4] | xor[5] | xor[6] | xor[7]; \ + match_key = 0; \ + if (or == 0) \ + match_key = 1; \ + } \ + break; \ + \ + default: \ + match_key = 0; \ + if (keycmp(bkt_key, pkt_key, key_mask, f->key_size) == 0) \ + match_key = 1; \ + } \ } #define lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index)\ @@ -619,20 +686,20 @@ static int rte_table_hash_lru_lookup_unoptimized( struct bucket *bkt10, *bkt11, *buckets = t->buckets; \ uint8_t *key10, *key11; \ uint64_t bucket_mask = t->bucket_mask; \ - rte_table_hash_op_hash_nomask f_hash = t->f_hash; \ + rte_table_hash_op_hash f_hash = t->f_hash; \ uint64_t seed = t->seed; \ uint32_t key_size = t->key_size; \ uint32_t key_offset = t->key_offset; \ \ mbuf10 = pkts[pkt10_index]; \ key10 = RTE_MBUF_METADATA_UINT8_PTR(mbuf10, key_offset);\ - sig10 = (uint64_t) f_hash(key10, key_size, seed); \ + sig10 = (uint64_t) f_hash(key10, t->key_mask, key_size, seed);\ bkt10_index = sig10 & bucket_mask; \ bkt10 = &buckets[bkt10_index]; \ \ mbuf11 = pkts[pkt11_index]; \ key11 = RTE_MBUF_METADATA_UINT8_PTR(mbuf11, key_offset);\ - sig11 = (uint64_t) f_hash(key11, key_size, seed); \ + sig11 = (uint64_t) f_hash(key11, t->key_mask, key_size, seed);\ bkt11_index = sig11 & bucket_mask; \ bkt11 = &buckets[bkt11_index]; \ \ diff --git a/test/test-pipeline/pipeline_hash.c b/test/test-pipeline/pipeline_hash.c index 13d309d8d6..3c48640a60 100644 --- a/test/test-pipeline/pipeline_hash.c +++ b/test/test-pipeline/pipeline_hash.c @@ -204,14 +204,15 @@ app_main_loop_worker_pipeline_hash(void) { case e_APP_PIPELINE_HASH_KEY16_LRU: case e_APP_PIPELINE_HASH_KEY32_LRU: { - struct rte_table_hash_lru_params table_hash_params = { + struct rte_table_hash_params table_hash_params = { + .name = "TABLE", .key_size = key_size, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, .n_keys = 1 << 24, .n_buckets = 1 << 22, - .f_hash = test_hash, + .f_hash = (rte_table_hash_op_hash)test_hash, .seed = 0, - .signature_offset = APP_METADATA_OFFSET(0), - .key_offset = APP_METADATA_OFFSET(32), }; struct rte_pipeline_table_params table_params = { -- 2.20.1