X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_table%2Frte_table_hash_ext.c;h=e7181026f4d58d48b115ba450a20d2ca109a242c;hb=e24b05bfd20d637a920c6dc1d7c4ee0aa107287e;hp=fb3e6d20033e2f94604c714abdb557264e42bc23;hpb=8595428e50e70d982e4a47784557d345913c4a56;p=dpdk.git diff --git a/lib/librte_table/rte_table_hash_ext.c b/lib/librte_table/rte_table_hash_ext.c index fb3e6d2003..e7181026f4 100644 --- a/lib/librte_table/rte_table_hash_ext.c +++ b/lib/librte_table/rte_table_hash_ext.c @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -73,6 +74,20 @@ do \ (bucket)->next = (bucket2)->next; \ while (0) +#ifdef RTE_TABLE_STATS_COLLECT + +#define RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(table, val) \ + table->stats.n_pkts_in += val +#define RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(table, val) \ + table->stats.n_pkts_lookup_miss += val + +#else + +#define RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(table, val) +#define RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(table, val) + +#endif + struct grinder { struct bucket *bkt; uint64_t sig; @@ -81,6 +96,8 @@ struct grinder { }; struct rte_table_hash { + struct rte_table_stats stats; + /* Input parameters */ uint32_t key_size; uint32_t entry_size; @@ -148,19 +165,6 @@ check_params_create(struct rte_table_hash_ext_params *params) return -EINVAL; } - /* signature offset */ - if ((params->signature_offset & 0x3) != 0) { - RTE_LOG(ERR, TABLE, "%s: signature_offset invalid value\n", - __func__); - return -EINVAL; - } - - /* key offset */ - if ((params->key_offset & 0x7) != 0) { - RTE_LOG(ERR, TABLE, "%s: key_offset invalid value\n", __func__); - return -EINVAL; - } - return 0; } @@ -168,9 +172,9 @@ static void * rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) { struct rte_table_hash_ext_params *p = - (struct rte_table_hash_ext_params *) params; + params; struct rte_table_hash *t; - uint32_t total_size, table_meta_sz, table_meta_offset; + uint32_t total_size, table_meta_sz; uint32_t bucket_sz, bucket_ext_sz, key_sz; uint32_t key_stack_sz, bkt_ext_stack_sz, data_sz; uint32_t bucket_offset, bucket_ext_offset, key_offset; @@ -180,24 +184,24 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) /* Check input parameters */ if ((check_params_create(p) != 0) || (!rte_is_power_of_2(entry_size)) || - ((sizeof(struct rte_table_hash) % CACHE_LINE_SIZE) != 0) || - (sizeof(struct bucket) != (CACHE_LINE_SIZE / 2))) + ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) || + (sizeof(struct bucket) != (RTE_CACHE_LINE_SIZE / 2))) return NULL; /* Memory allocation */ - table_meta_sz = CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash)); - bucket_sz = CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket)); + table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash)); + bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket)); bucket_ext_sz = - CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(struct bucket)); - key_sz = CACHE_LINE_ROUNDUP(p->n_keys * p->key_size); - key_stack_sz = CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t)); + RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(struct bucket)); + key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size); + key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t)); bkt_ext_stack_sz = - CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(uint32_t)); - data_sz = CACHE_LINE_ROUNDUP(p->n_keys * entry_size); + RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(uint32_t)); + data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size); total_size = table_meta_sz + bucket_sz + bucket_ext_sz + key_sz + key_stack_sz + bkt_ext_stack_sz + data_sz; - t = rte_zmalloc_socket("TABLE", total_size, CACHE_LINE_SIZE, socket_id); + t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); if (t == NULL) { RTE_LOG(ERR, TABLE, "%s: Cannot allocate %u bytes for hash table\n", @@ -224,8 +228,7 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) t->data_size_shl = __builtin_ctzl(entry_size); /* Tables */ - table_meta_offset = 0; - bucket_offset = table_meta_offset + table_meta_sz; + bucket_offset = 0; bucket_ext_offset = bucket_offset + bucket_sz; key_offset = bucket_ext_offset + bucket_ext_sz; key_stack_offset = key_offset + key_sz; @@ -255,7 +258,7 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) static int rte_table_hash_ext_free(void *table) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; /* Check input parameters */ if (t == NULL) @@ -269,7 +272,7 @@ static int rte_table_hash_ext_entry_add(void *table, void *key, void *entry, int *key_found, void **entry_ptr) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; struct bucket *bkt0, *bkt, *bkt_prev; uint64_t sig; uint32_t bkt_index, i; @@ -370,7 +373,7 @@ static int rte_table_hash_ext_entry_delete(void *table, void *key, int *key_found, void *entry) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; struct bucket *bkt0, *bkt, *bkt_prev; uint64_t sig; uint32_t bkt_index, i; @@ -440,6 +443,8 @@ static int rte_table_hash_ext_lookup_unoptimized( struct rte_table_hash *t = (struct rte_table_hash *) table; uint64_t pkts_mask_out = 0; + __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); + for ( ; pkts_mask; ) { struct bucket *bkt0, *bkt; struct rte_mbuf *pkt; @@ -641,6 +646,7 @@ static int rte_table_hash_ext_lookup_unoptimized( { \ uint64_t pkt00_mask, pkt01_mask; \ struct rte_mbuf *mbuf00, *mbuf01; \ + uint32_t key_offset = t->key_offset; \ \ pkt00_index = __builtin_ctzll(pkts_mask); \ pkt00_mask = 1LLU << pkt00_index; \ @@ -652,8 +658,8 @@ static int rte_table_hash_ext_lookup_unoptimized( pkts_mask &= ~pkt01_mask; \ mbuf01 = pkts[pkt01_index]; \ \ - rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0)); \ - rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0)); \ + rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\ + rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\ } #define lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask, pkt00_index, \ @@ -661,6 +667,7 @@ static int rte_table_hash_ext_lookup_unoptimized( { \ uint64_t pkt00_mask, pkt01_mask; \ struct rte_mbuf *mbuf00, *mbuf01; \ + uint32_t key_offset = t->key_offset; \ \ pkt00_index = __builtin_ctzll(pkts_mask); \ pkt00_mask = 1LLU << pkt00_index; \ @@ -674,8 +681,8 @@ static int rte_table_hash_ext_lookup_unoptimized( pkts_mask &= ~pkt01_mask; \ mbuf01 = pkts[pkt01_index]; \ \ - rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0)); \ - rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0)); \ + rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\ + rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\ } #define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index) \ @@ -861,10 +868,17 @@ static int rte_table_hash_ext_lookup( uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0; int status = 0; + __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); + RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(t, n_pkts_in); + /* Cannot run the pipeline with less than 7 packets */ - if (__builtin_popcountll(pkts_mask) < 7) - return rte_table_hash_ext_lookup_unoptimized(table, pkts, + if (__builtin_popcountll(pkts_mask) < 7) { + status = rte_table_hash_ext_lookup_unoptimized(table, pkts, pkts_mask, lookup_hit_mask, entries, 0); + RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - + __builtin_popcountll(*lookup_hit_mask)); + return status; + } /* Pipeline stage 0 */ lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); @@ -973,6 +987,7 @@ static int rte_table_hash_ext_lookup( } *lookup_hit_mask = pkts_mask_out; + RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out)); return status; } @@ -990,10 +1005,17 @@ static int rte_table_hash_ext_lookup_dosig( uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0; int status = 0; + __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); + RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(t, n_pkts_in); + /* Cannot run the pipeline with less than 7 packets */ - if (__builtin_popcountll(pkts_mask) < 7) - return rte_table_hash_ext_lookup_unoptimized(table, pkts, + if (__builtin_popcountll(pkts_mask) < 7) { + status = rte_table_hash_ext_lookup_unoptimized(table, pkts, pkts_mask, lookup_hit_mask, entries, 1); + RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - + __builtin_popcountll(*lookup_hit_mask)); + return status; + } /* Pipeline stage 0 */ lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); @@ -1102,15 +1124,33 @@ static int rte_table_hash_ext_lookup_dosig( } *lookup_hit_mask = pkts_mask_out; + RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out)); return status; } +static int +rte_table_hash_ext_stats_read(void *table, struct rte_table_stats *stats, int clear) +{ + struct rte_table_hash *t = table; + + if (stats != NULL) + memcpy(stats, &t->stats, sizeof(t->stats)); + + if (clear) + memset(&t->stats, 0, sizeof(t->stats)); + + return 0; +} + struct rte_table_ops rte_table_hash_ext_ops = { .f_create = rte_table_hash_ext_create, .f_free = rte_table_hash_ext_free, .f_add = rte_table_hash_ext_entry_add, .f_delete = rte_table_hash_ext_entry_delete, + .f_add_bulk = NULL, + .f_delete_bulk = NULL, .f_lookup = rte_table_hash_ext_lookup, + .f_stats = rte_table_hash_ext_stats_read, }; struct rte_table_ops rte_table_hash_ext_dosig_ops = { @@ -1118,5 +1158,8 @@ struct rte_table_ops rte_table_hash_ext_dosig_ops = { .f_free = rte_table_hash_ext_free, .f_add = rte_table_hash_ext_entry_add, .f_delete = rte_table_hash_ext_entry_delete, + .f_add_bulk = NULL, + .f_delete_bulk = NULL, .f_lookup = rte_table_hash_ext_lookup_dosig, + .f_stats = rte_table_hash_ext_stats_read, };