#include <rte_common.h>
#include <rte_mbuf.h>
+#include <rte_memory.h>
#include <rte_malloc.h>
#include <rte_log.h>
(bucket)->next = (bucket2)->next; \
while (0)
+#ifdef RTE_TABLE_STATS_COLLECT
+
+#define RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(table, val) \
+ table->stats.n_pkts_in += val
+#define RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(table, val) \
+ table->stats.n_pkts_lookup_miss += val
+
+#else
+
+#define RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(table, val)
+#define RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(table, val)
+
+#endif
+
struct grinder {
struct bucket *bkt;
uint64_t sig;
};
struct rte_table_hash {
+ struct rte_table_stats stats;
+
/* Input parameters */
uint32_t key_size;
uint32_t entry_size;
return -EINVAL;
}
- /* signature offset */
- if ((params->signature_offset & 0x3) != 0) {
- RTE_LOG(ERR, TABLE, "%s: signature_offset invalid value\n",
- __func__);
- return -EINVAL;
- }
-
- /* key offset */
- if ((params->key_offset & 0x7) != 0) {
- RTE_LOG(ERR, TABLE, "%s: key_offset invalid value\n", __func__);
- return -EINVAL;
- }
-
return 0;
}
rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size)
{
struct rte_table_hash_ext_params *p =
- (struct rte_table_hash_ext_params *) params;
+ params;
struct rte_table_hash *t;
- uint32_t total_size, table_meta_sz, table_meta_offset;
+ uint32_t total_size, table_meta_sz;
uint32_t bucket_sz, bucket_ext_sz, key_sz;
uint32_t key_stack_sz, bkt_ext_stack_sz, data_sz;
uint32_t bucket_offset, bucket_ext_offset, key_offset;
/* Check input parameters */
if ((check_params_create(p) != 0) ||
(!rte_is_power_of_2(entry_size)) ||
- ((sizeof(struct rte_table_hash) % CACHE_LINE_SIZE) != 0) ||
- (sizeof(struct bucket) != (CACHE_LINE_SIZE / 2)))
+ ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
+ (sizeof(struct bucket) != (RTE_CACHE_LINE_SIZE / 2)))
return NULL;
/* Memory allocation */
- table_meta_sz = CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash));
- bucket_sz = CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket));
+ table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash));
+ bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket));
bucket_ext_sz =
- CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(struct bucket));
- key_sz = CACHE_LINE_ROUNDUP(p->n_keys * p->key_size);
- key_stack_sz = CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t));
+ RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(struct bucket));
+ key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size);
+ key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t));
bkt_ext_stack_sz =
- CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(uint32_t));
- data_sz = CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
+ RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(uint32_t));
+ data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
total_size = table_meta_sz + bucket_sz + bucket_ext_sz + key_sz +
key_stack_sz + bkt_ext_stack_sz + data_sz;
- t = rte_zmalloc_socket("TABLE", total_size, CACHE_LINE_SIZE, socket_id);
+ t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
if (t == NULL) {
RTE_LOG(ERR, TABLE,
"%s: Cannot allocate %u bytes for hash table\n",
t->data_size_shl = __builtin_ctzl(entry_size);
/* Tables */
- table_meta_offset = 0;
- bucket_offset = table_meta_offset + table_meta_sz;
+ bucket_offset = 0;
bucket_ext_offset = bucket_offset + bucket_sz;
key_offset = bucket_ext_offset + bucket_ext_sz;
key_stack_offset = key_offset + key_sz;
static int
rte_table_hash_ext_free(void *table)
{
- struct rte_table_hash *t = (struct rte_table_hash *) table;
+ struct rte_table_hash *t = table;
/* Check input parameters */
if (t == NULL)
rte_table_hash_ext_entry_add(void *table, void *key, void *entry,
int *key_found, void **entry_ptr)
{
- struct rte_table_hash *t = (struct rte_table_hash *) table;
+ struct rte_table_hash *t = table;
struct bucket *bkt0, *bkt, *bkt_prev;
uint64_t sig;
uint32_t bkt_index, i;
rte_table_hash_ext_entry_delete(void *table, void *key, int *key_found,
void *entry)
{
- struct rte_table_hash *t = (struct rte_table_hash *) table;
+ struct rte_table_hash *t = table;
struct bucket *bkt0, *bkt, *bkt_prev;
uint64_t sig;
uint32_t bkt_index, i;
struct rte_table_hash *t = (struct rte_table_hash *) table;
uint64_t pkts_mask_out = 0;
+ __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
+
for ( ; pkts_mask; ) {
struct bucket *bkt0, *bkt;
struct rte_mbuf *pkt;
{ \
uint64_t pkt00_mask, pkt01_mask; \
struct rte_mbuf *mbuf00, *mbuf01; \
+ uint32_t key_offset = t->key_offset; \
\
pkt00_index = __builtin_ctzll(pkts_mask); \
pkt00_mask = 1LLU << pkt00_index; \
pkts_mask &= ~pkt01_mask; \
mbuf01 = pkts[pkt01_index]; \
\
- rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0)); \
- rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0)); \
+ rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\
+ rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\
}
#define lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask, pkt00_index, \
{ \
uint64_t pkt00_mask, pkt01_mask; \
struct rte_mbuf *mbuf00, *mbuf01; \
+ uint32_t key_offset = t->key_offset; \
\
pkt00_index = __builtin_ctzll(pkts_mask); \
pkt00_mask = 1LLU << pkt00_index; \
pkts_mask &= ~pkt01_mask; \
mbuf01 = pkts[pkt01_index]; \
\
- rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0)); \
- rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0)); \
+ rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\
+ rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\
}
#define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index) \
uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0;
int status = 0;
+ __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
+ RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(t, n_pkts_in);
+
/* Cannot run the pipeline with less than 7 packets */
- if (__builtin_popcountll(pkts_mask) < 7)
- return rte_table_hash_ext_lookup_unoptimized(table, pkts,
+ if (__builtin_popcountll(pkts_mask) < 7) {
+ status = rte_table_hash_ext_lookup_unoptimized(table, pkts,
pkts_mask, lookup_hit_mask, entries, 0);
+ RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in -
+ __builtin_popcountll(*lookup_hit_mask));
+ return status;
+ }
/* Pipeline stage 0 */
lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
}
*lookup_hit_mask = pkts_mask_out;
+ RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));
return status;
}
uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0;
int status = 0;
+ __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
+ RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(t, n_pkts_in);
+
/* Cannot run the pipeline with less than 7 packets */
- if (__builtin_popcountll(pkts_mask) < 7)
- return rte_table_hash_ext_lookup_unoptimized(table, pkts,
+ if (__builtin_popcountll(pkts_mask) < 7) {
+ status = rte_table_hash_ext_lookup_unoptimized(table, pkts,
pkts_mask, lookup_hit_mask, entries, 1);
+ RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in -
+ __builtin_popcountll(*lookup_hit_mask));
+ return status;
+ }
/* Pipeline stage 0 */
lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
}
*lookup_hit_mask = pkts_mask_out;
+ RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));
return status;
}
+static int
+rte_table_hash_ext_stats_read(void *table, struct rte_table_stats *stats, int clear)
+{
+ struct rte_table_hash *t = table;
+
+ if (stats != NULL)
+ memcpy(stats, &t->stats, sizeof(t->stats));
+
+ if (clear)
+ memset(&t->stats, 0, sizeof(t->stats));
+
+ return 0;
+}
+
struct rte_table_ops rte_table_hash_ext_ops = {
.f_create = rte_table_hash_ext_create,
.f_free = rte_table_hash_ext_free,
.f_add = rte_table_hash_ext_entry_add,
.f_delete = rte_table_hash_ext_entry_delete,
+ .f_add_bulk = NULL,
+ .f_delete_bulk = NULL,
.f_lookup = rte_table_hash_ext_lookup,
+ .f_stats = rte_table_hash_ext_stats_read,
};
struct rte_table_ops rte_table_hash_ext_dosig_ops = {
.f_free = rte_table_hash_ext_free,
.f_add = rte_table_hash_ext_entry_add,
.f_delete = rte_table_hash_ext_entry_delete,
+ .f_add_bulk = NULL,
+ .f_delete_bulk = NULL,
.f_lookup = rte_table_hash_ext_lookup_dosig,
+ .f_stats = rte_table_hash_ext_stats_read,
};
git.droids-corp.org / dpdk.git / blobdiff