#include <rte_common.h>
#include <rte_mbuf.h>
+#include <rte_memory.h>
#include <rte_malloc.h>
#include <rte_log.h>
#define KEYS_PER_BUCKET 4
+#ifdef RTE_TABLE_STATS_COLLECT
+
+#define RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(table, val) \
+ table->stats.n_pkts_in += val
+#define RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(table, val) \
+ table->stats.n_pkts_lookup_miss += val
+
+#else
+
+#define RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(table, val)
+#define RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(table, val)
+
+#endif
+
struct bucket {
union {
struct bucket *next;
};
struct rte_table_hash {
+ struct rte_table_stats stats;
+
/* Input parameters */
uint32_t key_size;
uint32_t entry_size;
return -EINVAL;
}
- /* signature offset */
- if ((params->signature_offset & 0x3) != 0) {
- RTE_LOG(ERR, TABLE, "%s: signature_offset invalid value\n",
- __func__);
- return -EINVAL;
- }
-
- /* key offset */
- if ((params->key_offset & 0x7) != 0) {
- RTE_LOG(ERR, TABLE, "%s: key_offset invalid value\n", __func__);
- return -EINVAL;
- }
-
return 0;
}
rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size)
{
struct rte_table_hash_lru_params *p =
- (struct rte_table_hash_lru_params *) params;
+ params;
struct rte_table_hash *t;
uint32_t total_size, table_meta_sz;
uint32_t bucket_sz, key_sz, key_stack_sz, data_sz;
/* Check input parameters */
if ((check_params_create(p) != 0) ||
(!rte_is_power_of_2(entry_size)) ||
- ((sizeof(struct rte_table_hash) % CACHE_LINE_SIZE) != 0) ||
- (sizeof(struct bucket) != (CACHE_LINE_SIZE / 2))) {
+ ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
+ (sizeof(struct bucket) != (RTE_CACHE_LINE_SIZE / 2))) {
return NULL;
}
/* Memory allocation */
- table_meta_sz = CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash));
- bucket_sz = CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket));
- key_sz = CACHE_LINE_ROUNDUP(p->n_keys * p->key_size);
- key_stack_sz = CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t));
- data_sz = CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
+ table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash));
+ bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket));
+ key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size);
+ key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t));
+ data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
total_size = table_meta_sz + bucket_sz + key_sz + key_stack_sz +
data_sz;
- t = rte_zmalloc_socket("TABLE", total_size, CACHE_LINE_SIZE, socket_id);
+ t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
if (t == NULL) {
RTE_LOG(ERR, TABLE,
"%s: Cannot allocate %u bytes for hash table\n",
static int
rte_table_hash_lru_free(void *table)
{
- struct rte_table_hash *t = (struct rte_table_hash *) table;
+ struct rte_table_hash *t = table;
/* Check input parameters */
if (t == NULL)
rte_table_hash_lru_entry_add(void *table, void *key, void *entry,
int *key_found, void **entry_ptr)
{
- struct rte_table_hash *t = (struct rte_table_hash *) table;
+ struct rte_table_hash *t = table;
struct bucket *bkt;
uint64_t sig;
uint32_t bkt_index, i;
rte_table_hash_lru_entry_delete(void *table, void *key, int *key_found,
void *entry)
{
- struct rte_table_hash *t = (struct rte_table_hash *) table;
+ struct rte_table_hash *t = table;
struct bucket *bkt;
uint64_t sig;
uint32_t bkt_index, i;
struct rte_table_hash *t = (struct rte_table_hash *) table;
uint64_t pkts_mask_out = 0;
+ __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
+ RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(t, n_pkts_in);
+
for ( ; pkts_mask; ) {
struct bucket *bkt;
struct rte_mbuf *pkt;
}
*lookup_hit_mask = pkts_mask_out;
+ RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));
return 0;
}
{ \
uint64_t pkt00_mask, pkt01_mask; \
struct rte_mbuf *mbuf00, *mbuf01; \
+ uint32_t key_offset = t->key_offset; \
\
pkt00_index = __builtin_ctzll(pkts_mask); \
pkt00_mask = 1LLU << pkt00_index; \
pkts_mask &= ~pkt01_mask; \
mbuf01 = pkts[pkt01_index]; \
\
- rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0)); \
- rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0)); \
+ rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\
+ rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\
}
#define lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask, pkt00_index, \
{ \
uint64_t pkt00_mask, pkt01_mask; \
struct rte_mbuf *mbuf00, *mbuf01; \
+ uint32_t key_offset = t->key_offset; \
\
pkt00_index = __builtin_ctzll(pkts_mask); \
pkt00_mask = 1LLU << pkt00_index; \
pkts_mask &= ~pkt01_mask; \
mbuf01 = pkts[pkt01_index]; \
\
- rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0)); \
- rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0)); \
+ rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\
+ rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\
}
#define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index) \
uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0;
int status = 0;
+ __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
+ RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(t, n_pkts_in);
+
/* Cannot run the pipeline with less than 7 packets */
if (__builtin_popcountll(pkts_mask) < 7)
return rte_table_hash_lru_lookup_unoptimized(table, pkts,
}
*lookup_hit_mask = pkts_mask_out;
+ RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));
return status;
}
uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0;
int status = 0;
+ __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
+ RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(t, n_pkts_in);
+
/* Cannot run the pipeline with less than 7 packets */
if (__builtin_popcountll(pkts_mask) < 7)
return rte_table_hash_lru_lookup_unoptimized(table, pkts,
}
*lookup_hit_mask = pkts_mask_out;
+ RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));
return status;
}
+static int
+rte_table_hash_lru_stats_read(void *table, struct rte_table_stats *stats, int clear)
+{
+ struct rte_table_hash *t = table;
+
+ if (stats != NULL)
+ memcpy(stats, &t->stats, sizeof(t->stats));
+
+ if (clear)
+ memset(&t->stats, 0, sizeof(t->stats));
+
+ return 0;
+}
+
struct rte_table_ops rte_table_hash_lru_ops = {
.f_create = rte_table_hash_lru_create,
.f_free = rte_table_hash_lru_free,
.f_add = rte_table_hash_lru_entry_add,
.f_delete = rte_table_hash_lru_entry_delete,
+ .f_add_bulk = NULL,
+ .f_delete_bulk = NULL,
.f_lookup = rte_table_hash_lru_lookup,
+ .f_stats = rte_table_hash_lru_stats_read,
};
struct rte_table_ops rte_table_hash_lru_dosig_ops = {
.f_free = rte_table_hash_lru_free,
.f_add = rte_table_hash_lru_entry_add,
.f_delete = rte_table_hash_lru_entry_delete,
+ .f_add_bulk = NULL,
+ .f_delete_bulk = NULL,
.f_lookup = rte_table_hash_lru_lookup_dosig,
+ .f_stats = rte_table_hash_lru_stats_read,
};