kni: remove continuous memory restriction

[dpdk.git] / lib / librte_table / rte_table_hash_key8.c

diff --git a/lib/librte_table/rte_table_hash_key8.c b/lib/librte_table/rte_table_hash_key8.c
index 379748e..b04f60d 100644 (file)
--- a/lib/librte_table/rte_table_hash_key8.c
+++ b/lib/librte_table/rte_table_hash_key8.c
@@ -82,6 +82,7 @@ struct rte_table_hash {
        uint32_t bucket_size;
        uint32_t signature_offset;
        uint32_t key_offset;
+       uint64_t key_mask;
        rte_table_hash_op_hash f_hash;
        uint64_t seed;
 
@@ -124,7 +125,7 @@ rte_table_hash_create_key8_lru(void *params, int socket_id, uint32_t entry_size)
        /* Check input parameters */
        if ((check_params_create_lru(p) != 0) ||
                ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
-               ((sizeof(struct rte_bucket_4_8) % RTE_CACHE_LINE_SIZE) != 0)) {
+               ((sizeof(struct rte_bucket_4_8) % 64) != 0)) {
                return NULL;
        }
        n_entries_per_bucket = 4;
@@ -160,6 +161,11 @@ rte_table_hash_create_key8_lru(void *params, int socket_id, uint32_t entry_size)
        f->f_hash = p->f_hash;
        f->seed = p->seed;
 
+       if (p->key_mask != NULL)
+               f->key_mask = ((uint64_t *)p->key_mask)[0];
+       else
+               f->key_mask = 0xFFFFFFFFFFFFFFFFLLU;
+
        for (i = 0; i < n_buckets; i++) {
                struct rte_bucket_4_8 *bucket;
 
@@ -326,7 +332,7 @@ rte_table_hash_create_key8_ext(void *params, int socket_id, uint32_t entry_size)
        /* Check input parameters */
        if ((check_params_create_ext(p) != 0) ||
                ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
-               ((sizeof(struct rte_bucket_4_8) % RTE_CACHE_LINE_SIZE) != 0))
+               ((sizeof(struct rte_bucket_4_8) % 64) != 0))
                return NULL;
 
        n_entries_per_bucket = 4;
@@ -372,6 +378,11 @@ rte_table_hash_create_key8_ext(void *params, int socket_id, uint32_t entry_size)
        f->stack = (uint32_t *)
                &f->memory[(n_buckets + n_buckets_ext) * f->bucket_size];
 
+       if (p->key_mask != NULL)
+               f->key_mask = ((uint64_t *)p->key_mask)[0];
+       else
+               f->key_mask = 0xFFFFFFFFFFFFFFFFLLU;
+
        for (i = 0; i < n_buckets_ext; i++)
                f->stack[i] = i;
 
@@ -557,16 +568,17 @@ rte_table_hash_entry_delete_key8_ext(
                pos = 3;                                        \
 }
 
-#define lookup1_stage0(pkt0_index, mbuf0, pkts, pkts_mask)     \
+#define lookup1_stage0(pkt0_index, mbuf0, pkts, pkts_mask, f)  \
 {                                                              \
        uint64_t pkt_mask;                                      \
+       uint32_t key_offset = f->key_offset;\
                                                                \
        pkt0_index = __builtin_ctzll(pkts_mask);                \
        pkt_mask = 1LLU << pkt0_index;                          \
        pkts_mask &= ~pkt_mask;                                 \
                                                                \
        mbuf0 = pkts[pkt0_index];                               \
-       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf0, 0));   \
+       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf0, key_offset));  \
 }
 
 #define lookup1_stage1(mbuf1, bucket1, f)                      \
@@ -586,9 +598,12 @@ rte_table_hash_entry_delete_key8_ext(
        uint64_t *key;                                          \
        uint64_t signature;                                     \
        uint32_t bucket_index;                                  \
+       uint64_t hash_key_buffer;                               \
                                                                \
        key = RTE_MBUF_METADATA_UINT64_PTR(mbuf1, f->key_offset);\
-       signature = f->f_hash(key, RTE_TABLE_HASH_KEY_SIZE, f->seed);\
+       hash_key_buffer = *key & f->key_mask;                   \
+       signature = f->f_hash(&hash_key_buffer,                 \
+               RTE_TABLE_HASH_KEY_SIZE, f->seed);              \
        bucket_index = signature & (f->n_buckets - 1);          \
        bucket1 = (struct rte_bucket_4_8 *)                     \
                &f->memory[bucket_index * f->bucket_size];      \
@@ -602,10 +617,12 @@ rte_table_hash_entry_delete_key8_ext(
        uint64_t pkt_mask;                                      \
        uint64_t *key;                                          \
        uint32_t pos;                                           \
+       uint64_t hash_key_buffer;                               \
                                                                \
        key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\
+       hash_key_buffer = key[0] & f->key_mask;                 \
                                                                \
-       lookup_key8_cmp(key, bucket2, pos);                     \
+       lookup_key8_cmp((&hash_key_buffer), bucket2, pos);      \
                                                                \
        pkt_mask = ((bucket2->signature >> pos) & 1LLU) << pkt2_index;\
        pkts_mask_out |= pkt_mask;                              \
@@ -624,10 +641,12 @@ rte_table_hash_entry_delete_key8_ext(
        uint64_t pkt_mask, bucket_mask;                         \
        uint64_t *key;                                          \
        uint32_t pos;                                           \
+       uint64_t hash_key_buffer;                               \
                                                                \
        key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\
+       hash_key_buffer = *key & f->key_mask;                   \
                                                                \
-       lookup_key8_cmp(key, bucket2, pos);                     \
+       lookup_key8_cmp((&hash_key_buffer), bucket2, pos);      \
                                                                \
        pkt_mask = ((bucket2->signature >> pos) & 1LLU) << pkt2_index;\
        pkts_mask_out |= pkt_mask;                              \
@@ -651,11 +670,13 @@ rte_table_hash_entry_delete_key8_ext(
        uint64_t pkt_mask, bucket_mask;                         \
        uint64_t *key;                                          \
        uint32_t pos;                                           \
+       uint64_t hash_key_buffer;                               \
                                                                \
        bucket = buckets[pkt_index];                            \
        key = keys[pkt_index];                                  \
+       hash_key_buffer = (*key) & f->key_mask;                 \
                                                                \
-       lookup_key8_cmp(key, bucket, pos);                      \
+       lookup_key8_cmp((&hash_key_buffer), bucket, pos);       \
                                                                \
        pkt_mask = ((bucket->signature >> pos) & 1LLU) << pkt_index;\
        pkts_mask_out |= pkt_mask;                              \
@@ -673,36 +694,38 @@ rte_table_hash_entry_delete_key8_ext(
 }
 
 #define lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01,\
-       pkts, pkts_mask)                                        \
+       pkts, pkts_mask, f)                                     \
 {                                                              \
        uint64_t pkt00_mask, pkt01_mask;                        \
+       uint32_t key_offset = f->key_offset;            \
                                                                \
        pkt00_index = __builtin_ctzll(pkts_mask);               \
        pkt00_mask = 1LLU << pkt00_index;                       \
        pkts_mask &= ~pkt00_mask;                               \
                                                                \
        mbuf00 = pkts[pkt00_index];                             \
-       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0));  \
+       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\
                                                                \
        pkt01_index = __builtin_ctzll(pkts_mask);               \
        pkt01_mask = 1LLU << pkt01_index;                       \
        pkts_mask &= ~pkt01_mask;                               \
                                                                \
        mbuf01 = pkts[pkt01_index];                             \
-       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0));  \
+       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\
 }
 
 #define lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,\
-       mbuf00, mbuf01, pkts, pkts_mask)                        \
+       mbuf00, mbuf01, pkts, pkts_mask, f)                     \
 {                                                              \
        uint64_t pkt00_mask, pkt01_mask;                        \
+       uint32_t key_offset = f->key_offset;            \
                                                                \
        pkt00_index = __builtin_ctzll(pkts_mask);               \
        pkt00_mask = 1LLU << pkt00_index;                       \
        pkts_mask &= ~pkt00_mask;                               \
                                                                \
        mbuf00 = pkts[pkt00_index];                             \
-       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0));  \
+       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\
                                                                \
        pkt01_index = __builtin_ctzll(pkts_mask);               \
        if (pkts_mask == 0)                                     \
@@ -712,7 +735,7 @@ rte_table_hash_entry_delete_key8_ext(
        pkts_mask &= ~pkt01_mask;                               \
                                                                \
        mbuf01 = pkts[pkt01_index];                             \
-       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0));  \
+       rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\
 }
 
 #define lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f)  \
@@ -736,6 +759,8 @@ rte_table_hash_entry_delete_key8_ext(
 #define lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f)\
 {                                                              \
        uint64_t *key10, *key11;                                \
+       uint64_t hash_offset_buffer10;                          \
+       uint64_t hash_offset_buffer11;                          \
        uint64_t signature10, signature11;                      \
        uint32_t bucket10_index, bucket11_index;                \
        rte_table_hash_op_hash f_hash = f->f_hash;              \
@@ -744,14 +769,18 @@ rte_table_hash_entry_delete_key8_ext(
                                                                \
        key10 = RTE_MBUF_METADATA_UINT64_PTR(mbuf10, key_offset);\
        key11 = RTE_MBUF_METADATA_UINT64_PTR(mbuf11, key_offset);\
+       hash_offset_buffer10 = *key10 & f->key_mask;            \
+       hash_offset_buffer11 = *key11 & f->key_mask;            \
                                                                \
-       signature10 = f_hash(key10, RTE_TABLE_HASH_KEY_SIZE, seed);\
+       signature10 = f_hash(&hash_offset_buffer10,             \
+               RTE_TABLE_HASH_KEY_SIZE, seed);                 \
        bucket10_index = signature10 & (f->n_buckets - 1);      \
        bucket10 = (struct rte_bucket_4_8 *)                    \
                &f->memory[bucket10_index * f->bucket_size];    \
        rte_prefetch0(bucket10);                                \
                                                                \
-       signature11 = f_hash(key11, RTE_TABLE_HASH_KEY_SIZE, seed);\
+       signature11 = f_hash(&hash_offset_buffer11,             \
+               RTE_TABLE_HASH_KEY_SIZE, seed);                 \
        bucket11_index = signature11 & (f->n_buckets - 1);      \
        bucket11 = (struct rte_bucket_4_8 *)                    \
                &f->memory[bucket11_index * f->bucket_size];    \
@@ -764,13 +793,17 @@ rte_table_hash_entry_delete_key8_ext(
        void *a20, *a21;                                        \
        uint64_t pkt20_mask, pkt21_mask;                        \
        uint64_t *key20, *key21;                                \
+       uint64_t hash_offset_buffer20;                          \
+       uint64_t hash_offset_buffer21;                          \
        uint32_t pos20, pos21;                                  \
                                                                \
        key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\
        key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\
+       hash_offset_buffer20 = *key20 & f->key_mask;            \
+       hash_offset_buffer21 = *key21 & f->key_mask;            \
                                                                \
-       lookup_key8_cmp(key20, bucket20, pos20);                \
-       lookup_key8_cmp(key21, bucket21, pos21);                \
+       lookup_key8_cmp((&hash_offset_buffer20), bucket20, pos20);\
+       lookup_key8_cmp((&hash_offset_buffer21), bucket21, pos21);\
                                                                \
        pkt20_mask = ((bucket20->signature >> pos20) & 1LLU) << pkt20_index;\
        pkt21_mask = ((bucket21->signature >> pos21) & 1LLU) << pkt21_index;\
@@ -793,13 +826,17 @@ rte_table_hash_entry_delete_key8_ext(
        void *a20, *a21;                                        \
        uint64_t pkt20_mask, pkt21_mask, bucket20_mask, bucket21_mask;\
        uint64_t *key20, *key21;                                \
+       uint64_t hash_offset_buffer20;                          \
+       uint64_t hash_offset_buffer21;                          \
        uint32_t pos20, pos21;                                  \
                                                                \
        key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\
        key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\
+       hash_offset_buffer20 = *key20 & f->key_mask;            \
+       hash_offset_buffer21 = *key21 & f->key_mask;            \
                                                                \
-       lookup_key8_cmp(key20, bucket20, pos20);                \
-       lookup_key8_cmp(key21, bucket21, pos21);                \
+       lookup_key8_cmp((&hash_offset_buffer20), bucket20, pos20);\
+       lookup_key8_cmp((&hash_offset_buffer21), bucket21, pos21);\
                                                                \
        pkt20_mask = ((bucket20->signature >> pos20) & 1LLU) << pkt20_index;\
        pkt21_mask = ((bucket21->signature >> pos21) & 1LLU) << pkt21_index;\
@@ -848,7 +885,7 @@ rte_table_hash_lookup_key8_lru(
                        struct rte_mbuf *mbuf;
                        uint32_t pkt_index;
 
-                       lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask);
+                       lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f);
                        lookup1_stage1(mbuf, bucket, f);
                        lookup1_stage2_lru(pkt_index, mbuf, bucket,
                                        pkts_mask_out, entries, f);
@@ -865,7 +902,7 @@ rte_table_hash_lookup_key8_lru(
         */
        /* Pipeline stage 0 */
        lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
-               pkts_mask);
+               pkts_mask, f);
 
        /* Pipeline feed */
        mbuf10 = mbuf00;
@@ -875,7 +912,7 @@ rte_table_hash_lookup_key8_lru(
 
        /* Pipeline stage 0 */
        lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
-               pkts_mask);
+               pkts_mask, f);
 
        /* Pipeline stage 1 */
        lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
@@ -899,7 +936,7 @@ rte_table_hash_lookup_key8_lru(
 
                /* Pipeline stage 0 */
                lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,
-                       mbuf00, mbuf01, pkts, pkts_mask);
+                       mbuf00, mbuf01, pkts, pkts_mask, f);
 
                /* Pipeline stage 1 */
                lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
@@ -974,7 +1011,7 @@ rte_table_hash_lookup_key8_lru_dosig(
                        struct rte_mbuf *mbuf;
                        uint32_t pkt_index;
 
-                       lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask);
+                       lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f);
                        lookup1_stage1_dosig(mbuf, bucket, f);
                        lookup1_stage2_lru(pkt_index, mbuf, bucket,
                                pkts_mask_out, entries, f);
@@ -991,7 +1028,7 @@ rte_table_hash_lookup_key8_lru_dosig(
         */
        /* Pipeline stage 0 */
        lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
-               pkts_mask);
+               pkts_mask, f);
 
        /* Pipeline feed */
        mbuf10 = mbuf00;
@@ -1001,7 +1038,7 @@ rte_table_hash_lookup_key8_lru_dosig(
 
        /* Pipeline stage 0 */
        lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
-               pkts_mask);
+               pkts_mask, f);
 
        /* Pipeline stage 1 */
        lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
@@ -1025,7 +1062,7 @@ rte_table_hash_lookup_key8_lru_dosig(
 
                /* Pipeline stage 0 */
                lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,
-                       mbuf00, mbuf01, pkts, pkts_mask);
+                       mbuf00, mbuf01, pkts, pkts_mask, f);
 
                /* Pipeline stage 1 */
                lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
@@ -1102,7 +1139,7 @@ rte_table_hash_lookup_key8_ext(
                        struct rte_mbuf *mbuf;
                        uint32_t pkt_index;
 
-                       lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask);
+                       lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f);
                        lookup1_stage1(mbuf, bucket, f);
                        lookup1_stage2_ext(pkt_index, mbuf, bucket,
                                pkts_mask_out, entries, buckets_mask, buckets,
@@ -1118,7 +1155,7 @@ rte_table_hash_lookup_key8_ext(
         */
        /* Pipeline stage 0 */
        lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
-               pkts_mask);
+               pkts_mask, f);
 
        /* Pipeline feed */
        mbuf10 = mbuf00;
@@ -1128,7 +1165,7 @@ rte_table_hash_lookup_key8_ext(
 
        /* Pipeline stage 0 */
        lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
-               pkts_mask);
+               pkts_mask, f);
 
        /* Pipeline stage 1 */
        lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
@@ -1152,7 +1189,7 @@ rte_table_hash_lookup_key8_ext(
 
                /* Pipeline stage 0 */
                lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,
-                       mbuf00, mbuf01, pkts, pkts_mask);
+                       mbuf00, mbuf01, pkts, pkts_mask, f);
 
                /* Pipeline stage 1 */
                lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
@@ -1252,7 +1289,7 @@ rte_table_hash_lookup_key8_ext_dosig(
                        struct rte_mbuf *mbuf;
                        uint32_t pkt_index;
 
-                       lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask);
+                       lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f);
                        lookup1_stage1_dosig(mbuf, bucket, f);
                        lookup1_stage2_ext(pkt_index, mbuf, bucket,
                                pkts_mask_out, entries, buckets_mask,
@@ -1268,7 +1305,7 @@ rte_table_hash_lookup_key8_ext_dosig(
         */
        /* Pipeline stage 0 */
        lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
-               pkts_mask);
+               pkts_mask, f);
 
        /* Pipeline feed */
        mbuf10 = mbuf00;
@@ -1278,7 +1315,7 @@ rte_table_hash_lookup_key8_ext_dosig(
 
        /* Pipeline stage 0 */
        lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
-               pkts_mask);
+               pkts_mask, f);
 
        /* Pipeline stage 1 */
        lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);
@@ -1302,7 +1339,7 @@ rte_table_hash_lookup_key8_ext_dosig(
 
                /* Pipeline stage 0 */
                lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,
-                       mbuf00, mbuf01, pkts, pkts_mask);
+                       mbuf00, mbuf01, pkts, pkts_mask, f);
 
                /* Pipeline stage 1 */
                lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f);