lib/librte_table/rte_table_hash_lru.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_mbuf.h>
  39 #include <rte_memory.h>
  40 #include <rte_malloc.h>
  41 #include <rte_log.h>
  42
  43 #include "rte_table_hash.h"
  44 #include "rte_lru.h"
  45
  46 #define KEYS_PER_BUCKET 4
  47
  48 #ifdef RTE_TABLE_STATS_COLLECT
  49
  50 #define RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(table, val) \
  51         table->stats.n_pkts_in += val
  52 #define RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(table, val) \
  53         table->stats.n_pkts_lookup_miss += val
  54
  55 #else
  56
  57 #define RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(table, val)
  58 #define RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(table, val)
  59
  60 #endif
  61
  62 struct bucket {
  63         union {
  64                 struct bucket *next;
  65                 uint64_t lru_list;
  66         };
  67         uint16_t sig[KEYS_PER_BUCKET];
  68         uint32_t key_pos[KEYS_PER_BUCKET];
  69 };
  70
  71 struct grinder {
  72         struct bucket *bkt;
  73         uint64_t sig;
  74         uint64_t match;
  75         uint64_t match_pos;
  76         uint32_t key_index;
  77 };
  78
  79 struct rte_table_hash {
  80         struct rte_table_stats stats;
  81
  82         /* Input parameters */
  83         uint32_t key_size;
  84         uint32_t entry_size;
  85         uint32_t n_keys;
  86         uint32_t n_buckets;
  87         rte_table_hash_op_hash f_hash;
  88         uint64_t seed;
  89         uint32_t key_offset;
  90
  91         /* Internal */
  92         uint64_t bucket_mask;
  93         uint32_t key_size_shl;
  94         uint32_t data_size_shl;
  95         uint32_t key_stack_tos;
  96
  97         /* Grinder */
  98         struct grinder grinders[RTE_PORT_IN_BURST_SIZE_MAX];
  99
 100         /* Tables */
 101         uint64_t *key_mask;
 102         struct bucket *buckets;
 103         uint8_t *key_mem;
 104         uint8_t *data_mem;
 105         uint32_t *key_stack;
 106
 107         /* Table memory */
 108         uint8_t memory[0] __rte_cache_aligned;
 109 };
 110
 111 static int
 112 keycmp(void *a, void *b, void *b_mask, uint32_t n_bytes)
 113 {
 114         uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask;
 115         uint32_t i;
 116
 117         for (i = 0; i < n_bytes / sizeof(uint64_t); i++)
 118                 if (a64[i] != (b64[i] & b_mask64[i]))
 119                         return 1;
 120
 121         return 0;
 122 }
 123
 124 static void
 125 keycpy(void *dst, void *src, void *src_mask, uint32_t n_bytes)
 126 {
 127         uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask;
 128         uint32_t i;
 129
 130         for (i = 0; i < n_bytes / sizeof(uint64_t); i++)
 131                 dst64[i] = src64[i] & src_mask64[i];
 132 }
 133
 134 static int
 135 check_params_create(struct rte_table_hash_params *params)
 136 {
 137         /* name */
 138         if (params->name == NULL) {
 139                 RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__);
 140                 return -EINVAL;
 141         }
 142
 143         /* key_size */
 144         if ((params->key_size < sizeof(uint64_t)) ||
 145                 (!rte_is_power_of_2(params->key_size))) {
 146                 RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__);
 147                 return -EINVAL;
 148         }
 149
 150         /* n_keys */
 151         if ((params->n_keys == 0) ||
 152                 (!rte_is_power_of_2(params->n_keys))) {
 153                 RTE_LOG(ERR, TABLE, "%s: n_keys invalid value\n", __func__);
 154                 return -EINVAL;
 155         }
 156
 157         /* n_buckets */
 158         if ((params->n_buckets == 0) ||
 159                 (!rte_is_power_of_2(params->n_keys))) {
 160                 RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__);
 161                 return -EINVAL;
 162         }
 163
 164         /* f_hash */
 165         if (params->f_hash == NULL) {
 166                 RTE_LOG(ERR, TABLE, "%s: f_hash invalid value\n", __func__);
 167                 return -EINVAL;
 168         }
 169
 170         return 0;
 171 }
 172
 173 static void *
 174 rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size)
 175 {
 176         struct rte_table_hash_params *p = params;
 177         struct rte_table_hash *t;
 178         uint64_t table_meta_sz, key_mask_sz, bucket_sz, key_sz, key_stack_sz;
 179         uint64_t data_sz, total_size;
 180         uint64_t key_mask_offset, bucket_offset, key_offset, key_stack_offset;
 181         uint64_t data_offset;
 182         uint32_t n_buckets, i;
 183
 184         /* Check input parameters */
 185         if ((check_params_create(p) != 0) ||
 186                 (!rte_is_power_of_2(entry_size)) ||
 187                 ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
 188                 (sizeof(struct bucket) != (RTE_CACHE_LINE_SIZE / 2))) {
 189                 return NULL;
 190         }
 191
 192         /*
 193          * Table dimensioning
 194          *
 195          * Objective: Pick the number of buckets (n_buckets) so that there a chance
 196          * to store n_keys keys in the table.
 197          *
 198          * Note: Since the buckets do not get extended, it is not possible to
 199          * guarantee that n_keys keys can be stored in the table at any time. In the
 200          * worst case scenario when all the n_keys fall into the same bucket, only
 201          * a maximum of KEYS_PER_BUCKET keys will be stored in the table. This case
 202          * defeats the purpose of the hash table. It indicates unsuitable f_hash or
 203          * n_keys to n_buckets ratio.
 204          *
 205          * MIN(n_buckets) = (n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET
 206          */
 207         n_buckets = rte_align32pow2(
 208                 (p->n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET);
 209         n_buckets = RTE_MAX(n_buckets, p->n_buckets);
 210
 211         /* Memory allocation */
 212         table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash));
 213         key_mask_sz = RTE_CACHE_LINE_ROUNDUP(p->key_size);
 214         bucket_sz = RTE_CACHE_LINE_ROUNDUP(n_buckets * sizeof(struct bucket));
 215         key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size);
 216         key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t));
 217         data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
 218         total_size = table_meta_sz + key_mask_sz + bucket_sz + key_sz +
 219                 key_stack_sz + data_sz;
 220
 221         if (total_size > SIZE_MAX) {
 222                 RTE_LOG(ERR, TABLE,
 223                         "%s: Cannot allocate %" PRIu64 " bytes for hash "
 224                         "table %s\n",
 225                         __func__, total_size, p->name);
 226                 return NULL;
 227         }
 228
 229         t = rte_zmalloc_socket(p->name,
 230                 (size_t)total_size,
 231                 RTE_CACHE_LINE_SIZE,
 232                 socket_id);
 233         if (t == NULL) {
 234                 RTE_LOG(ERR, TABLE,
 235                         "%s: Cannot allocate %" PRIu64 " bytes for hash "
 236                         "table %s\n",
 237                         __func__, total_size, p->name);
 238                 return NULL;
 239         }
 240         RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table %s memory footprint"
 241                 " is %" PRIu64 " bytes\n",
 242                 __func__, p->key_size, p->name, total_size);
 243
 244         /* Memory initialization */
 245         t->key_size = p->key_size;
 246         t->entry_size = entry_size;
 247         t->n_keys = p->n_keys;
 248         t->n_buckets = n_buckets;
 249         t->f_hash = p->f_hash;
 250         t->seed = p->seed;
 251         t->key_offset = p->key_offset;
 252
 253         /* Internal */
 254         t->bucket_mask = t->n_buckets - 1;
 255         t->key_size_shl = __builtin_ctzl(p->key_size);
 256         t->data_size_shl = __builtin_ctzl(entry_size);
 257
 258         /* Tables */
 259         key_mask_offset = 0;
 260         bucket_offset = key_mask_offset + key_mask_sz;
 261         key_offset = bucket_offset + bucket_sz;
 262         key_stack_offset = key_offset + key_sz;
 263         data_offset = key_stack_offset + key_stack_sz;
 264
 265         t->key_mask = (uint64_t *) &t->memory[key_mask_offset];
 266         t->buckets = (struct bucket *) &t->memory[bucket_offset];
 267         t->key_mem = &t->memory[key_offset];
 268         t->key_stack = (uint32_t *) &t->memory[key_stack_offset];
 269         t->data_mem = &t->memory[data_offset];
 270
 271         /* Key mask */
 272         if (p->key_mask == NULL)
 273                 memset(t->key_mask, 0xFF, p->key_size);
 274         else
 275                 memcpy(t->key_mask, p->key_mask, p->key_size);
 276
 277         /* Key stack */
 278         for (i = 0; i < t->n_keys; i++)
 279                 t->key_stack[i] = t->n_keys - 1 - i;
 280         t->key_stack_tos = t->n_keys;
 281
 282         /* LRU */
 283         for (i = 0; i < t->n_buckets; i++) {
 284                 struct bucket *bkt = &t->buckets[i];
 285
 286                 lru_init(bkt);
 287         }
 288
 289         return t;
 290 }
 291
 292 static int
 293 rte_table_hash_lru_free(void *table)
 294 {
 295         struct rte_table_hash *t = table;
 296
 297         /* Check input parameters */
 298         if (t == NULL)
 299                 return -EINVAL;
 300
 301         rte_free(t);
 302         return 0;
 303 }
 304
 305 static int
 306 rte_table_hash_lru_entry_add(void *table, void *key, void *entry,
 307         int *key_found, void **entry_ptr)
 308 {
 309         struct rte_table_hash *t = table;
 310         struct bucket *bkt;
 311         uint64_t sig;
 312         uint32_t bkt_index, i;
 313
 314         sig = t->f_hash(key, t->key_mask, t->key_size, t->seed);
 315         bkt_index = sig & t->bucket_mask;
 316         bkt = &t->buckets[bkt_index];
 317         sig = (sig >> 16) | 1LLU;
 318
 319         /* Key is present in the bucket */
 320         for (i = 0; i < KEYS_PER_BUCKET; i++) {
 321                 uint64_t bkt_sig = (uint64_t) bkt->sig[i];
 322                 uint32_t bkt_key_index = bkt->key_pos[i];
 323                 uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
 324                         t->key_size_shl];
 325
 326                 if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask,
 327                         t->key_size) == 0)) {
 328                         uint8_t *data = &t->data_mem[bkt_key_index <<
 329                                 t->data_size_shl];
 330
 331                         memcpy(data, entry, t->entry_size);
 332                         lru_update(bkt, i);
 333                         *key_found = 1;
 334                         *entry_ptr = (void *) data;
 335                         return 0;
 336                 }
 337         }
 338
 339         /* Key is not present in the bucket */
 340         for (i = 0; i < KEYS_PER_BUCKET; i++) {
 341                 uint64_t bkt_sig = (uint64_t) bkt->sig[i];
 342
 343                 if (bkt_sig == 0) {
 344                         uint32_t bkt_key_index;
 345                         uint8_t *bkt_key, *data;
 346
 347                         /* Allocate new key */
 348                         if (t->key_stack_tos == 0) {
 349                                 /* No keys available */
 350                                 return -ENOSPC;
 351                         }
 352                         bkt_key_index = t->key_stack[--t->key_stack_tos];
 353
 354                         /* Install new key */
 355                         bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl];
 356                         data = &t->data_mem[bkt_key_index << t->data_size_shl];
 357
 358                         bkt->sig[i] = (uint16_t) sig;
 359                         bkt->key_pos[i] = bkt_key_index;
 360                         keycpy(bkt_key, key, t->key_mask, t->key_size);
 361                         memcpy(data, entry, t->entry_size);
 362                         lru_update(bkt, i);
 363
 364                         *key_found = 0;
 365                         *entry_ptr = (void *) data;
 366                         return 0;
 367                 }
 368         }
 369
 370         /* Bucket full */
 371         {
 372                 uint64_t pos = lru_pos(bkt);
 373                 uint32_t bkt_key_index = bkt->key_pos[pos];
 374                 uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
 375                         t->key_size_shl];
 376                 uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl];
 377
 378                 bkt->sig[pos] = (uint16_t) sig;
 379                 keycpy(bkt_key, key, t->key_mask, t->key_size);
 380                 memcpy(data, entry, t->entry_size);
 381                 lru_update(bkt, pos);
 382
 383                 *key_found = 0;
 384                 *entry_ptr = (void *) data;
 385                 return 0;
 386         }
 387 }
 388
 389 static int
 390 rte_table_hash_lru_entry_delete(void *table, void *key, int *key_found,
 391         void *entry)
 392 {
 393         struct rte_table_hash *t = table;
 394         struct bucket *bkt;
 395         uint64_t sig;
 396         uint32_t bkt_index, i;
 397
 398         sig = t->f_hash(key, t->key_mask, t->key_size, t->seed);
 399         bkt_index = sig & t->bucket_mask;
 400         bkt = &t->buckets[bkt_index];
 401         sig = (sig >> 16) | 1LLU;
 402
 403         /* Key is present in the bucket */
 404         for (i = 0; i < KEYS_PER_BUCKET; i++) {
 405                 uint64_t bkt_sig = (uint64_t) bkt->sig[i];
 406                 uint32_t bkt_key_index = bkt->key_pos[i];
 407                 uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
 408                         t->key_size_shl];
 409
 410                 if ((sig == bkt_sig) &&
 411                         (keycmp(bkt_key, key, t->key_mask, t->key_size) == 0)) {
 412                         uint8_t *data = &t->data_mem[bkt_key_index <<
 413                                 t->data_size_shl];
 414
 415                         bkt->sig[i] = 0;
 416                         t->key_stack[t->key_stack_tos++] = bkt_key_index;
 417                         *key_found = 1;
 418                         if (entry)
 419                                 memcpy(entry, data, t->entry_size);
 420                         return 0;
 421                 }
 422         }
 423
 424         /* Key is not present in the bucket */
 425         *key_found = 0;
 426         return 0;
 427 }
 428
 429 static int rte_table_hash_lru_lookup_unoptimized(
 430         void *table,
 431         struct rte_mbuf **pkts,
 432         uint64_t pkts_mask,
 433         uint64_t *lookup_hit_mask,
 434         void **entries)
 435 {
 436         struct rte_table_hash *t = (struct rte_table_hash *) table;
 437         uint64_t pkts_mask_out = 0;
 438
 439         __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
 440         RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(t, n_pkts_in);
 441
 442         for ( ; pkts_mask; ) {
 443                 struct bucket *bkt;
 444                 struct rte_mbuf *pkt;
 445                 uint8_t *key;
 446                 uint64_t pkt_mask, sig;
 447                 uint32_t pkt_index, bkt_index, i;
 448
 449                 pkt_index = __builtin_ctzll(pkts_mask);
 450                 pkt_mask = 1LLU << pkt_index;
 451                 pkts_mask &= ~pkt_mask;
 452
 453                 pkt = pkts[pkt_index];
 454                 key = RTE_MBUF_METADATA_UINT8_PTR(pkt, t->key_offset);
 455                 sig = (uint64_t) t->f_hash(key, t->key_mask, t->key_size, t->seed);
 456
 457                 bkt_index = sig & t->bucket_mask;
 458                 bkt = &t->buckets[bkt_index];
 459                 sig = (sig >> 16) | 1LLU;
 460
 461                 /* Key is present in the bucket */
 462                 for (i = 0; i < KEYS_PER_BUCKET; i++) {
 463                         uint64_t bkt_sig = (uint64_t) bkt->sig[i];
 464                         uint32_t bkt_key_index = bkt->key_pos[i];
 465                         uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
 466                                 t->key_size_shl];
 467
 468                         if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask,
 469                                 t->key_size) == 0)) {
 470                                 uint8_t *data = &t->data_mem[bkt_key_index <<
 471                                         t->data_size_shl];
 472
 473                                 lru_update(bkt, i);
 474                                 pkts_mask_out |= pkt_mask;
 475                                 entries[pkt_index] = (void *) data;
 476                                 break;
 477                         }
 478                 }
 479         }
 480
 481         *lookup_hit_mask = pkts_mask_out;
 482         RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));
 483         return 0;
 484 }
 485
 486 /***
 487 *
 488 * mask = match bitmask
 489 * match = at least one match
 490 * match_many = more than one match
 491 * match_pos = position of first match
 492 *
 493 * ----------------------------------------
 494 * mask           match   match_many       match_pos
 495 * ----------------------------------------
 496 * 0000           0               0                        00
 497 * 0001           1               0                        00
 498 * 0010           1               0                        01
 499 * 0011           1               1                        00
 500 * ----------------------------------------
 501 * 0100           1               0                        10
 502 * 0101           1               1                        00
 503 * 0110           1               1                        01
 504 * 0111           1               1                        00
 505 * ----------------------------------------
 506 * 1000           1               0                        11
 507 * 1001           1               1                        00
 508 * 1010           1               1                        01
 509 * 1011           1               1                        00
 510 * ----------------------------------------
 511 * 1100           1               1                        10
 512 * 1101           1               1                        00
 513 * 1110           1               1                        01
 514 * 1111           1               1                        00
 515 * ----------------------------------------
 516 *
 517 * match = 1111_1111_1111_1110
 518 * match_many = 1111_1110_1110_1000
 519 * match_pos = 0001_0010_0001_0011__0001_0010_0001_0000
 520 *
 521 * match = 0xFFFELLU
 522 * match_many = 0xFEE8LLU
 523 * match_pos = 0x12131210LLU
 524 *
 525 ***/
 526
 527 #define LUT_MATCH                                               0xFFFELLU
 528 #define LUT_MATCH_MANY                                          0xFEE8LLU
 529 #define LUT_MATCH_POS                                           0x12131210LLU
 530
 531 #define lookup_cmp_sig(mbuf_sig, bucket, match, match_many, match_pos)\
 532 {                                                               \
 533         uint64_t bucket_sig[4], mask[4], mask_all;              \
 534                                                                 \
 535         bucket_sig[0] = bucket->sig[0];                         \
 536         bucket_sig[1] = bucket->sig[1];                         \
 537         bucket_sig[2] = bucket->sig[2];                         \
 538         bucket_sig[3] = bucket->sig[3];                         \
 539                                                                 \
 540         bucket_sig[0] ^= mbuf_sig;                              \
 541         bucket_sig[1] ^= mbuf_sig;                              \
 542         bucket_sig[2] ^= mbuf_sig;                              \
 543         bucket_sig[3] ^= mbuf_sig;                              \
 544                                                                 \
 545         mask[0] = 0;                                            \
 546         mask[1] = 0;                                            \
 547         mask[2] = 0;                                            \
 548         mask[3] = 0;                                            \
 549                                                                 \
 550         if (bucket_sig[0] == 0)                                 \
 551                 mask[0] = 1;                                    \
 552         if (bucket_sig[1] == 0)                                 \
 553                 mask[1] = 2;                                    \
 554         if (bucket_sig[2] == 0)                                 \
 555                 mask[2] = 4;                                    \
 556         if (bucket_sig[3] == 0)                                 \
 557                 mask[3] = 8;                                    \
 558                                                                 \
 559         mask_all = (mask[0] | mask[1]) | (mask[2] | mask[3]);   \
 560                                                                 \
 561         match = (LUT_MATCH >> mask_all) & 1;                    \
 562         match_many = (LUT_MATCH_MANY >> mask_all) & 1;          \
 563         match_pos = (LUT_MATCH_POS >> (mask_all << 1)) & 3;     \
 564 }
 565
 566 #define lookup_cmp_key(mbuf, key, match_key, f)                         \
 567 {                                                                       \
 568         uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(mbuf, f->key_offset);\
 569         uint64_t *bkt_key = (uint64_t *) key;                           \
 570         uint64_t *key_mask = f->key_mask;                                       \
 571                                                                         \
 572         switch (f->key_size) {                                          \
 573         case 8:                                                         \
 574         {                                                               \
 575                 uint64_t xor = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \
 576                 match_key = 0;                                          \
 577                 if (xor == 0)                                           \
 578                         match_key = 1;                                  \
 579         }                                                               \
 580         break;                                                          \
 581                                                                         \
 582         case 16:                                                        \
 583         {                                                               \
 584                 uint64_t xor[2], or;                                    \
 585                                                                         \
 586                 xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0];               \
 587                 xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1];               \
 588                 or = xor[0] | xor[1];                                   \
 589                 match_key = 0;                                          \
 590                 if (or == 0)                                            \
 591                         match_key = 1;                                  \
 592         }                                                               \
 593         break;                                                          \
 594                                                                         \
 595         case 32:                                                        \
 596         {                                                               \
 597                 uint64_t xor[4], or;                                    \
 598                                                                         \
 599                 xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0];               \
 600                 xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1];               \
 601                 xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2];               \
 602                 xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3];               \
 603                 or = xor[0] | xor[1] | xor[2] | xor[3];                 \
 604                 match_key = 0;                                          \
 605                 if (or == 0)                                            \
 606                         match_key = 1;                                  \
 607         }                                                               \
 608         break;                                                          \
 609                                                                         \
 610         case 64:                                                        \
 611         {                                                               \
 612                 uint64_t xor[8], or;                                    \
 613                                                                         \
 614                 xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0];               \
 615                 xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1];               \
 616                 xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2];               \
 617                 xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3];               \
 618                 xor[4] = (pkt_key[4] & key_mask[4]) ^ bkt_key[4];               \
 619                 xor[5] = (pkt_key[5] & key_mask[5]) ^ bkt_key[5];               \
 620                 xor[6] = (pkt_key[6] & key_mask[6]) ^ bkt_key[6];               \
 621                 xor[7] = (pkt_key[7] & key_mask[7]) ^ bkt_key[7];               \
 622                 or = xor[0] | xor[1] | xor[2] | xor[3] |                \
 623                         xor[4] | xor[5] | xor[6] | xor[7];              \
 624                 match_key = 0;                                          \
 625                 if (or == 0)                                            \
 626                         match_key = 1;                                  \
 627         }                                                               \
 628         break;                                                          \
 629                                                                         \
 630         default:                                                        \
 631                 match_key = 0;                                          \
 632                 if (keycmp(bkt_key, pkt_key, key_mask, f->key_size) == 0)       \
 633                         match_key = 1;                                  \
 634         }                                                               \
 635 }
 636
 637 #define lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index)\
 638 {                                                               \
 639         uint64_t pkt00_mask, pkt01_mask;                        \
 640         struct rte_mbuf *mbuf00, *mbuf01;                       \
 641         uint32_t key_offset = t->key_offset;            \
 642                                                                 \
 643         pkt00_index = __builtin_ctzll(pkts_mask);               \
 644         pkt00_mask = 1LLU << pkt00_index;                       \
 645         pkts_mask &= ~pkt00_mask;                               \
 646         mbuf00 = pkts[pkt00_index];                             \
 647                                                                 \
 648         pkt01_index = __builtin_ctzll(pkts_mask);               \
 649         pkt01_mask = 1LLU << pkt01_index;                       \
 650         pkts_mask &= ~pkt01_mask;                               \
 651         mbuf01 = pkts[pkt01_index];                             \
 652                                                                 \
 653         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\
 654         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\
 655 }
 656
 657 #define lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask, pkt00_index, \
 658         pkt01_index)                                            \
 659 {                                                               \
 660         uint64_t pkt00_mask, pkt01_mask;                        \
 661         struct rte_mbuf *mbuf00, *mbuf01;                       \
 662         uint32_t key_offset = t->key_offset;            \
 663                                                                 \
 664         pkt00_index = __builtin_ctzll(pkts_mask);               \
 665         pkt00_mask = 1LLU << pkt00_index;                       \
 666         pkts_mask &= ~pkt00_mask;                               \
 667         mbuf00 = pkts[pkt00_index];                             \
 668                                                                 \
 669         pkt01_index = __builtin_ctzll(pkts_mask);               \
 670         if (pkts_mask == 0)                                     \
 671                 pkt01_index = pkt00_index;                      \
 672                                                                 \
 673         pkt01_mask = 1LLU << pkt01_index;                       \
 674         pkts_mask &= ~pkt01_mask;                               \
 675         mbuf01 = pkts[pkt01_index];                             \
 676                                                                 \
 677         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, key_offset));\
 678         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\
 679 }
 680
 681 #define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index)\
 682 {                                                               \
 683         struct grinder *g10, *g11;                              \
 684         uint64_t sig10, sig11, bkt10_index, bkt11_index;        \
 685         struct rte_mbuf *mbuf10, *mbuf11;                       \
 686         struct bucket *bkt10, *bkt11, *buckets = t->buckets;    \
 687         uint8_t *key10, *key11;                                 \
 688         uint64_t bucket_mask = t->bucket_mask;                  \
 689         rte_table_hash_op_hash f_hash = t->f_hash;              \
 690         uint64_t seed = t->seed;                                \
 691         uint32_t key_size = t->key_size;                        \
 692         uint32_t key_offset = t->key_offset;                    \
 693                                                                 \
 694         mbuf10 = pkts[pkt10_index];                             \
 695         key10 = RTE_MBUF_METADATA_UINT8_PTR(mbuf10, key_offset);\
 696         sig10 = (uint64_t) f_hash(key10, t->key_mask, key_size, seed);\
 697         bkt10_index = sig10 & bucket_mask;                      \
 698         bkt10 = &buckets[bkt10_index];                          \
 699                                                                 \
 700         mbuf11 = pkts[pkt11_index];                             \
 701         key11 = RTE_MBUF_METADATA_UINT8_PTR(mbuf11, key_offset);\
 702         sig11 = (uint64_t) f_hash(key11, t->key_mask, key_size, seed);\
 703         bkt11_index = sig11 & bucket_mask;                      \
 704         bkt11 = &buckets[bkt11_index];                          \
 705                                                                 \
 706         rte_prefetch0(bkt10);                                   \
 707         rte_prefetch0(bkt11);                                   \
 708                                                                 \
 709         g10 = &g[pkt10_index];                                  \
 710         g10->sig = sig10;                                       \
 711         g10->bkt = bkt10;                                       \
 712                                                                 \
 713         g11 = &g[pkt11_index];                                  \
 714         g11->sig = sig11;                                       \
 715         g11->bkt = bkt11;                                       \
 716 }
 717
 718 #define lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many)\
 719 {                                                               \
 720         struct grinder *g20, *g21;                              \
 721         uint64_t sig20, sig21;                                  \
 722         struct bucket *bkt20, *bkt21;                           \
 723         uint8_t *key20, *key21, *key_mem = t->key_mem;          \
 724         uint64_t match20, match21, match_many20, match_many21;  \
 725         uint64_t match_pos20, match_pos21;                      \
 726         uint32_t key20_index, key21_index, key_size_shl = t->key_size_shl;\
 727                                                                 \
 728         g20 = &g[pkt20_index];                                  \
 729         sig20 = g20->sig;                                       \
 730         bkt20 = g20->bkt;                                       \
 731         sig20 = (sig20 >> 16) | 1LLU;                           \
 732         lookup_cmp_sig(sig20, bkt20, match20, match_many20, match_pos20);\
 733         match20 <<= pkt20_index;                                \
 734         match_many20 <<= pkt20_index;                           \
 735         key20_index = bkt20->key_pos[match_pos20];              \
 736         key20 = &key_mem[key20_index << key_size_shl];          \
 737                                                                 \
 738         g21 = &g[pkt21_index];                                  \
 739         sig21 = g21->sig;                                       \
 740         bkt21 = g21->bkt;                                       \
 741         sig21 = (sig21 >> 16) | 1LLU;                           \
 742         lookup_cmp_sig(sig21, bkt21, match21, match_many21, match_pos21);\
 743         match21 <<= pkt21_index;                                \
 744         match_many21 <<= pkt21_index;                           \
 745         key21_index = bkt21->key_pos[match_pos21];              \
 746         key21 = &key_mem[key21_index << key_size_shl];          \
 747                                                                 \
 748         rte_prefetch0(key20);                                   \
 749         rte_prefetch0(key21);                                   \
 750                                                                 \
 751         pkts_mask_match_many |= match_many20 | match_many21;    \
 752                                                                 \
 753         g20->match = match20;                                   \
 754         g20->match_pos = match_pos20;                           \
 755         g20->key_index = key20_index;                           \
 756                                                                 \
 757         g21->match = match21;                                   \
 758         g21->match_pos = match_pos21;                           \
 759         g21->key_index = key21_index;                           \
 760 }
 761
 762 #define lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, \
 763         entries)                                                \
 764 {                                                               \
 765         struct grinder *g30, *g31;                              \
 766         struct rte_mbuf *mbuf30, *mbuf31;                       \
 767         struct bucket *bkt30, *bkt31;                           \
 768         uint8_t *key30, *key31, *key_mem = t->key_mem;          \
 769         uint8_t *data30, *data31, *data_mem = t->data_mem;      \
 770         uint64_t match30, match31, match_pos30, match_pos31;    \
 771         uint64_t match_key30, match_key31, match_keys;          \
 772         uint32_t key30_index, key31_index;                      \
 773         uint32_t key_size_shl = t->key_size_shl;                \
 774         uint32_t data_size_shl = t->data_size_shl;              \
 775                                                                 \
 776         mbuf30 = pkts[pkt30_index];                             \
 777         g30 = &g[pkt30_index];                                  \
 778         bkt30 = g30->bkt;                                       \
 779         match30 = g30->match;                                   \
 780         match_pos30 = g30->match_pos;                           \
 781         key30_index = g30->key_index;                           \
 782         key30 = &key_mem[key30_index << key_size_shl];          \
 783         lookup_cmp_key(mbuf30, key30, match_key30, t);          \
 784         match_key30 <<= pkt30_index;                            \
 785         match_key30 &= match30;                                 \
 786         data30 = &data_mem[key30_index << data_size_shl];       \
 787         entries[pkt30_index] = data30;                          \
 788                                                                 \
 789         mbuf31 = pkts[pkt31_index];                             \
 790         g31 = &g[pkt31_index];                                  \
 791         bkt31 = g31->bkt;                                       \
 792         match31 = g31->match;                                   \
 793         match_pos31 = g31->match_pos;                           \
 794         key31_index = g31->key_index;                           \
 795         key31 = &key_mem[key31_index << key_size_shl];          \
 796         lookup_cmp_key(mbuf31, key31, match_key31, t);          \
 797         match_key31 <<= pkt31_index;                            \
 798         match_key31 &= match31;                                 \
 799         data31 = &data_mem[key31_index << data_size_shl];       \
 800         entries[pkt31_index] = data31;                          \
 801                                                                 \
 802         rte_prefetch0(data30);                                  \
 803         rte_prefetch0(data31);                                  \
 804                                                                 \
 805         match_keys = match_key30 | match_key31;                 \
 806         pkts_mask_out |= match_keys;                            \
 807                                                                 \
 808         if (match_key30 == 0)                                   \
 809                 match_pos30 = 4;                                \
 810         lru_update(bkt30, match_pos30);                         \
 811                                                                 \
 812         if (match_key31 == 0)                                   \
 813                 match_pos31 = 4;                                \
 814         lru_update(bkt31, match_pos31);                         \
 815 }
 816
 817 /***
 818 * The lookup function implements a 4-stage pipeline, with each stage processing
 819 * two different packets. The purpose of pipelined implementation is to hide the
 820 * latency of prefetching the data structures and loosen the data dependency
 821 * between instructions.
 822 *
 823 *   p00  _______   p10  _______   p20  _______   p30  _______
 824 * ----->|       |----->|       |----->|       |----->|       |----->
 825 *       |   0   |      |   1   |      |   2   |      |   3   |
 826 * ----->|_______|----->|_______|----->|_______|----->|_______|----->
 827 *   p01            p11            p21            p31
 828 *
 829 * The naming convention is:
 830 *         pXY = packet Y of stage X, X = 0 .. 3, Y = 0 .. 1
 831 *
 832 ***/
 833 static int rte_table_hash_lru_lookup(
 834         void *table,
 835         struct rte_mbuf **pkts,
 836         uint64_t pkts_mask,
 837         uint64_t *lookup_hit_mask,
 838         void **entries)
 839 {
 840         struct rte_table_hash *t = (struct rte_table_hash *) table;
 841         struct grinder *g = t->grinders;
 842         uint64_t pkt00_index, pkt01_index, pkt10_index, pkt11_index;
 843         uint64_t pkt20_index, pkt21_index, pkt30_index, pkt31_index;
 844         uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0;
 845         int status = 0;
 846
 847         __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
 848         RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(t, n_pkts_in);
 849
 850         /* Cannot run the pipeline with less than 7 packets */
 851         if (__builtin_popcountll(pkts_mask) < 7)
 852                 return rte_table_hash_lru_lookup_unoptimized(table, pkts,
 853                         pkts_mask, lookup_hit_mask, entries);
 854
 855         /* Pipeline stage 0 */
 856         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
 857
 858         /* Pipeline feed */
 859         pkt10_index = pkt00_index;
 860         pkt11_index = pkt01_index;
 861
 862         /* Pipeline stage 0 */
 863         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
 864
 865         /* Pipeline stage 1 */
 866         lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);
 867
 868         /* Pipeline feed */
 869         pkt20_index = pkt10_index;
 870         pkt21_index = pkt11_index;
 871         pkt10_index = pkt00_index;
 872         pkt11_index = pkt01_index;
 873
 874         /* Pipeline stage 0 */
 875         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
 876
 877         /* Pipeline stage 1 */
 878         lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);
 879
 880         /* Pipeline stage 2 */
 881         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
 882
 883         /*
 884         * Pipeline run
 885         *
 886         */
 887         for ( ; pkts_mask; ) {
 888                 /* Pipeline feed */
 889                 pkt30_index = pkt20_index;
 890                 pkt31_index = pkt21_index;
 891                 pkt20_index = pkt10_index;
 892                 pkt21_index = pkt11_index;
 893                 pkt10_index = pkt00_index;
 894                 pkt11_index = pkt01_index;
 895
 896                 /* Pipeline stage 0 */
 897                 lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask,
 898                         pkt00_index, pkt01_index);
 899
 900                 /* Pipeline stage 1 */
 901                 lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);
 902
 903                 /* Pipeline stage 2 */
 904                 lookup2_stage2(t, g, pkt20_index, pkt21_index,
 905                         pkts_mask_match_many);
 906
 907                 /* Pipeline stage 3 */
 908                 lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index,
 909                         pkts_mask_out, entries);
 910         }
 911
 912         /* Pipeline feed */
 913         pkt30_index = pkt20_index;
 914         pkt31_index = pkt21_index;
 915         pkt20_index = pkt10_index;
 916         pkt21_index = pkt11_index;
 917         pkt10_index = pkt00_index;
 918         pkt11_index = pkt01_index;
 919
 920         /* Pipeline stage 1 */
 921         lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);
 922
 923         /* Pipeline stage 2 */
 924         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
 925
 926         /* Pipeline stage 3 */
 927         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
 928                 entries);
 929
 930         /* Pipeline feed */
 931         pkt30_index = pkt20_index;
 932         pkt31_index = pkt21_index;
 933         pkt20_index = pkt10_index;
 934         pkt21_index = pkt11_index;
 935
 936         /* Pipeline stage 2 */
 937         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
 938
 939         /* Pipeline stage 3 */
 940         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
 941                 entries);
 942
 943         /* Pipeline feed */
 944         pkt30_index = pkt20_index;
 945         pkt31_index = pkt21_index;
 946
 947         /* Pipeline stage 3 */
 948         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
 949                 entries);
 950
 951         /* Slow path */
 952         pkts_mask_match_many &= ~pkts_mask_out;
 953         if (pkts_mask_match_many) {
 954                 uint64_t pkts_mask_out_slow = 0;
 955
 956                 status = rte_table_hash_lru_lookup_unoptimized(table, pkts,
 957                         pkts_mask_match_many, &pkts_mask_out_slow, entries);
 958                 pkts_mask_out |= pkts_mask_out_slow;
 959         }
 960
 961         *lookup_hit_mask = pkts_mask_out;
 962         RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));
 963         return status;
 964 }
 965
 966 static int
 967 rte_table_hash_lru_stats_read(void *table, struct rte_table_stats *stats, int clear)
 968 {
 969         struct rte_table_hash *t = table;
 970
 971         if (stats != NULL)
 972                 memcpy(stats, &t->stats, sizeof(t->stats));
 973
 974         if (clear)
 975                 memset(&t->stats, 0, sizeof(t->stats));
 976
 977         return 0;
 978 }
 979
 980 struct rte_table_ops rte_table_hash_lru_ops = {
 981         .f_create = rte_table_hash_lru_create,
 982         .f_free = rte_table_hash_lru_free,
 983         .f_add = rte_table_hash_lru_entry_add,
 984         .f_delete = rte_table_hash_lru_entry_delete,
 985         .f_add_bulk = NULL,
 986         .f_delete_bulk = NULL,
 987         .f_lookup = rte_table_hash_lru_lookup,
 988         .f_stats = rte_table_hash_lru_stats_read,
 989 };