lib/librte_table/rte_table_hash_ext.c

   1 /*-
   2  *       BSD LICENSE
   3  *
   4  *       Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *       All rights reserved.
   6  *
   7  *       Redistribution and use in source and binary forms, with or without
   8  *       modification, are permitted provided that the following conditions
   9  *       are met:
  10  *
  11  *      * Redistributions of source code must retain the above copyright
  12  *               notice, this list of conditions and the following disclaimer.
  13  *      * Redistributions in binary form must reproduce the above copyright
  14  *               notice, this list of conditions and the following disclaimer in
  15  *               the documentation and/or other materials provided with the
  16  *               distribution.
  17  *      * Neither the name of Intel Corporation nor the names of its
  18  *               contributors may be used to endorse or promote products derived
  19  *               from this software without specific prior written permission.
  20  *
  21  *       THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *       "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *       LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *       A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *       OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *       SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *       LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *       DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *       THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *       (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *       OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_mbuf.h>
  39 #include <rte_memory.h>
  40 #include <rte_malloc.h>
  41 #include <rte_log.h>
  42
  43 #include "rte_table_hash.h"
  44
  45 #define KEYS_PER_BUCKET 4
  46
  47 struct bucket {
  48         union {
  49                 uintptr_t next;
  50                 uint64_t lru_list;
  51         };
  52         uint16_t sig[KEYS_PER_BUCKET];
  53         uint32_t key_pos[KEYS_PER_BUCKET];
  54 };
  55
  56 #define BUCKET_NEXT(bucket)                                             \
  57         ((void *) ((bucket)->next & (~1LU)))
  58
  59 #define BUCKET_NEXT_VALID(bucket)                                       \
  60         ((bucket)->next & 1LU)
  61
  62 #define BUCKET_NEXT_SET(bucket, bucket_next)                            \
  63 do                                                                      \
  64         (bucket)->next = (((uintptr_t) ((void *) (bucket_next))) | 1LU);\
  65 while (0)
  66
  67 #define BUCKET_NEXT_SET_NULL(bucket)                                    \
  68 do                                                                      \
  69         (bucket)->next = 0;                                             \
  70 while (0)
  71
  72 #define BUCKET_NEXT_COPY(bucket, bucket2)                               \
  73 do                                                                      \
  74         (bucket)->next = (bucket2)->next;                               \
  75 while (0)
  76
  77 struct grinder {
  78         struct bucket *bkt;
  79         uint64_t sig;
  80         uint64_t match;
  81         uint32_t key_index;
  82 };
  83
  84 struct rte_table_hash {
  85         /* Input parameters */
  86         uint32_t key_size;
  87         uint32_t entry_size;
  88         uint32_t n_keys;
  89         uint32_t n_buckets;
  90         uint32_t n_buckets_ext;
  91         rte_table_hash_op_hash f_hash;
  92         uint64_t seed;
  93         uint32_t signature_offset;
  94         uint32_t key_offset;
  95
  96         /* Internal */
  97         uint64_t bucket_mask;
  98         uint32_t key_size_shl;
  99         uint32_t data_size_shl;
 100         uint32_t key_stack_tos;
 101         uint32_t bkt_ext_stack_tos;
 102
 103         /* Grinder */
 104         struct grinder grinders[RTE_PORT_IN_BURST_SIZE_MAX];
 105
 106         /* Tables */
 107         struct bucket *buckets;
 108         struct bucket *buckets_ext;
 109         uint8_t *key_mem;
 110         uint8_t *data_mem;
 111         uint32_t *key_stack;
 112         uint32_t *bkt_ext_stack;
 113
 114         /* Table memory */
 115         uint8_t memory[0] __rte_cache_aligned;
 116 };
 117
 118 static int
 119 check_params_create(struct rte_table_hash_ext_params *params)
 120 {
 121         uint32_t n_buckets_min;
 122
 123         /* key_size */
 124         if ((params->key_size == 0) ||
 125                 (!rte_is_power_of_2(params->key_size))) {
 126                 RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__);
 127                 return -EINVAL;
 128         }
 129
 130         /* n_keys */
 131         if ((params->n_keys == 0) ||
 132                 (!rte_is_power_of_2(params->n_keys))) {
 133                 RTE_LOG(ERR, TABLE, "%s: n_keys invalid value\n", __func__);
 134                 return -EINVAL;
 135         }
 136
 137         /* n_buckets */
 138         n_buckets_min = (params->n_keys + KEYS_PER_BUCKET - 1) / params->n_keys;
 139         if ((params->n_buckets == 0) ||
 140                 (!rte_is_power_of_2(params->n_keys)) ||
 141                 (params->n_buckets < n_buckets_min)) {
 142                 RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__);
 143                 return -EINVAL;
 144         }
 145
 146         /* f_hash */
 147         if (params->f_hash == NULL) {
 148                 RTE_LOG(ERR, TABLE, "%s: f_hash invalid value\n", __func__);
 149                 return -EINVAL;
 150         }
 151
 152         return 0;
 153 }
 154
 155 static void *
 156 rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size)
 157 {
 158         struct rte_table_hash_ext_params *p =
 159                 (struct rte_table_hash_ext_params *) params;
 160         struct rte_table_hash *t;
 161         uint32_t total_size, table_meta_sz;
 162         uint32_t bucket_sz, bucket_ext_sz, key_sz;
 163         uint32_t key_stack_sz, bkt_ext_stack_sz, data_sz;
 164         uint32_t bucket_offset, bucket_ext_offset, key_offset;
 165         uint32_t key_stack_offset, bkt_ext_stack_offset, data_offset;
 166         uint32_t i;
 167
 168         /* Check input parameters */
 169         if ((check_params_create(p) != 0) ||
 170                 (!rte_is_power_of_2(entry_size)) ||
 171                 ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
 172                 (sizeof(struct bucket) != (RTE_CACHE_LINE_SIZE / 2)))
 173                 return NULL;
 174
 175         /* Memory allocation */
 176         table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash));
 177         bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket));
 178         bucket_ext_sz =
 179                 RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(struct bucket));
 180         key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size);
 181         key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t));
 182         bkt_ext_stack_sz =
 183                 RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(uint32_t));
 184         data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
 185         total_size = table_meta_sz + bucket_sz + bucket_ext_sz + key_sz +
 186                 key_stack_sz + bkt_ext_stack_sz + data_sz;
 187
 188         t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
 189         if (t == NULL) {
 190                 RTE_LOG(ERR, TABLE,
 191                         "%s: Cannot allocate %u bytes for hash table\n",
 192                         __func__, total_size);
 193                 return NULL;
 194         }
 195         RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table memory footprint is "
 196                 "%u bytes\n", __func__, p->key_size, total_size);
 197
 198         /* Memory initialization */
 199         t->key_size = p->key_size;
 200         t->entry_size = entry_size;
 201         t->n_keys = p->n_keys;
 202         t->n_buckets = p->n_buckets;
 203         t->n_buckets_ext = p->n_buckets_ext;
 204         t->f_hash = p->f_hash;
 205         t->seed = p->seed;
 206         t->signature_offset = p->signature_offset;
 207         t->key_offset = p->key_offset;
 208
 209         /* Internal */
 210         t->bucket_mask = t->n_buckets - 1;
 211         t->key_size_shl = __builtin_ctzl(p->key_size);
 212         t->data_size_shl = __builtin_ctzl(entry_size);
 213
 214         /* Tables */
 215         bucket_offset = 0;
 216         bucket_ext_offset = bucket_offset + bucket_sz;
 217         key_offset = bucket_ext_offset + bucket_ext_sz;
 218         key_stack_offset = key_offset + key_sz;
 219         bkt_ext_stack_offset = key_stack_offset + key_stack_sz;
 220         data_offset = bkt_ext_stack_offset + bkt_ext_stack_sz;
 221
 222         t->buckets = (struct bucket *) &t->memory[bucket_offset];
 223         t->buckets_ext = (struct bucket *) &t->memory[bucket_ext_offset];
 224         t->key_mem = &t->memory[key_offset];
 225         t->key_stack = (uint32_t *) &t->memory[key_stack_offset];
 226         t->bkt_ext_stack = (uint32_t *) &t->memory[bkt_ext_stack_offset];
 227         t->data_mem = &t->memory[data_offset];
 228
 229         /* Key stack */
 230         for (i = 0; i < t->n_keys; i++)
 231                 t->key_stack[i] = t->n_keys - 1 - i;
 232         t->key_stack_tos = t->n_keys;
 233
 234         /* Bucket ext stack */
 235         for (i = 0; i < t->n_buckets_ext; i++)
 236                 t->bkt_ext_stack[i] = t->n_buckets_ext - 1 - i;
 237         t->bkt_ext_stack_tos = t->n_buckets_ext;
 238
 239         return t;
 240 }
 241
 242 static int
 243 rte_table_hash_ext_free(void *table)
 244 {
 245         struct rte_table_hash *t = (struct rte_table_hash *) table;
 246
 247         /* Check input parameters */
 248         if (t == NULL)
 249                 return -EINVAL;
 250
 251         rte_free(t);
 252         return 0;
 253 }
 254
 255 static int
 256 rte_table_hash_ext_entry_add(void *table, void *key, void *entry,
 257         int *key_found, void **entry_ptr)
 258 {
 259         struct rte_table_hash *t = (struct rte_table_hash *) table;
 260         struct bucket *bkt0, *bkt, *bkt_prev;
 261         uint64_t sig;
 262         uint32_t bkt_index, i;
 263
 264         sig = t->f_hash(key, t->key_size, t->seed);
 265         bkt_index = sig & t->bucket_mask;
 266         bkt0 = &t->buckets[bkt_index];
 267         sig = (sig >> 16) | 1LLU;
 268
 269         /* Key is present in the bucket */
 270         for (bkt = bkt0; bkt != NULL; bkt = BUCKET_NEXT(bkt))
 271                 for (i = 0; i < KEYS_PER_BUCKET; i++) {
 272                         uint64_t bkt_sig = (uint64_t) bkt->sig[i];
 273                         uint32_t bkt_key_index = bkt->key_pos[i];
 274                         uint8_t *bkt_key =
 275                                 &t->key_mem[bkt_key_index << t->key_size_shl];
 276
 277                         if ((sig == bkt_sig) && (memcmp(key, bkt_key,
 278                                 t->key_size) == 0)) {
 279                                 uint8_t *data = &t->data_mem[bkt_key_index <<
 280                                         t->data_size_shl];
 281
 282                                 memcpy(data, entry, t->entry_size);
 283                                 *key_found = 1;
 284                                 *entry_ptr = (void *) data;
 285                                 return 0;
 286                         }
 287                 }
 288
 289         /* Key is not present in the bucket */
 290         for (bkt_prev = NULL, bkt = bkt0; bkt != NULL; bkt_prev = bkt,
 291                 bkt = BUCKET_NEXT(bkt))
 292                 for (i = 0; i < KEYS_PER_BUCKET; i++) {
 293                         uint64_t bkt_sig = (uint64_t) bkt->sig[i];
 294
 295                         if (bkt_sig == 0) {
 296                                 uint32_t bkt_key_index;
 297                                 uint8_t *bkt_key, *data;
 298
 299                                 /* Allocate new key */
 300                                 if (t->key_stack_tos == 0) /* No free keys */
 301                                         return -ENOSPC;
 302
 303                                 bkt_key_index = t->key_stack[
 304                                         --t->key_stack_tos];
 305
 306                                 /* Install new key */
 307                                 bkt_key = &t->key_mem[bkt_key_index <<
 308                                         t->key_size_shl];
 309                                 data = &t->data_mem[bkt_key_index <<
 310                                         t->data_size_shl];
 311
 312                                 bkt->sig[i] = (uint16_t) sig;
 313                                 bkt->key_pos[i] = bkt_key_index;
 314                                 memcpy(bkt_key, key, t->key_size);
 315                                 memcpy(data, entry, t->entry_size);
 316
 317                                 *key_found = 0;
 318                                 *entry_ptr = (void *) data;
 319                                 return 0;
 320                         }
 321                 }
 322
 323         /* Bucket full: extend bucket */
 324         if ((t->bkt_ext_stack_tos > 0) && (t->key_stack_tos > 0)) {
 325                 uint32_t bkt_key_index;
 326                 uint8_t *bkt_key, *data;
 327
 328                 /* Allocate new bucket ext */
 329                 bkt_index = t->bkt_ext_stack[--t->bkt_ext_stack_tos];
 330                 bkt = &t->buckets_ext[bkt_index];
 331
 332                 /* Chain the new bucket ext */
 333                 BUCKET_NEXT_SET(bkt_prev, bkt);
 334                 BUCKET_NEXT_SET_NULL(bkt);
 335
 336                 /* Allocate new key */
 337                 bkt_key_index = t->key_stack[--t->key_stack_tos];
 338                 bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl];
 339
 340                 data = &t->data_mem[bkt_key_index << t->data_size_shl];
 341
 342                 /* Install new key into bucket */
 343                 bkt->sig[0] = (uint16_t) sig;
 344                 bkt->key_pos[0] = bkt_key_index;
 345                 memcpy(bkt_key, key, t->key_size);
 346                 memcpy(data, entry, t->entry_size);
 347
 348                 *key_found = 0;
 349                 *entry_ptr = (void *) data;
 350                 return 0;
 351         }
 352
 353         return -ENOSPC;
 354 }
 355
 356 static int
 357 rte_table_hash_ext_entry_delete(void *table, void *key, int *key_found,
 358 void *entry)
 359 {
 360         struct rte_table_hash *t = (struct rte_table_hash *) table;
 361         struct bucket *bkt0, *bkt, *bkt_prev;
 362         uint64_t sig;
 363         uint32_t bkt_index, i;
 364
 365         sig = t->f_hash(key, t->key_size, t->seed);
 366         bkt_index = sig & t->bucket_mask;
 367         bkt0 = &t->buckets[bkt_index];
 368         sig = (sig >> 16) | 1LLU;
 369
 370         /* Key is present in the bucket */
 371         for (bkt_prev = NULL, bkt = bkt0; bkt != NULL; bkt_prev = bkt,
 372                 bkt = BUCKET_NEXT(bkt))
 373                 for (i = 0; i < KEYS_PER_BUCKET; i++) {
 374                         uint64_t bkt_sig = (uint64_t) bkt->sig[i];
 375                         uint32_t bkt_key_index = bkt->key_pos[i];
 376                         uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
 377                                 t->key_size_shl];
 378
 379                         if ((sig == bkt_sig) && (memcmp(key, bkt_key,
 380                                 t->key_size) == 0)) {
 381                                 uint8_t *data = &t->data_mem[bkt_key_index <<
 382                                         t->data_size_shl];
 383
 384                                 /* Uninstall key from bucket */
 385                                 bkt->sig[i] = 0;
 386                                 *key_found = 1;
 387                                 if (entry)
 388                                         memcpy(entry, data, t->entry_size);
 389
 390                                 /* Free key */
 391                                 t->key_stack[t->key_stack_tos++] =
 392                                         bkt_key_index;
 393
 394                                 /*Check if bucket is unused */
 395                                 if ((bkt_prev != NULL) &&
 396                                     (bkt->sig[0] == 0) && (bkt->sig[1] == 0) &&
 397                                     (bkt->sig[2] == 0) && (bkt->sig[3] == 0)) {
 398                                         /* Unchain bucket */
 399                                         BUCKET_NEXT_COPY(bkt_prev, bkt);
 400
 401                                         /* Clear bucket */
 402                                         memset(bkt, 0, sizeof(struct bucket));
 403
 404                                         /* Free bucket back to buckets ext */
 405                                         bkt_index = bkt - t->buckets_ext;
 406                                         t->bkt_ext_stack[t->bkt_ext_stack_tos++]
 407                                                 = bkt_index;
 408                                 }
 409
 410                                 return 0;
 411                         }
 412                 }
 413
 414         /* Key is not present in the bucket */
 415         *key_found = 0;
 416         return 0;
 417 }
 418
 419 static int rte_table_hash_ext_lookup_unoptimized(
 420         void *table,
 421         struct rte_mbuf **pkts,
 422         uint64_t pkts_mask,
 423         uint64_t *lookup_hit_mask,
 424         void **entries,
 425         int dosig)
 426 {
 427         struct rte_table_hash *t = (struct rte_table_hash *) table;
 428         uint64_t pkts_mask_out = 0;
 429
 430         for ( ; pkts_mask; ) {
 431                 struct bucket *bkt0, *bkt;
 432                 struct rte_mbuf *pkt;
 433                 uint8_t *key;
 434                 uint64_t pkt_mask, sig;
 435                 uint32_t pkt_index, bkt_index, i;
 436
 437                 pkt_index = __builtin_ctzll(pkts_mask);
 438                 pkt_mask = 1LLU << pkt_index;
 439                 pkts_mask &= ~pkt_mask;
 440
 441                 pkt = pkts[pkt_index];
 442                 key = RTE_MBUF_METADATA_UINT8_PTR(pkt, t->key_offset);
 443                 if (dosig)
 444                         sig = (uint64_t) t->f_hash(key, t->key_size, t->seed);
 445                 else
 446                         sig = RTE_MBUF_METADATA_UINT32(pkt,
 447                                 t->signature_offset);
 448
 449                 bkt_index = sig & t->bucket_mask;
 450                 bkt0 = &t->buckets[bkt_index];
 451                 sig = (sig >> 16) | 1LLU;
 452
 453                 /* Key is present in the bucket */
 454                 for (bkt = bkt0; bkt != NULL; bkt = BUCKET_NEXT(bkt))
 455                         for (i = 0; i < KEYS_PER_BUCKET; i++) {
 456                                 uint64_t bkt_sig = (uint64_t) bkt->sig[i];
 457                                 uint32_t bkt_key_index = bkt->key_pos[i];
 458                                 uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
 459                                         t->key_size_shl];
 460
 461                                 if ((sig == bkt_sig) && (memcmp(key, bkt_key,
 462                                         t->key_size) == 0)) {
 463                                         uint8_t *data = &t->data_mem[
 464                                         bkt_key_index << t->data_size_shl];
 465
 466                                         pkts_mask_out |= pkt_mask;
 467                                         entries[pkt_index] = (void *) data;
 468                                         break;
 469                                 }
 470                         }
 471         }
 472
 473         *lookup_hit_mask = pkts_mask_out;
 474         return 0;
 475 }
 476
 477 /***
 478  *
 479  * mask = match bitmask
 480  * match = at least one match
 481  * match_many = more than one match
 482  * match_pos = position of first match
 483  *
 484  *----------------------------------------
 485  * mask          match   match_many       match_pos
 486  *----------------------------------------
 487  * 0000          0               0                        00
 488  * 0001          1               0                        00
 489  * 0010          1               0                        01
 490  * 0011          1               1                        00
 491  *----------------------------------------
 492  * 0100          1               0                        10
 493  * 0101          1               1                        00
 494  * 0110          1               1                        01
 495  * 0111          1               1                        00
 496  *----------------------------------------
 497  * 1000          1               0                        11
 498  * 1001          1               1                        00
 499  * 1010          1               1                        01
 500  * 1011          1               1                        00
 501  *----------------------------------------
 502  * 1100          1               1                        10
 503  * 1101          1               1                        00
 504  * 1110          1               1                        01
 505  * 1111          1               1                        00
 506  *----------------------------------------
 507  *
 508  * match = 1111_1111_1111_1110
 509  * match_many = 1111_1110_1110_1000
 510  * match_pos = 0001_0010_0001_0011__0001_0010_0001_0000
 511  *
 512  * match = 0xFFFELLU
 513  * match_many = 0xFEE8LLU
 514  * match_pos = 0x12131210LLU
 515  *
 516  ***/
 517
 518 #define LUT_MATCH                                               0xFFFELLU
 519 #define LUT_MATCH_MANY                                          0xFEE8LLU
 520 #define LUT_MATCH_POS                                           0x12131210LLU
 521
 522 #define lookup_cmp_sig(mbuf_sig, bucket, match, match_many, match_pos)  \
 523 {                                                                       \
 524         uint64_t bucket_sig[4], mask[4], mask_all;                      \
 525                                                                         \
 526         bucket_sig[0] = bucket->sig[0];                                 \
 527         bucket_sig[1] = bucket->sig[1];                                 \
 528         bucket_sig[2] = bucket->sig[2];                                 \
 529         bucket_sig[3] = bucket->sig[3];                                 \
 530                                                                         \
 531         bucket_sig[0] ^= mbuf_sig;                                      \
 532         bucket_sig[1] ^= mbuf_sig;                                      \
 533         bucket_sig[2] ^= mbuf_sig;                                      \
 534         bucket_sig[3] ^= mbuf_sig;                                      \
 535                                                                         \
 536         mask[0] = 0;                                                    \
 537         mask[1] = 0;                                                    \
 538         mask[2] = 0;                                                    \
 539         mask[3] = 0;                                                    \
 540                                                                         \
 541         if (bucket_sig[0] == 0)                                         \
 542                 mask[0] = 1;                                            \
 543         if (bucket_sig[1] == 0)                                         \
 544                 mask[1] = 2;                                            \
 545         if (bucket_sig[2] == 0)                                         \
 546                 mask[2] = 4;                                            \
 547         if (bucket_sig[3] == 0)                                         \
 548                 mask[3] = 8;                                            \
 549                                                                         \
 550         mask_all = (mask[0] | mask[1]) | (mask[2] | mask[3]);           \
 551                                                                         \
 552         match = (LUT_MATCH >> mask_all) & 1;                            \
 553         match_many = (LUT_MATCH_MANY >> mask_all) & 1;                  \
 554         match_pos = (LUT_MATCH_POS >> (mask_all << 1)) & 3;             \
 555 }
 556
 557 #define lookup_cmp_key(mbuf, key, match_key, f)                         \
 558 {                                                                       \
 559         uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(mbuf, f->key_offset);\
 560         uint64_t *bkt_key = (uint64_t *) key;                           \
 561                                                                         \
 562         switch (f->key_size) {                                          \
 563         case 8:                                                         \
 564         {                                                               \
 565                 uint64_t xor = pkt_key[0] ^ bkt_key[0];                 \
 566                 match_key = 0;                                          \
 567                 if (xor == 0)                                           \
 568                         match_key = 1;                                  \
 569         }                                                               \
 570         break;                                                          \
 571                                                                         \
 572         case 16:                                                        \
 573         {                                                               \
 574                 uint64_t xor[2], or;                                    \
 575                                                                         \
 576                 xor[0] = pkt_key[0] ^ bkt_key[0];                       \
 577                 xor[1] = pkt_key[1] ^ bkt_key[1];                       \
 578                 or = xor[0] | xor[1];                                   \
 579                 match_key = 0;                                          \
 580                 if (or == 0)                                            \
 581                         match_key = 1;                                  \
 582         }                                                               \
 583         break;                                                          \
 584                                                                         \
 585         case 32:                                                        \
 586         {                                                               \
 587                 uint64_t xor[4], or;                                    \
 588                                                                         \
 589                 xor[0] = pkt_key[0] ^ bkt_key[0];                       \
 590                 xor[1] = pkt_key[1] ^ bkt_key[1];                       \
 591                 xor[2] = pkt_key[2] ^ bkt_key[2];                       \
 592                 xor[3] = pkt_key[3] ^ bkt_key[3];                       \
 593                 or = xor[0] | xor[1] | xor[2] | xor[3];                 \
 594                 match_key = 0;                                          \
 595                 if (or == 0)                                            \
 596                         match_key = 1;                                  \
 597         }                                                               \
 598         break;                                                          \
 599                                                                         \
 600         case 64:                                                        \
 601         {                                                               \
 602                 uint64_t xor[8], or;                                    \
 603                                                                         \
 604                 xor[0] = pkt_key[0] ^ bkt_key[0];                       \
 605                 xor[1] = pkt_key[1] ^ bkt_key[1];                       \
 606                 xor[2] = pkt_key[2] ^ bkt_key[2];                       \
 607                 xor[3] = pkt_key[3] ^ bkt_key[3];                       \
 608                 xor[4] = pkt_key[4] ^ bkt_key[4];                       \
 609                 xor[5] = pkt_key[5] ^ bkt_key[5];                       \
 610                 xor[6] = pkt_key[6] ^ bkt_key[6];                       \
 611                 xor[7] = pkt_key[7] ^ bkt_key[7];                       \
 612                 or = xor[0] | xor[1] | xor[2] | xor[3] |                \
 613                         xor[4] | xor[5] | xor[6] | xor[7];              \
 614                 match_key = 0;                                          \
 615                 if (or == 0)                                            \
 616                         match_key = 1;                                  \
 617         }                                                               \
 618         break;                                                          \
 619                                                                         \
 620         default:                                                        \
 621                 match_key = 0;                                          \
 622                 if (memcmp(pkt_key, bkt_key, f->key_size) == 0)         \
 623                         match_key = 1;                                  \
 624         }                                                               \
 625 }
 626
 627 #define lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index) \
 628 {                                                                       \
 629         uint64_t pkt00_mask, pkt01_mask;                                \
 630         struct rte_mbuf *mbuf00, *mbuf01;                               \
 631                                                                         \
 632         pkt00_index = __builtin_ctzll(pkts_mask);                       \
 633         pkt00_mask = 1LLU << pkt00_index;                               \
 634         pkts_mask &= ~pkt00_mask;                                       \
 635         mbuf00 = pkts[pkt00_index];                                     \
 636                                                                         \
 637         pkt01_index = __builtin_ctzll(pkts_mask);                       \
 638         pkt01_mask = 1LLU << pkt01_index;                               \
 639         pkts_mask &= ~pkt01_mask;                                       \
 640         mbuf01 = pkts[pkt01_index];                                     \
 641                                                                         \
 642         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0));          \
 643         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0));          \
 644 }
 645
 646 #define lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask, pkt00_index, \
 647         pkt01_index)                                                    \
 648 {                                                                       \
 649         uint64_t pkt00_mask, pkt01_mask;                                \
 650         struct rte_mbuf *mbuf00, *mbuf01;                               \
 651                                                                         \
 652         pkt00_index = __builtin_ctzll(pkts_mask);                       \
 653         pkt00_mask = 1LLU << pkt00_index;                               \
 654         pkts_mask &= ~pkt00_mask;                                       \
 655         mbuf00 = pkts[pkt00_index];                                     \
 656                                                                         \
 657         pkt01_index = __builtin_ctzll(pkts_mask);                       \
 658         if (pkts_mask == 0)                                             \
 659                 pkt01_index = pkt00_index;                              \
 660         pkt01_mask = 1LLU << pkt01_index;                               \
 661         pkts_mask &= ~pkt01_mask;                                       \
 662         mbuf01 = pkts[pkt01_index];                                     \
 663                                                                         \
 664         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0));          \
 665         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0));          \
 666 }
 667
 668 #define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index)            \
 669 {                                                                       \
 670         struct grinder *g10, *g11;                                      \
 671         uint64_t sig10, sig11, bkt10_index, bkt11_index;                \
 672         struct rte_mbuf *mbuf10, *mbuf11;                               \
 673         struct bucket *bkt10, *bkt11, *buckets = t->buckets;            \
 674         uint64_t bucket_mask = t->bucket_mask;                          \
 675         uint32_t signature_offset = t->signature_offset;                \
 676                                                                         \
 677         mbuf10 = pkts[pkt10_index];                                     \
 678         sig10 = (uint64_t) RTE_MBUF_METADATA_UINT32(mbuf10, signature_offset);\
 679         bkt10_index = sig10 & bucket_mask;                              \
 680         bkt10 = &buckets[bkt10_index];                                  \
 681                                                                         \
 682         mbuf11 = pkts[pkt11_index];                                     \
 683         sig11 = (uint64_t) RTE_MBUF_METADATA_UINT32(mbuf11, signature_offset);\
 684         bkt11_index = sig11 & bucket_mask;                              \
 685         bkt11 = &buckets[bkt11_index];                                  \
 686                                                                         \
 687         rte_prefetch0(bkt10);                                           \
 688         rte_prefetch0(bkt11);                                           \
 689                                                                         \
 690         g10 = &g[pkt10_index];                                          \
 691         g10->sig = sig10;                                               \
 692         g10->bkt = bkt10;                                               \
 693                                                                         \
 694         g11 = &g[pkt11_index];                                          \
 695         g11->sig = sig11;                                               \
 696         g11->bkt = bkt11;                                               \
 697 }
 698
 699 #define lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index)      \
 700 {                                                                       \
 701         struct grinder *g10, *g11;                                      \
 702         uint64_t sig10, sig11, bkt10_index, bkt11_index;                \
 703         struct rte_mbuf *mbuf10, *mbuf11;                               \
 704         struct bucket *bkt10, *bkt11, *buckets = t->buckets;            \
 705         uint8_t *key10, *key11;                                         \
 706         uint64_t bucket_mask = t->bucket_mask;                          \
 707         rte_table_hash_op_hash f_hash = t->f_hash;                      \
 708         uint64_t seed = t->seed;                                        \
 709         uint32_t key_size = t->key_size;                                \
 710         uint32_t key_offset = t->key_offset;                            \
 711                                                                         \
 712         mbuf10 = pkts[pkt10_index];                                     \
 713         key10 = RTE_MBUF_METADATA_UINT8_PTR(mbuf10, key_offset);        \
 714         sig10 = (uint64_t) f_hash(key10, key_size, seed);               \
 715         bkt10_index = sig10 & bucket_mask;                              \
 716         bkt10 = &buckets[bkt10_index];                                  \
 717                                                                         \
 718         mbuf11 = pkts[pkt11_index];                                     \
 719         key11 = RTE_MBUF_METADATA_UINT8_PTR(mbuf11, key_offset);        \
 720         sig11 = (uint64_t) f_hash(key11, key_size, seed);               \
 721         bkt11_index = sig11 & bucket_mask;                              \
 722         bkt11 = &buckets[bkt11_index];                                  \
 723                                                                         \
 724         rte_prefetch0(bkt10);                                           \
 725         rte_prefetch0(bkt11);                                           \
 726                                                                         \
 727         g10 = &g[pkt10_index];                                          \
 728         g10->sig = sig10;                                               \
 729         g10->bkt = bkt10;                                               \
 730                                                                         \
 731         g11 = &g[pkt11_index];                                          \
 732         g11->sig = sig11;                                               \
 733         g11->bkt = bkt11;                                               \
 734 }
 735
 736 #define lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many)\
 737 {                                                                       \
 738         struct grinder *g20, *g21;                                      \
 739         uint64_t sig20, sig21;                                          \
 740         struct bucket *bkt20, *bkt21;                                   \
 741         uint8_t *key20, *key21, *key_mem = t->key_mem;                  \
 742         uint64_t match20, match21, match_many20, match_many21;          \
 743         uint64_t match_pos20, match_pos21;                              \
 744         uint32_t key20_index, key21_index, key_size_shl = t->key_size_shl;\
 745                                                                         \
 746         g20 = &g[pkt20_index];                                          \
 747         sig20 = g20->sig;                                               \
 748         bkt20 = g20->bkt;                                               \
 749         sig20 = (sig20 >> 16) | 1LLU;                                   \
 750         lookup_cmp_sig(sig20, bkt20, match20, match_many20, match_pos20);\
 751         match20 <<= pkt20_index;                                        \
 752         match_many20 |= BUCKET_NEXT_VALID(bkt20);                       \
 753         match_many20 <<= pkt20_index;                                   \
 754         key20_index = bkt20->key_pos[match_pos20];                      \
 755         key20 = &key_mem[key20_index << key_size_shl];                  \
 756                                                                         \
 757         g21 = &g[pkt21_index];                                          \
 758         sig21 = g21->sig;                                               \
 759         bkt21 = g21->bkt;                                               \
 760         sig21 = (sig21 >> 16) | 1LLU;                                   \
 761         lookup_cmp_sig(sig21, bkt21, match21, match_many21, match_pos21);\
 762         match21 <<= pkt21_index;                                        \
 763         match_many21 |= BUCKET_NEXT_VALID(bkt21);                       \
 764         match_many21 <<= pkt21_index;                                   \
 765         key21_index = bkt21->key_pos[match_pos21];                      \
 766         key21 = &key_mem[key21_index << key_size_shl];                  \
 767                                                                         \
 768         rte_prefetch0(key20);                                           \
 769         rte_prefetch0(key21);                                           \
 770                                                                         \
 771         pkts_mask_match_many |= match_many20 | match_many21;            \
 772                                                                         \
 773         g20->match = match20;                                           \
 774         g20->key_index = key20_index;                                   \
 775                                                                         \
 776         g21->match = match21;                                           \
 777         g21->key_index = key21_index;                                   \
 778 }
 779
 780 #define lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, \
 781         entries)                                                        \
 782 {                                                                       \
 783         struct grinder *g30, *g31;                                      \
 784         struct rte_mbuf *mbuf30, *mbuf31;                               \
 785         uint8_t *key30, *key31, *key_mem = t->key_mem;                  \
 786         uint8_t *data30, *data31, *data_mem = t->data_mem;              \
 787         uint64_t match30, match31, match_key30, match_key31, match_keys;\
 788         uint32_t key30_index, key31_index;                              \
 789         uint32_t key_size_shl = t->key_size_shl;                        \
 790         uint32_t data_size_shl = t->data_size_shl;                      \
 791                                                                         \
 792         mbuf30 = pkts[pkt30_index];                                     \
 793         g30 = &g[pkt30_index];                                          \
 794         match30 = g30->match;                                           \
 795         key30_index = g30->key_index;                                   \
 796         key30 = &key_mem[key30_index << key_size_shl];                  \
 797         lookup_cmp_key(mbuf30, key30, match_key30, t);                  \
 798         match_key30 <<= pkt30_index;                                    \
 799         match_key30 &= match30;                                         \
 800         data30 = &data_mem[key30_index << data_size_shl];               \
 801         entries[pkt30_index] = data30;                                  \
 802                                                                         \
 803         mbuf31 = pkts[pkt31_index];                                     \
 804         g31 = &g[pkt31_index];                                          \
 805         match31 = g31->match;                                           \
 806         key31_index = g31->key_index;                                   \
 807         key31 = &key_mem[key31_index << key_size_shl];                  \
 808         lookup_cmp_key(mbuf31, key31, match_key31, t);                  \
 809         match_key31 <<= pkt31_index;                                    \
 810         match_key31 &= match31;                                         \
 811         data31 = &data_mem[key31_index << data_size_shl];               \
 812         entries[pkt31_index] = data31;                                  \
 813                                                                         \
 814         rte_prefetch0(data30);                                          \
 815         rte_prefetch0(data31);                                          \
 816                                                                         \
 817         match_keys = match_key30 | match_key31;                         \
 818         pkts_mask_out |= match_keys;                                    \
 819 }
 820
 821 /***
 822 * The lookup function implements a 4-stage pipeline, with each stage processing
 823 * two different packets. The purpose of pipelined implementation is to hide the
 824 * latency of prefetching the data structures and loosen the data dependency
 825 * between instructions.
 826 *
 827 *  p00  _______   p10  _______   p20  _______   p30  _______
 828 *----->|       |----->|       |----->|       |----->|       |----->
 829 *      |   0   |      |   1   |      |   2   |      |   3   |
 830 *----->|_______|----->|_______|----->|_______|----->|_______|----->
 831 *  p01            p11            p21            p31
 832 *
 833 * The naming convention is:
 834 *    pXY = packet Y of stage X, X = 0 .. 3, Y = 0 .. 1
 835 *
 836 ***/
 837 static int rte_table_hash_ext_lookup(
 838         void *table,
 839         struct rte_mbuf **pkts,
 840         uint64_t pkts_mask,
 841         uint64_t *lookup_hit_mask,
 842         void **entries)
 843 {
 844         struct rte_table_hash *t = (struct rte_table_hash *) table;
 845         struct grinder *g = t->grinders;
 846         uint64_t pkt00_index, pkt01_index, pkt10_index, pkt11_index;
 847         uint64_t pkt20_index, pkt21_index, pkt30_index, pkt31_index;
 848         uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0;
 849         int status = 0;
 850
 851         /* Cannot run the pipeline with less than 7 packets */
 852         if (__builtin_popcountll(pkts_mask) < 7)
 853                 return rte_table_hash_ext_lookup_unoptimized(table, pkts,
 854                         pkts_mask, lookup_hit_mask, entries, 0);
 855
 856         /* Pipeline stage 0 */
 857         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
 858
 859         /* Pipeline feed */
 860         pkt10_index = pkt00_index;
 861         pkt11_index = pkt01_index;
 862
 863         /* Pipeline stage 0 */
 864         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
 865
 866         /* Pipeline stage 1 */
 867         lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);
 868
 869         /* Pipeline feed */
 870         pkt20_index = pkt10_index;
 871         pkt21_index = pkt11_index;
 872         pkt10_index = pkt00_index;
 873         pkt11_index = pkt01_index;
 874
 875         /* Pipeline stage 0 */
 876         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
 877
 878         /* Pipeline stage 1 */
 879         lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);
 880
 881         /* Pipeline stage 2 */
 882         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
 883
 884         /*
 885         * Pipeline run
 886         *
 887         */
 888         for ( ; pkts_mask; ) {
 889                 /* Pipeline feed */
 890                 pkt30_index = pkt20_index;
 891                 pkt31_index = pkt21_index;
 892                 pkt20_index = pkt10_index;
 893                 pkt21_index = pkt11_index;
 894                 pkt10_index = pkt00_index;
 895                 pkt11_index = pkt01_index;
 896
 897                 /* Pipeline stage 0 */
 898                 lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask,
 899                         pkt00_index, pkt01_index);
 900
 901                 /* Pipeline stage 1 */
 902                 lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);
 903
 904                 /* Pipeline stage 2 */
 905                 lookup2_stage2(t, g, pkt20_index, pkt21_index,
 906                         pkts_mask_match_many);
 907
 908                 /* Pipeline stage 3 */
 909                 lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index,
 910                         pkts_mask_out, entries);
 911         }
 912
 913         /* Pipeline feed */
 914         pkt30_index = pkt20_index;
 915         pkt31_index = pkt21_index;
 916         pkt20_index = pkt10_index;
 917         pkt21_index = pkt11_index;
 918         pkt10_index = pkt00_index;
 919         pkt11_index = pkt01_index;
 920
 921         /* Pipeline stage 1 */
 922         lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index);
 923
 924         /* Pipeline stage 2 */
 925         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
 926
 927         /* Pipeline stage 3 */
 928         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
 929                 entries);
 930
 931         /* Pipeline feed */
 932         pkt30_index = pkt20_index;
 933         pkt31_index = pkt21_index;
 934         pkt20_index = pkt10_index;
 935         pkt21_index = pkt11_index;
 936
 937         /* Pipeline stage 2 */
 938         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
 939
 940         /* Pipeline stage 3 */
 941         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
 942                 entries);
 943
 944         /* Pipeline feed */
 945         pkt30_index = pkt20_index;
 946         pkt31_index = pkt21_index;
 947
 948         /* Pipeline stage 3 */
 949         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
 950                 entries);
 951
 952         /* Slow path */
 953         pkts_mask_match_many &= ~pkts_mask_out;
 954         if (pkts_mask_match_many) {
 955                 uint64_t pkts_mask_out_slow = 0;
 956
 957                 status = rte_table_hash_ext_lookup_unoptimized(table, pkts,
 958                         pkts_mask_match_many, &pkts_mask_out_slow, entries, 0);
 959                 pkts_mask_out |= pkts_mask_out_slow;
 960         }
 961
 962         *lookup_hit_mask = pkts_mask_out;
 963         return status;
 964 }
 965
 966 static int rte_table_hash_ext_lookup_dosig(
 967         void *table,
 968         struct rte_mbuf **pkts,
 969         uint64_t pkts_mask,
 970         uint64_t *lookup_hit_mask,
 971         void **entries)
 972 {
 973         struct rte_table_hash *t = (struct rte_table_hash *) table;
 974         struct grinder *g = t->grinders;
 975         uint64_t pkt00_index, pkt01_index, pkt10_index, pkt11_index;
 976         uint64_t pkt20_index, pkt21_index, pkt30_index, pkt31_index;
 977         uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0;
 978         int status = 0;
 979
 980         /* Cannot run the pipeline with less than 7 packets */
 981         if (__builtin_popcountll(pkts_mask) < 7)
 982                 return rte_table_hash_ext_lookup_unoptimized(table, pkts,
 983                         pkts_mask, lookup_hit_mask, entries, 1);
 984
 985         /* Pipeline stage 0 */
 986         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
 987
 988         /* Pipeline feed */
 989         pkt10_index = pkt00_index;
 990         pkt11_index = pkt01_index;
 991
 992         /* Pipeline stage 0 */
 993         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
 994
 995         /* Pipeline stage 1 */
 996         lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index);
 997
 998         /* Pipeline feed */
 999         pkt20_index = pkt10_index;
1000         pkt21_index = pkt11_index;
1001         pkt10_index = pkt00_index;
1002         pkt11_index = pkt01_index;
1003
1004         /* Pipeline stage 0 */
1005         lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
1006
1007         /* Pipeline stage 1 */
1008         lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index);
1009
1010         /* Pipeline stage 2 */
1011         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
1012
1013         /*
1014         * Pipeline run
1015         *
1016         */
1017         for ( ; pkts_mask; ) {
1018                 /* Pipeline feed */
1019                 pkt30_index = pkt20_index;
1020                 pkt31_index = pkt21_index;
1021                 pkt20_index = pkt10_index;
1022                 pkt21_index = pkt11_index;
1023                 pkt10_index = pkt00_index;
1024                 pkt11_index = pkt01_index;
1025
1026                 /* Pipeline stage 0 */
1027                 lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask,
1028                         pkt00_index, pkt01_index);
1029
1030                 /* Pipeline stage 1 */
1031                 lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index);
1032
1033                 /* Pipeline stage 2 */
1034                 lookup2_stage2(t, g, pkt20_index, pkt21_index,
1035                         pkts_mask_match_many);
1036
1037                 /* Pipeline stage 3 */
1038                 lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index,
1039                         pkts_mask_out, entries);
1040         }
1041
1042         /* Pipeline feed */
1043         pkt30_index = pkt20_index;
1044         pkt31_index = pkt21_index;
1045         pkt20_index = pkt10_index;
1046         pkt21_index = pkt11_index;
1047         pkt10_index = pkt00_index;
1048         pkt11_index = pkt01_index;
1049
1050         /* Pipeline stage 1 */
1051         lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index);
1052
1053         /* Pipeline stage 2 */
1054         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
1055
1056         /* Pipeline stage 3 */
1057         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
1058                 entries);
1059
1060         /* Pipeline feed */
1061         pkt30_index = pkt20_index;
1062         pkt31_index = pkt21_index;
1063         pkt20_index = pkt10_index;
1064         pkt21_index = pkt11_index;
1065
1066         /* Pipeline stage 2 */
1067         lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
1068
1069         /* Pipeline stage 3 */
1070         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
1071                 entries);
1072
1073         /* Pipeline feed */
1074         pkt30_index = pkt20_index;
1075         pkt31_index = pkt21_index;
1076
1077         /* Pipeline stage 3 */
1078         lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
1079                 entries);
1080
1081         /* Slow path */
1082         pkts_mask_match_many &= ~pkts_mask_out;
1083         if (pkts_mask_match_many) {
1084                 uint64_t pkts_mask_out_slow = 0;
1085
1086                 status = rte_table_hash_ext_lookup_unoptimized(table, pkts,
1087                         pkts_mask_match_many, &pkts_mask_out_slow, entries, 1);
1088                 pkts_mask_out |= pkts_mask_out_slow;
1089         }
1090
1091         *lookup_hit_mask = pkts_mask_out;
1092         return status;
1093 }
1094
1095 struct rte_table_ops rte_table_hash_ext_ops      = {
1096         .f_create = rte_table_hash_ext_create,
1097         .f_free = rte_table_hash_ext_free,
1098         .f_add = rte_table_hash_ext_entry_add,
1099         .f_delete = rte_table_hash_ext_entry_delete,
1100         .f_lookup = rte_table_hash_ext_lookup,
1101 };
1102
1103 struct rte_table_ops rte_table_hash_ext_dosig_ops  = {
1104         .f_create = rte_table_hash_ext_create,
1105         .f_free = rte_table_hash_ext_free,
1106         .f_add = rte_table_hash_ext_entry_add,
1107         .f_delete = rte_table_hash_ext_entry_delete,
1108         .f_lookup = rte_table_hash_ext_lookup_dosig,
1109 };