lib/librte_table/rte_table_hash_key16.c

   1 /*-
   2  *       BSD LICENSE
   3  *
   4  *       Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *       All rights reserved.
   6  *
   7  *       Redistribution and use in source and binary forms, with or without
   8  *       modification, are permitted provided that the following conditions
   9  *       are met:
  10  *
  11  *      * Redistributions of source code must retain the above copyright
  12  *               notice, this list of conditions and the following disclaimer.
  13  *      * Redistributions in binary form must reproduce the above copyright
  14  *               notice, this list of conditions and the following disclaimer in
  15  *               the documentation and/or other materials provided with the
  16  *               distribution.
  17  *      * Neither the name of Intel Corporation nor the names of its
  18  *               contributors may be used to endorse or promote products derived
  19  *               from this software without specific prior written permission.
  20  *
  21  *       THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *       "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *       LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *       A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *       OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *       SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *       LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *       DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *       THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *       (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *       OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 #include <string.h>
  34 #include <stdio.h>
  35
  36 #include <rte_common.h>
  37 #include <rte_mbuf.h>
  38 #include <rte_memory.h>
  39 #include <rte_malloc.h>
  40 #include <rte_log.h>
  41
  42 #include "rte_table_hash.h"
  43 #include "rte_lru.h"
  44
  45 #define RTE_TABLE_HASH_KEY_SIZE                                         16
  46
  47 #define RTE_BUCKET_ENTRY_VALID                                          0x1LLU
  48
  49 #ifdef RTE_TABLE_STATS_COLLECT
  50
  51 #define RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(table, val) \
  52         table->stats.n_pkts_in += val
  53 #define RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(table, val) \
  54         table->stats.n_pkts_lookup_miss += val
  55
  56 #else
  57
  58 #define RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(table, val)
  59 #define RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(table, val)
  60
  61 #endif
  62
  63 struct rte_bucket_4_16 {
  64         /* Cache line 0 */
  65         uint64_t signature[4 + 1];
  66         uint64_t lru_list;
  67         struct rte_bucket_4_16 *next;
  68         uint64_t next_valid;
  69
  70         /* Cache line 1 */
  71         uint64_t key[4][2];
  72
  73         /* Cache line 2 */
  74         uint8_t data[0];
  75 };
  76
  77 struct rte_table_hash {
  78         struct rte_table_stats stats;
  79
  80         /* Input parameters */
  81         uint32_t n_buckets;
  82         uint32_t n_entries_per_bucket;
  83         uint32_t key_size;
  84         uint32_t entry_size;
  85         uint32_t bucket_size;
  86         uint32_t signature_offset;
  87         uint32_t key_offset;
  88         rte_table_hash_op_hash f_hash;
  89         uint64_t seed;
  90
  91         /* Extendible buckets */
  92         uint32_t n_buckets_ext;
  93         uint32_t stack_pos;
  94         uint32_t *stack;
  95
  96         /* Lookup table */
  97         uint8_t memory[0] __rte_cache_aligned;
  98 };
  99
 100 static int
 101 check_params_create_lru(struct rte_table_hash_key16_lru_params *params) {
 102         /* n_entries */
 103         if (params->n_entries == 0) {
 104                 RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__);
 105                 return -EINVAL;
 106         }
 107
 108         /* f_hash */
 109         if (params->f_hash == NULL) {
 110                 RTE_LOG(ERR, TABLE,
 111                         "%s: f_hash function pointer is NULL\n", __func__);
 112                 return -EINVAL;
 113         }
 114
 115         return 0;
 116 }
 117
 118 static void *
 119 rte_table_hash_create_key16_lru(void *params,
 120                 int socket_id,
 121                 uint32_t entry_size)
 122 {
 123         struct rte_table_hash_key16_lru_params *p =
 124                         (struct rte_table_hash_key16_lru_params *) params;
 125         struct rte_table_hash *f;
 126         uint32_t n_buckets, n_entries_per_bucket,
 127                         key_size, bucket_size_cl, total_size, i;
 128
 129         /* Check input parameters */
 130         if ((check_params_create_lru(p) != 0) ||
 131                 ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
 132                 ((sizeof(struct rte_bucket_4_16) % RTE_CACHE_LINE_SIZE) != 0))
 133                 return NULL;
 134         n_entries_per_bucket = 4;
 135         key_size = 16;
 136
 137         /* Memory allocation */
 138         n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) /
 139                 n_entries_per_bucket);
 140         bucket_size_cl = (sizeof(struct rte_bucket_4_16) + n_entries_per_bucket
 141                 * entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE;
 142         total_size = sizeof(struct rte_table_hash) + n_buckets *
 143                 bucket_size_cl * RTE_CACHE_LINE_SIZE;
 144
 145         f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
 146         if (f == NULL) {
 147                 RTE_LOG(ERR, TABLE,
 148                 "%s: Cannot allocate %u bytes for hash table\n",
 149                 __func__, total_size);
 150                 return NULL;
 151         }
 152         RTE_LOG(INFO, TABLE,
 153                 "%s: Hash table memory footprint is %u bytes\n",
 154                 __func__, total_size);
 155
 156         /* Memory initialization */
 157         f->n_buckets = n_buckets;
 158         f->n_entries_per_bucket = n_entries_per_bucket;
 159         f->key_size = key_size;
 160         f->entry_size = entry_size;
 161         f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE;
 162         f->signature_offset = p->signature_offset;
 163         f->key_offset = p->key_offset;
 164         f->f_hash = p->f_hash;
 165         f->seed = p->seed;
 166
 167         for (i = 0; i < n_buckets; i++) {
 168                 struct rte_bucket_4_16 *bucket;
 169
 170                 bucket = (struct rte_bucket_4_16 *) &f->memory[i *
 171                         f->bucket_size];
 172                 lru_init(bucket);
 173         }
 174
 175         return f;
 176 }
 177
 178 static int
 179 rte_table_hash_free_key16_lru(void *table)
 180 {
 181         struct rte_table_hash *f = (struct rte_table_hash *) table;
 182
 183         /* Check input parameters */
 184         if (f == NULL) {
 185                 RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
 186                 return -EINVAL;
 187         }
 188
 189         rte_free(f);
 190         return 0;
 191 }
 192
 193 static int
 194 rte_table_hash_entry_add_key16_lru(
 195         void *table,
 196         void *key,
 197         void *entry,
 198         int *key_found,
 199         void **entry_ptr)
 200 {
 201         struct rte_table_hash *f = (struct rte_table_hash *) table;
 202         struct rte_bucket_4_16 *bucket;
 203         uint64_t signature, pos;
 204         uint32_t bucket_index, i;
 205
 206         signature = f->f_hash(key, f->key_size, f->seed);
 207         bucket_index = signature & (f->n_buckets - 1);
 208         bucket = (struct rte_bucket_4_16 *)
 209                         &f->memory[bucket_index * f->bucket_size];
 210         signature |= RTE_BUCKET_ENTRY_VALID;
 211
 212         /* Key is present in the bucket */
 213         for (i = 0; i < 4; i++) {
 214                 uint64_t bucket_signature = bucket->signature[i];
 215                 uint8_t *bucket_key = (uint8_t *) bucket->key[i];
 216
 217                 if ((bucket_signature == signature) &&
 218                                 (memcmp(key, bucket_key, f->key_size) == 0)) {
 219                         uint8_t *bucket_data = &bucket->data[i * f->entry_size];
 220
 221                         memcpy(bucket_data, entry, f->entry_size);
 222                         lru_update(bucket, i);
 223                         *key_found = 1;
 224                         *entry_ptr = (void *) bucket_data;
 225                         return 0;
 226                 }
 227         }
 228
 229         /* Key is not present in the bucket */
 230         for (i = 0; i < 4; i++) {
 231                 uint64_t bucket_signature = bucket->signature[i];
 232                 uint8_t *bucket_key = (uint8_t *) bucket->key[i];
 233
 234                 if (bucket_signature == 0) {
 235                         uint8_t *bucket_data = &bucket->data[i * f->entry_size];
 236
 237                         bucket->signature[i] = signature;
 238                         memcpy(bucket_key, key, f->key_size);
 239                         memcpy(bucket_data, entry, f->entry_size);
 240                         lru_update(bucket, i);
 241                         *key_found = 0;
 242                         *entry_ptr = (void *) bucket_data;
 243
 244                         return 0;
 245                 }
 246         }
 247
 248         /* Bucket full: replace LRU entry */
 249         pos = lru_pos(bucket);
 250         bucket->signature[pos] = signature;
 251         memcpy(bucket->key[pos], key, f->key_size);
 252         memcpy(&bucket->data[pos * f->entry_size], entry, f->entry_size);
 253         lru_update(bucket, pos);
 254         *key_found = 0;
 255         *entry_ptr = (void *) &bucket->data[pos * f->entry_size];
 256
 257         return 0;
 258 }
 259
 260 static int
 261 rte_table_hash_entry_delete_key16_lru(
 262         void *table,
 263         void *key,
 264         int *key_found,
 265         void *entry)
 266 {
 267         struct rte_table_hash *f = (struct rte_table_hash *) table;
 268         struct rte_bucket_4_16 *bucket;
 269         uint64_t signature;
 270         uint32_t bucket_index, i;
 271
 272         signature = f->f_hash(key, f->key_size, f->seed);
 273         bucket_index = signature & (f->n_buckets - 1);
 274         bucket = (struct rte_bucket_4_16 *)
 275                         &f->memory[bucket_index * f->bucket_size];
 276         signature |= RTE_BUCKET_ENTRY_VALID;
 277
 278         /* Key is present in the bucket */
 279         for (i = 0; i < 4; i++) {
 280                 uint64_t bucket_signature = bucket->signature[i];
 281                 uint8_t *bucket_key = (uint8_t *) bucket->key[i];
 282
 283                 if ((bucket_signature == signature) &&
 284                                 (memcmp(key, bucket_key, f->key_size) == 0)) {
 285                         uint8_t *bucket_data = &bucket->data[i * f->entry_size];
 286
 287                         bucket->signature[i] = 0;
 288                         *key_found = 1;
 289                         if (entry)
 290                                 memcpy(entry, bucket_data, f->entry_size);
 291                         return 0;
 292                 }
 293         }
 294
 295         /* Key is not present in the bucket */
 296         *key_found = 0;
 297         return 0;
 298 }
 299
 300 static int
 301 check_params_create_ext(struct rte_table_hash_key16_ext_params *params) {
 302         /* n_entries */
 303         if (params->n_entries == 0) {
 304                 RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__);
 305                 return -EINVAL;
 306         }
 307
 308         /* n_entries_ext */
 309         if (params->n_entries_ext == 0) {
 310                 RTE_LOG(ERR, TABLE, "%s: n_entries_ext is zero\n", __func__);
 311                 return -EINVAL;
 312         }
 313
 314         /* f_hash */
 315         if (params->f_hash == NULL) {
 316                 RTE_LOG(ERR, TABLE,
 317                         "%s: f_hash function pointer is NULL\n", __func__);
 318                 return -EINVAL;
 319         }
 320
 321         return 0;
 322 }
 323
 324 static void *
 325 rte_table_hash_create_key16_ext(void *params,
 326                 int socket_id,
 327                 uint32_t entry_size)
 328 {
 329         struct rte_table_hash_key16_ext_params *p =
 330                         (struct rte_table_hash_key16_ext_params *) params;
 331         struct rte_table_hash *f;
 332         uint32_t n_buckets, n_buckets_ext, n_entries_per_bucket, key_size,
 333                         bucket_size_cl, stack_size_cl, total_size, i;
 334
 335         /* Check input parameters */
 336         if ((check_params_create_ext(p) != 0) ||
 337                 ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
 338                 ((sizeof(struct rte_bucket_4_16) % RTE_CACHE_LINE_SIZE) != 0))
 339                 return NULL;
 340
 341         n_entries_per_bucket = 4;
 342         key_size = 16;
 343
 344         /* Memory allocation */
 345         n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) /
 346                 n_entries_per_bucket);
 347         n_buckets_ext = (p->n_entries_ext + n_entries_per_bucket - 1) /
 348                 n_entries_per_bucket;
 349         bucket_size_cl = (sizeof(struct rte_bucket_4_16) + n_entries_per_bucket
 350                 * entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE;
 351         stack_size_cl = (n_buckets_ext * sizeof(uint32_t) + RTE_CACHE_LINE_SIZE - 1)
 352                 / RTE_CACHE_LINE_SIZE;
 353         total_size = sizeof(struct rte_table_hash) +
 354                 ((n_buckets + n_buckets_ext) * bucket_size_cl + stack_size_cl) *
 355                 RTE_CACHE_LINE_SIZE;
 356
 357         f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
 358         if (f == NULL) {
 359                 RTE_LOG(ERR, TABLE,
 360                         "%s: Cannot allocate %u bytes for hash table\n",
 361                         __func__, total_size);
 362                 return NULL;
 363         }
 364         RTE_LOG(INFO, TABLE,
 365                 "%s: Hash table memory footprint is %u bytes\n",
 366                 __func__, total_size);
 367
 368         /* Memory initialization */
 369         f->n_buckets = n_buckets;
 370         f->n_entries_per_bucket = n_entries_per_bucket;
 371         f->key_size = key_size;
 372         f->entry_size = entry_size;
 373         f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE;
 374         f->signature_offset = p->signature_offset;
 375         f->key_offset = p->key_offset;
 376         f->f_hash = p->f_hash;
 377         f->seed = p->seed;
 378
 379         f->n_buckets_ext = n_buckets_ext;
 380         f->stack_pos = n_buckets_ext;
 381         f->stack = (uint32_t *)
 382                 &f->memory[(n_buckets + n_buckets_ext) * f->bucket_size];
 383
 384         for (i = 0; i < n_buckets_ext; i++)
 385                 f->stack[i] = i;
 386
 387         return f;
 388 }
 389
 390 static int
 391 rte_table_hash_free_key16_ext(void *table)
 392 {
 393         struct rte_table_hash *f = (struct rte_table_hash *) table;
 394
 395         /* Check input parameters */
 396         if (f == NULL) {
 397                 RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
 398                 return -EINVAL;
 399         }
 400
 401         rte_free(f);
 402         return 0;
 403 }
 404
 405 static int
 406 rte_table_hash_entry_add_key16_ext(
 407         void *table,
 408         void *key,
 409         void *entry,
 410         int *key_found,
 411         void **entry_ptr)
 412 {
 413         struct rte_table_hash *f = (struct rte_table_hash *) table;
 414         struct rte_bucket_4_16 *bucket0, *bucket, *bucket_prev;
 415         uint64_t signature;
 416         uint32_t bucket_index, i;
 417
 418         signature = f->f_hash(key, f->key_size, f->seed);
 419         bucket_index = signature & (f->n_buckets - 1);
 420         bucket0 = (struct rte_bucket_4_16 *)
 421                         &f->memory[bucket_index * f->bucket_size];
 422         signature |= RTE_BUCKET_ENTRY_VALID;
 423
 424         /* Key is present in the bucket */
 425         for (bucket = bucket0; bucket != NULL; bucket = bucket->next)
 426                 for (i = 0; i < 4; i++) {
 427                         uint64_t bucket_signature = bucket->signature[i];
 428                         uint8_t *bucket_key = (uint8_t *) bucket->key[i];
 429
 430                         if ((bucket_signature == signature) &&
 431                                 (memcmp(key, bucket_key, f->key_size) == 0)) {
 432                                 uint8_t *bucket_data = &bucket->data[i *
 433                                         f->entry_size];
 434
 435                                 memcpy(bucket_data, entry, f->entry_size);
 436                                 *key_found = 1;
 437                                 *entry_ptr = (void *) bucket_data;
 438                                 return 0;
 439                         }
 440                 }
 441
 442         /* Key is not present in the bucket */
 443         for (bucket_prev = NULL, bucket = bucket0; bucket != NULL;
 444                          bucket_prev = bucket, bucket = bucket->next)
 445                 for (i = 0; i < 4; i++) {
 446                         uint64_t bucket_signature = bucket->signature[i];
 447                         uint8_t *bucket_key = (uint8_t *) bucket->key[i];
 448
 449                         if (bucket_signature == 0) {
 450                                 uint8_t *bucket_data = &bucket->data[i *
 451                                         f->entry_size];
 452
 453                                 bucket->signature[i] = signature;
 454                                 memcpy(bucket_key, key, f->key_size);
 455                                 memcpy(bucket_data, entry, f->entry_size);
 456                                 *key_found = 0;
 457                                 *entry_ptr = (void *) bucket_data;
 458
 459                                 return 0;
 460                         }
 461                 }
 462
 463         /* Bucket full: extend bucket */
 464         if (f->stack_pos > 0) {
 465                 bucket_index = f->stack[--f->stack_pos];
 466
 467                 bucket = (struct rte_bucket_4_16 *) &f->memory[(f->n_buckets +
 468                         bucket_index) * f->bucket_size];
 469                 bucket_prev->next = bucket;
 470                 bucket_prev->next_valid = 1;
 471
 472                 bucket->signature[0] = signature;
 473                 memcpy(bucket->key[0], key, f->key_size);
 474                 memcpy(&bucket->data[0], entry, f->entry_size);
 475                 *key_found = 0;
 476                 *entry_ptr = (void *) &bucket->data[0];
 477                 return 0;
 478         }
 479
 480         return -ENOSPC;
 481 }
 482
 483 static int
 484 rte_table_hash_entry_delete_key16_ext(
 485         void *table,
 486         void *key,
 487         int *key_found,
 488         void *entry)
 489 {
 490         struct rte_table_hash *f = (struct rte_table_hash *) table;
 491         struct rte_bucket_4_16 *bucket0, *bucket, *bucket_prev;
 492         uint64_t signature;
 493         uint32_t bucket_index, i;
 494
 495         signature = f->f_hash(key, f->key_size, f->seed);
 496         bucket_index = signature & (f->n_buckets - 1);
 497         bucket0 = (struct rte_bucket_4_16 *)
 498                 &f->memory[bucket_index * f->bucket_size];
 499         signature |= RTE_BUCKET_ENTRY_VALID;
 500
 501         /* Key is present in the bucket */
 502         for (bucket_prev = NULL, bucket = bucket0; bucket != NULL;
 503                 bucket_prev = bucket, bucket = bucket->next)
 504                 for (i = 0; i < 4; i++) {
 505                         uint64_t bucket_signature = bucket->signature[i];
 506                         uint8_t *bucket_key = (uint8_t *) bucket->key[i];
 507
 508                         if ((bucket_signature == signature) &&
 509                                 (memcmp(key, bucket_key, f->key_size) == 0)) {
 510                                 uint8_t *bucket_data = &bucket->data[i *
 511                                         f->entry_size];
 512
 513                                 bucket->signature[i] = 0;
 514                                 *key_found = 1;
 515                                 if (entry)
 516                                         memcpy(entry, bucket_data,
 517                                         f->entry_size);
 518
 519                                 if ((bucket->signature[0] == 0) &&
 520                                         (bucket->signature[1] == 0) &&
 521                                         (bucket->signature[2] == 0) &&
 522                                         (bucket->signature[3] == 0) &&
 523                                         (bucket_prev != NULL)) {
 524                                         bucket_prev->next = bucket->next;
 525                                         bucket_prev->next_valid =
 526                                                 bucket->next_valid;
 527
 528                                         memset(bucket, 0,
 529                                                 sizeof(struct rte_bucket_4_16));
 530                                         bucket_index = (((uint8_t *)bucket -
 531                                                 (uint8_t *)f->memory)/f->bucket_size) - f->n_buckets;
 532                                         f->stack[f->stack_pos++] = bucket_index;
 533                                 }
 534
 535                                 return 0;
 536                         }
 537                 }
 538
 539         /* Key is not present in the bucket */
 540         *key_found = 0;
 541         return 0;
 542 }
 543
 544 #define lookup_key16_cmp(key_in, bucket, pos)                   \
 545 {                                                               \
 546         uint64_t xor[4][2], or[4], signature[4];                \
 547                                                                 \
 548         signature[0] = (~bucket->signature[0]) & 1;             \
 549         signature[1] = (~bucket->signature[1]) & 1;             \
 550         signature[2] = (~bucket->signature[2]) & 1;             \
 551         signature[3] = (~bucket->signature[3]) & 1;             \
 552                                                                 \
 553         xor[0][0] = key_in[0] ^  bucket->key[0][0];             \
 554         xor[0][1] = key_in[1] ^  bucket->key[0][1];             \
 555                                                                 \
 556         xor[1][0] = key_in[0] ^  bucket->key[1][0];             \
 557         xor[1][1] = key_in[1] ^  bucket->key[1][1];             \
 558                                                                 \
 559         xor[2][0] = key_in[0] ^  bucket->key[2][0];             \
 560         xor[2][1] = key_in[1] ^  bucket->key[2][1];             \
 561                                                                 \
 562         xor[3][0] = key_in[0] ^  bucket->key[3][0];             \
 563         xor[3][1] = key_in[1] ^  bucket->key[3][1];             \
 564                                                                 \
 565         or[0] = xor[0][0] | xor[0][1] | signature[0];           \
 566         or[1] = xor[1][0] | xor[1][1] | signature[1];           \
 567         or[2] = xor[2][0] | xor[2][1] | signature[2];           \
 568         or[3] = xor[3][0] | xor[3][1] | signature[3];           \
 569                                                                 \
 570         pos = 4;                                                \
 571         if (or[0] == 0)                                         \
 572                 pos = 0;                                        \
 573         if (or[1] == 0)                                         \
 574                 pos = 1;                                        \
 575         if (or[2] == 0)                                         \
 576                 pos = 2;                                        \
 577         if (or[3] == 0)                                         \
 578                 pos = 3;                                        \
 579 }
 580
 581 #define lookup1_stage0(pkt0_index, mbuf0, pkts, pkts_mask)      \
 582 {                                                               \
 583         uint64_t pkt_mask;                                      \
 584                                                                 \
 585         pkt0_index = __builtin_ctzll(pkts_mask);                \
 586         pkt_mask = 1LLU << pkt0_index;                          \
 587         pkts_mask &= ~pkt_mask;                                 \
 588                                                                 \
 589         mbuf0 = pkts[pkt0_index];                               \
 590         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf0, 0));   \
 591 }
 592
 593 #define lookup1_stage1(mbuf1, bucket1, f)                       \
 594 {                                                               \
 595         uint64_t signature;                                     \
 596         uint32_t bucket_index;                                  \
 597                                                                 \
 598         signature = RTE_MBUF_METADATA_UINT32(mbuf1, f->signature_offset);\
 599         bucket_index = signature & (f->n_buckets - 1);          \
 600         bucket1 = (struct rte_bucket_4_16 *)                    \
 601                 &f->memory[bucket_index * f->bucket_size];      \
 602         rte_prefetch0(bucket1);                                 \
 603         rte_prefetch0((void *)(((uintptr_t) bucket1) + RTE_CACHE_LINE_SIZE));\
 604 }
 605
 606 #define lookup1_stage2_lru(pkt2_index, mbuf2, bucket2,          \
 607                 pkts_mask_out, entries, f)                      \
 608 {                                                               \
 609         void *a;                                                \
 610         uint64_t pkt_mask;                                      \
 611         uint64_t *key;                                          \
 612         uint32_t pos;                                           \
 613                                                                 \
 614         key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\
 615                                                                 \
 616         lookup_key16_cmp(key, bucket2, pos);                    \
 617                                                                 \
 618         pkt_mask = (bucket2->signature[pos] & 1LLU) << pkt2_index;\
 619         pkts_mask_out |= pkt_mask;                              \
 620                                                                 \
 621         a = (void *) &bucket2->data[pos * f->entry_size];       \
 622         rte_prefetch0(a);                                       \
 623         entries[pkt2_index] = a;                                \
 624         lru_update(bucket2, pos);                               \
 625 }
 626
 627 #define lookup1_stage2_ext(pkt2_index, mbuf2, bucket2, pkts_mask_out, entries, \
 628         buckets_mask, buckets, keys, f)                         \
 629 {                                                               \
 630         struct rte_bucket_4_16 *bucket_next;                    \
 631         void *a;                                                \
 632         uint64_t pkt_mask, bucket_mask;                         \
 633         uint64_t *key;                                          \
 634         uint32_t pos;                                           \
 635                                                                 \
 636         key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\
 637                                                                 \
 638         lookup_key16_cmp(key, bucket2, pos);                    \
 639                                                                 \
 640         pkt_mask = (bucket2->signature[pos] & 1LLU) << pkt2_index;\
 641         pkts_mask_out |= pkt_mask;                              \
 642                                                                 \
 643         a = (void *) &bucket2->data[pos * f->entry_size];       \
 644         rte_prefetch0(a);                                       \
 645         entries[pkt2_index] = a;                                \
 646                                                                 \
 647         bucket_mask = (~pkt_mask) & (bucket2->next_valid << pkt2_index);\
 648         buckets_mask |= bucket_mask;                            \
 649         bucket_next = bucket2->next;                            \
 650         buckets[pkt2_index] = bucket_next;                      \
 651         keys[pkt2_index] = key;                                 \
 652 }
 653
 654 #define lookup_grinder(pkt_index, buckets, keys, pkts_mask_out, entries,\
 655         buckets_mask, f)                                        \
 656 {                                                               \
 657         struct rte_bucket_4_16 *bucket, *bucket_next;           \
 658         void *a;                                                \
 659         uint64_t pkt_mask, bucket_mask;                         \
 660         uint64_t *key;                                          \
 661         uint32_t pos;                                           \
 662                                                                 \
 663         bucket = buckets[pkt_index];                            \
 664         key = keys[pkt_index];                                  \
 665                                                                 \
 666         lookup_key16_cmp(key, bucket, pos);                     \
 667                                                                 \
 668         pkt_mask = (bucket->signature[pos] & 1LLU) << pkt_index;\
 669         pkts_mask_out |= pkt_mask;                              \
 670                                                                 \
 671         a = (void *) &bucket->data[pos * f->entry_size];        \
 672         rte_prefetch0(a);                                       \
 673         entries[pkt_index] = a;                                 \
 674                                                                 \
 675         bucket_mask = (~pkt_mask) & (bucket->next_valid << pkt_index);\
 676         buckets_mask |= bucket_mask;                            \
 677         bucket_next = bucket->next;                             \
 678         rte_prefetch0(bucket_next);                             \
 679         rte_prefetch0((void *)(((uintptr_t) bucket_next) + RTE_CACHE_LINE_SIZE));\
 680         buckets[pkt_index] = bucket_next;                       \
 681         keys[pkt_index] = key;                                  \
 682 }
 683
 684 #define lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01,\
 685                 pkts, pkts_mask)                                \
 686 {                                                               \
 687         uint64_t pkt00_mask, pkt01_mask;                        \
 688                                                                 \
 689         pkt00_index = __builtin_ctzll(pkts_mask);               \
 690         pkt00_mask = 1LLU << pkt00_index;                       \
 691         pkts_mask &= ~pkt00_mask;                               \
 692                                                                 \
 693         mbuf00 = pkts[pkt00_index];                             \
 694         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0));  \
 695                                                                 \
 696         pkt01_index = __builtin_ctzll(pkts_mask);               \
 697         pkt01_mask = 1LLU << pkt01_index;                       \
 698         pkts_mask &= ~pkt01_mask;                               \
 699                                                                 \
 700         mbuf01 = pkts[pkt01_index];                             \
 701         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0));  \
 702 }
 703
 704 #define lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,\
 705                 mbuf00, mbuf01, pkts, pkts_mask)                \
 706 {                                                               \
 707         uint64_t pkt00_mask, pkt01_mask;                        \
 708                                                                 \
 709         pkt00_index = __builtin_ctzll(pkts_mask);               \
 710         pkt00_mask = 1LLU << pkt00_index;                       \
 711         pkts_mask &= ~pkt00_mask;                               \
 712                                                                 \
 713         mbuf00 = pkts[pkt00_index];                             \
 714         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf00, 0));  \
 715                                                                 \
 716         pkt01_index = __builtin_ctzll(pkts_mask);               \
 717         if (pkts_mask == 0)                                     \
 718                 pkt01_index = pkt00_index;                      \
 719         pkt01_mask = 1LLU << pkt01_index;                       \
 720         pkts_mask &= ~pkt01_mask;                               \
 721                                                                 \
 722         mbuf01 = pkts[pkt01_index];                             \
 723         rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, 0));  \
 724 }
 725
 726 #define lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f)   \
 727 {                                                               \
 728         uint64_t signature10, signature11;                      \
 729         uint32_t bucket10_index, bucket11_index;                \
 730                                                                 \
 731         signature10 = RTE_MBUF_METADATA_UINT32(mbuf10, f->signature_offset);\
 732         bucket10_index = signature10 & (f->n_buckets - 1);      \
 733         bucket10 = (struct rte_bucket_4_16 *)                   \
 734                 &f->memory[bucket10_index * f->bucket_size];    \
 735         rte_prefetch0(bucket10);                                \
 736         rte_prefetch0((void *)(((uintptr_t) bucket10) + RTE_CACHE_LINE_SIZE));\
 737                                                                 \
 738         signature11 = RTE_MBUF_METADATA_UINT32(mbuf11, f->signature_offset);\
 739         bucket11_index = signature11 & (f->n_buckets - 1);      \
 740         bucket11 = (struct rte_bucket_4_16 *)                   \
 741                 &f->memory[bucket11_index * f->bucket_size];    \
 742         rte_prefetch0(bucket11);                                \
 743         rte_prefetch0((void *)(((uintptr_t) bucket11) + RTE_CACHE_LINE_SIZE));\
 744 }
 745
 746 #define lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21,\
 747                 bucket20, bucket21, pkts_mask_out, entries, f)  \
 748 {                                                               \
 749         void *a20, *a21;                                        \
 750         uint64_t pkt20_mask, pkt21_mask;                        \
 751         uint64_t *key20, *key21;                                \
 752         uint32_t pos20, pos21;                                  \
 753                                                                 \
 754         key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\
 755         key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\
 756                                                                 \
 757         lookup_key16_cmp(key20, bucket20, pos20);               \
 758         lookup_key16_cmp(key21, bucket21, pos21);               \
 759                                                                 \
 760         pkt20_mask = (bucket20->signature[pos20] & 1LLU) << pkt20_index;\
 761         pkt21_mask = (bucket21->signature[pos21] & 1LLU) << pkt21_index;\
 762         pkts_mask_out |= pkt20_mask | pkt21_mask;                       \
 763                                                                 \
 764         a20 = (void *) &bucket20->data[pos20 * f->entry_size];  \
 765         a21 = (void *) &bucket21->data[pos21 * f->entry_size];  \
 766         rte_prefetch0(a20);                                     \
 767         rte_prefetch0(a21);                                     \
 768         entries[pkt20_index] = a20;                             \
 769         entries[pkt21_index] = a21;                             \
 770         lru_update(bucket20, pos20);                            \
 771         lru_update(bucket21, pos21);                            \
 772 }
 773
 774 #define lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21, bucket20, \
 775         bucket21, pkts_mask_out, entries, buckets_mask, buckets, keys, f) \
 776 {                                                               \
 777         struct rte_bucket_4_16 *bucket20_next, *bucket21_next;  \
 778         void *a20, *a21;                                        \
 779         uint64_t pkt20_mask, pkt21_mask, bucket20_mask, bucket21_mask;\
 780         uint64_t *key20, *key21;                                \
 781         uint32_t pos20, pos21;                                  \
 782                                                                 \
 783         key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\
 784         key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\
 785                                                                 \
 786         lookup_key16_cmp(key20, bucket20, pos20);               \
 787         lookup_key16_cmp(key21, bucket21, pos21);               \
 788                                                                 \
 789         pkt20_mask = (bucket20->signature[pos20] & 1LLU) << pkt20_index;\
 790         pkt21_mask = (bucket21->signature[pos21] & 1LLU) << pkt21_index;\
 791         pkts_mask_out |= pkt20_mask | pkt21_mask;               \
 792                                                                 \
 793         a20 = (void *) &bucket20->data[pos20 * f->entry_size];  \
 794         a21 = (void *) &bucket21->data[pos21 * f->entry_size];  \
 795         rte_prefetch0(a20);                                     \
 796         rte_prefetch0(a21);                                     \
 797         entries[pkt20_index] = a20;                             \
 798         entries[pkt21_index] = a21;                             \
 799                                                                 \
 800         bucket20_mask = (~pkt20_mask) & (bucket20->next_valid << pkt20_index);\
 801         bucket21_mask = (~pkt21_mask) & (bucket21->next_valid << pkt21_index);\
 802         buckets_mask |= bucket20_mask | bucket21_mask;          \
 803         bucket20_next = bucket20->next;                         \
 804         bucket21_next = bucket21->next;                         \
 805         buckets[pkt20_index] = bucket20_next;                   \
 806         buckets[pkt21_index] = bucket21_next;                   \
 807         keys[pkt20_index] = key20;                              \
 808         keys[pkt21_index] = key21;                              \
 809 }
 810
 811 static int
 812 rte_table_hash_lookup_key16_lru(
 813         void *table,
 814         struct rte_mbuf **pkts,
 815         uint64_t pkts_mask,
 816         uint64_t *lookup_hit_mask,
 817         void **entries)
 818 {
 819         struct rte_table_hash *f = (struct rte_table_hash *) table;
 820         struct rte_bucket_4_16 *bucket10, *bucket11, *bucket20, *bucket21;
 821         struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21;
 822         uint32_t pkt00_index, pkt01_index, pkt10_index;
 823         uint32_t pkt11_index, pkt20_index, pkt21_index;
 824         uint64_t pkts_mask_out = 0;
 825
 826         __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
 827         RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in);
 828
 829         /* Cannot run the pipeline with less than 5 packets */
 830         if (__builtin_popcountll(pkts_mask) < 5) {
 831                 for ( ; pkts_mask; ) {
 832                         struct rte_bucket_4_16 *bucket;
 833                         struct rte_mbuf *mbuf;
 834                         uint32_t pkt_index;
 835
 836                         lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask);
 837                         lookup1_stage1(mbuf, bucket, f);
 838                         lookup1_stage2_lru(pkt_index, mbuf, bucket,
 839                                 pkts_mask_out, entries, f);
 840                 }
 841
 842                 *lookup_hit_mask = pkts_mask_out;
 843                 RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out));
 844                 return 0;
 845         }
 846
 847         /*
 848          * Pipeline fill
 849          *
 850          */
 851         /* Pipeline stage 0 */
 852         lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
 853                 pkts_mask);
 854
 855         /* Pipeline feed */
 856         mbuf10 = mbuf00;
 857         mbuf11 = mbuf01;
 858         pkt10_index = pkt00_index;
 859         pkt11_index = pkt01_index;
 860
 861         /* Pipeline stage 0 */
 862         lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
 863                 pkts_mask);
 864
 865         /* Pipeline stage 1 */
 866         lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
 867
 868         /*
 869          * Pipeline run
 870          *
 871          */
 872         for ( ; pkts_mask; ) {
 873                 /* Pipeline feed */
 874                 bucket20 = bucket10;
 875                 bucket21 = bucket11;
 876                 mbuf20 = mbuf10;
 877                 mbuf21 = mbuf11;
 878                 mbuf10 = mbuf00;
 879                 mbuf11 = mbuf01;
 880                 pkt20_index = pkt10_index;
 881                 pkt21_index = pkt11_index;
 882                 pkt10_index = pkt00_index;
 883                 pkt11_index = pkt01_index;
 884
 885                 /* Pipeline stage 0 */
 886                 lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,
 887                         mbuf00, mbuf01, pkts, pkts_mask);
 888
 889                 /* Pipeline stage 1 */
 890                 lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
 891
 892                 /* Pipeline stage 2 */
 893                 lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21,
 894                         bucket20, bucket21, pkts_mask_out, entries, f);
 895         }
 896
 897         /*
 898          * Pipeline flush
 899          *
 900          */
 901         /* Pipeline feed */
 902         bucket20 = bucket10;
 903         bucket21 = bucket11;
 904         mbuf20 = mbuf10;
 905         mbuf21 = mbuf11;
 906         mbuf10 = mbuf00;
 907         mbuf11 = mbuf01;
 908         pkt20_index = pkt10_index;
 909         pkt21_index = pkt11_index;
 910         pkt10_index = pkt00_index;
 911         pkt11_index = pkt01_index;
 912
 913         /* Pipeline stage 1 */
 914         lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
 915
 916         /* Pipeline stage 2 */
 917         lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21,
 918                 bucket20, bucket21, pkts_mask_out, entries, f);
 919
 920         /* Pipeline feed */
 921         bucket20 = bucket10;
 922         bucket21 = bucket11;
 923         mbuf20 = mbuf10;
 924         mbuf21 = mbuf11;
 925         pkt20_index = pkt10_index;
 926         pkt21_index = pkt11_index;
 927
 928         /* Pipeline stage 2 */
 929         lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21,
 930                 bucket20, bucket21, pkts_mask_out, entries, f);
 931
 932         *lookup_hit_mask = pkts_mask_out;
 933         RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out));
 934         return 0;
 935 } /* rte_table_hash_lookup_key16_lru() */
 936
 937 static int
 938 rte_table_hash_lookup_key16_ext(
 939         void *table,
 940         struct rte_mbuf **pkts,
 941         uint64_t pkts_mask,
 942         uint64_t *lookup_hit_mask,
 943         void **entries)
 944 {
 945         struct rte_table_hash *f = (struct rte_table_hash *) table;
 946         struct rte_bucket_4_16 *bucket10, *bucket11, *bucket20, *bucket21;
 947         struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21;
 948         uint32_t pkt00_index, pkt01_index, pkt10_index;
 949         uint32_t pkt11_index, pkt20_index, pkt21_index;
 950         uint64_t pkts_mask_out = 0, buckets_mask = 0;
 951         struct rte_bucket_4_16 *buckets[RTE_PORT_IN_BURST_SIZE_MAX];
 952         uint64_t *keys[RTE_PORT_IN_BURST_SIZE_MAX];
 953
 954         __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
 955         RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in);
 956
 957         /* Cannot run the pipeline with less than 5 packets */
 958         if (__builtin_popcountll(pkts_mask) < 5) {
 959                 for ( ; pkts_mask; ) {
 960                         struct rte_bucket_4_16 *bucket;
 961                         struct rte_mbuf *mbuf;
 962                         uint32_t pkt_index;
 963
 964                         lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask);
 965                         lookup1_stage1(mbuf, bucket, f);
 966                         lookup1_stage2_ext(pkt_index, mbuf, bucket,
 967                                 pkts_mask_out, entries, buckets_mask,
 968                                 buckets, keys, f);
 969                 }
 970
 971                 goto grind_next_buckets;
 972         }
 973
 974         /*
 975          * Pipeline fill
 976          *
 977          */
 978         /* Pipeline stage 0 */
 979         lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
 980                 pkts_mask);
 981
 982         /* Pipeline feed */
 983         mbuf10 = mbuf00;
 984         mbuf11 = mbuf01;
 985         pkt10_index = pkt00_index;
 986         pkt11_index = pkt01_index;
 987
 988         /* Pipeline stage 0 */
 989         lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts,
 990                 pkts_mask);
 991
 992         /* Pipeline stage 1 */
 993         lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
 994
 995         /*
 996          * Pipeline run
 997          *
 998          */
 999         for ( ; pkts_mask; ) {
1000                 /* Pipeline feed */
1001                 bucket20 = bucket10;
1002                 bucket21 = bucket11;
1003                 mbuf20 = mbuf10;
1004                 mbuf21 = mbuf11;
1005                 mbuf10 = mbuf00;
1006                 mbuf11 = mbuf01;
1007                 pkt20_index = pkt10_index;
1008                 pkt21_index = pkt11_index;
1009                 pkt10_index = pkt00_index;
1010                 pkt11_index = pkt01_index;
1011
1012                 /* Pipeline stage 0 */
1013                 lookup2_stage0_with_odd_support(pkt00_index, pkt01_index,
1014                         mbuf00, mbuf01, pkts, pkts_mask);
1015
1016                 /* Pipeline stage 1 */
1017                 lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
1018
1019                 /* Pipeline stage 2 */
1020                 lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21,
1021                         bucket20, bucket21, pkts_mask_out, entries,
1022                         buckets_mask, buckets, keys, f);
1023         }
1024
1025         /*
1026          * Pipeline flush
1027          *
1028          */
1029         /* Pipeline feed */
1030         bucket20 = bucket10;
1031         bucket21 = bucket11;
1032         mbuf20 = mbuf10;
1033         mbuf21 = mbuf11;
1034         mbuf10 = mbuf00;
1035         mbuf11 = mbuf01;
1036         pkt20_index = pkt10_index;
1037         pkt21_index = pkt11_index;
1038         pkt10_index = pkt00_index;
1039         pkt11_index = pkt01_index;
1040
1041         /* Pipeline stage 1 */
1042         lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f);
1043
1044         /* Pipeline stage 2 */
1045         lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21,
1046                 bucket20, bucket21, pkts_mask_out, entries,
1047                 buckets_mask, buckets, keys, f);
1048
1049         /* Pipeline feed */
1050         bucket20 = bucket10;
1051         bucket21 = bucket11;
1052         mbuf20 = mbuf10;
1053         mbuf21 = mbuf11;
1054         pkt20_index = pkt10_index;
1055         pkt21_index = pkt11_index;
1056
1057         /* Pipeline stage 2 */
1058         lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21,
1059                 bucket20, bucket21, pkts_mask_out, entries,
1060                 buckets_mask, buckets, keys, f);
1061
1062 grind_next_buckets:
1063         /* Grind next buckets */
1064         for ( ; buckets_mask; ) {
1065                 uint64_t buckets_mask_next = 0;
1066
1067                 for ( ; buckets_mask; ) {
1068                         uint64_t pkt_mask;
1069                         uint32_t pkt_index;
1070
1071                         pkt_index = __builtin_ctzll(buckets_mask);
1072                         pkt_mask = 1LLU << pkt_index;
1073                         buckets_mask &= ~pkt_mask;
1074
1075                         lookup_grinder(pkt_index, buckets, keys, pkts_mask_out,
1076                                 entries, buckets_mask_next, f);
1077                 }
1078
1079                 buckets_mask = buckets_mask_next;
1080         }
1081
1082         *lookup_hit_mask = pkts_mask_out;
1083         RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out));
1084         return 0;
1085 } /* rte_table_hash_lookup_key16_ext() */
1086
1087 static int
1088 rte_table_hash_key16_stats_read(void *table, struct rte_table_stats *stats, int clear)
1089 {
1090         struct rte_table_hash *t = (struct rte_table_hash *) table;
1091
1092         if (stats != NULL)
1093                 memcpy(stats, &t->stats, sizeof(t->stats));
1094
1095         if (clear)
1096                 memset(&t->stats, 0, sizeof(t->stats));
1097
1098         return 0;
1099 }
1100
1101 struct rte_table_ops rte_table_hash_key16_lru_ops = {
1102         .f_create = rte_table_hash_create_key16_lru,
1103         .f_free = rte_table_hash_free_key16_lru,
1104         .f_add = rte_table_hash_entry_add_key16_lru,
1105         .f_delete = rte_table_hash_entry_delete_key16_lru,
1106         .f_lookup = rte_table_hash_lookup_key16_lru,
1107         .f_stats = rte_table_hash_key16_stats_read,
1108 };
1109
1110 struct rte_table_ops rte_table_hash_key16_ext_ops = {
1111         .f_create = rte_table_hash_create_key16_ext,
1112         .f_free = rte_table_hash_free_key16_ext,
1113         .f_add = rte_table_hash_entry_add_key16_ext,
1114         .f_delete = rte_table_hash_entry_delete_key16_ext,
1115         .f_lookup = rte_table_hash_lookup_key16_ext,
1116         .f_stats = rte_table_hash_key16_stats_read,
1117 };