test/test/test_hash_readwrite_lf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Arm Limited
   3  */
   4
   5 #include <inttypes.h>
   6 #include <locale.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_jhash.h>
  12 #include <rte_launch.h>
  13 #include <rte_malloc.h>
  14 #include <rte_random.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "test.h"
  18
  19 #ifndef RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF
  20 #define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF 0
  21 #endif
  22
  23 #define BULK_LOOKUP_SIZE 32
  24
  25 #define RUN_WITH_HTM_DISABLED 0
  26
  27 #if (RUN_WITH_HTM_DISABLED)
  28
  29 #define TOTAL_ENTRY (5*1024)
  30 #define TOTAL_INSERT (5*1024)
  31
  32 #else
  33
  34 #define TOTAL_ENTRY (4*1024*1024)
  35 #define TOTAL_INSERT (4*1024*1024)
  36
  37 #endif
  38
  39 #define READ_FAIL 1
  40 #define READ_PASS_NO_KEY_SHIFTS 2
  41 #define READ_PASS_SHIFT_PATH 4
  42 #define READ_PASS_NON_SHIFT_PATH 8
  43 #define BULK_LOOKUP 16
  44 #define NUM_TEST 3
  45 unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
  46
  47 struct rwc_perf {
  48         uint32_t w_no_ks_r_hit[2][NUM_TEST];
  49         uint32_t w_no_ks_r_miss[2][NUM_TEST];
  50         uint32_t w_ks_r_hit_nsp[2][NUM_TEST];
  51         uint32_t w_ks_r_hit_sp[2][NUM_TEST];
  52         uint32_t w_ks_r_miss[2][NUM_TEST];
  53         uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
  54 };
  55
  56 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
  57
  58 struct {
  59         uint32_t *keys;
  60         uint32_t *keys_no_ks;
  61         uint32_t *keys_ks;
  62         uint32_t *keys_absent;
  63         uint32_t *keys_shift_path;
  64         uint32_t *keys_non_shift_path;
  65         uint32_t count_keys_no_ks;
  66         uint32_t count_keys_ks;
  67         uint32_t count_keys_absent;
  68         uint32_t count_keys_shift_path;
  69         uint32_t count_keys_non_shift_path;
  70         uint32_t single_insert;
  71         struct rte_hash *h;
  72 } tbl_rwc_test_param;
  73
  74 static rte_atomic64_t gread_cycles;
  75 static rte_atomic64_t greads;
  76
  77 static volatile uint8_t writer_done;
  78 static volatile uint8_t multi_writer_done[4];
  79
  80 uint16_t enabled_core_ids[RTE_MAX_LCORE];
  81
  82 uint8_t *scanned_bkts;
  83
  84 static inline int
  85 get_enabled_cores_list(void)
  86 {
  87         uint32_t i = 0;
  88         uint16_t core_id;
  89         uint32_t max_cores = rte_lcore_count();
  90         for (core_id = 0; core_id < RTE_MAX_LCORE && i < max_cores; core_id++) {
  91                 if (rte_lcore_is_enabled(core_id)) {
  92                         enabled_core_ids[i] = core_id;
  93                         i++;
  94                 }
  95         }
  96
  97         if (i != max_cores) {
  98                 printf("Number of enabled cores in list is different from "
  99                         "number given by rte_lcore_count()\n");
 100                 return -1;
 101         }
 102         return 0;
 103 }
 104
 105 static inline int
 106 check_bucket(uint32_t bkt_idx, uint32_t key)
 107 {
 108         uint32_t iter;
 109         uint32_t prev_iter;
 110         uint32_t diff;
 111         uint32_t count = 0;
 112         const void *next_key;
 113         void *next_data;
 114
 115         /* Temporary bucket to hold the keys */
 116         uint32_t keys_in_bkt[8];
 117
 118         iter = bkt_idx * 8;
 119         prev_iter = iter;
 120         while (rte_hash_iterate(tbl_rwc_test_param.h,
 121                         &next_key, &next_data, &iter) >= 0) {
 122
 123                 /* Check for duplicate entries */
 124                 if (*(const uint32_t *)next_key == key)
 125                         return 1;
 126
 127                 /* Identify if there is any free entry in the bucket */
 128                 diff = iter - prev_iter;
 129                 if (diff > 1)
 130                         break;
 131
 132                 prev_iter = iter;
 133                 keys_in_bkt[count] = *(const uint32_t *)next_key;
 134                 count++;
 135
 136                 /* All entries in the bucket are occupied */
 137                 if (count == 8) {
 138
 139                         /*
 140                          * Check if bucket was not scanned before, to avoid
 141                          * duplicate keys.
 142                          */
 143                         if (scanned_bkts[bkt_idx] == 0) {
 144                                 /*
 145                                  * Since this bucket (pointed to by bkt_idx) is
 146                                  * full, it is likely that key(s) in this
 147                                  * bucket will be on the shift path, when
 148                                  * collision occurs. Thus, add it to
 149                                  * keys_shift_path.
 150                                  */
 151                                 memcpy(tbl_rwc_test_param.keys_shift_path +
 152                                         tbl_rwc_test_param.count_keys_shift_path
 153                                         , keys_in_bkt, 32);
 154                                 tbl_rwc_test_param.count_keys_shift_path += 8;
 155                                 scanned_bkts[bkt_idx] = 1;
 156                         }
 157                         return -1;
 158                 }
 159         }
 160         return 0;
 161 }
 162
 163 static int
 164 generate_keys(void)
 165 {
 166         uint32_t *keys = NULL;
 167         uint32_t *keys_no_ks = NULL;
 168         uint32_t *keys_ks = NULL;
 169         uint32_t *keys_absent = NULL;
 170         uint32_t *keys_non_shift_path = NULL;
 171         uint32_t *found = NULL;
 172         uint32_t count_keys_no_ks = 0;
 173         uint32_t count_keys_ks = 0;
 174         uint32_t i;
 175
 176         /*
 177          * keys will consist of a) keys whose addition to the hash table
 178          * will result in shifting of the existing keys to their alternate
 179          * locations b) keys whose addition to the hash table will not result
 180          * in shifting of the existing keys.
 181          */
 182         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 183         if (keys == NULL) {
 184                 printf("RTE_MALLOC failed\n");
 185                 goto err;
 186         }
 187
 188         /*
 189          * keys_no_ks (no key-shifts): Subset of 'keys' - consists of keys  that
 190          * will NOT result in shifting of the existing keys to their alternate
 191          * locations. Roughly around 900K keys.
 192          */
 193         keys_no_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 194         if (keys_no_ks == NULL) {
 195                 printf("RTE_MALLOC failed\n");
 196                 goto err;
 197         }
 198
 199         /*
 200          * keys_ks (key-shifts): Subset of 'keys' - consists of keys that will
 201          * result in shifting of the existing keys to their alternate locations.
 202          * Roughly around 146K keys. There might be repeating keys. More code is
 203          * required to filter out these keys which will complicate the test case
 204          */
 205         keys_ks = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 206         if (keys_ks == NULL) {
 207                 printf("RTE_MALLOC failed\n");
 208                 goto err;
 209         }
 210
 211         /* Used to identify keys not inserted in the hash table */
 212         found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 213         if (found == NULL) {
 214                 printf("RTE_MALLOC failed\n");
 215                 goto err;
 216         }
 217
 218         /*
 219          * This consist of keys not inserted to the hash table.
 220          * Used to test perf of lookup on keys that do not exist in the table.
 221          */
 222         keys_absent = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT, 0);
 223         if (keys_absent == NULL) {
 224                 printf("RTE_MALLOC failed\n");
 225                 goto err;
 226         }
 227
 228         /*
 229          * This consist of keys which are likely to be on the shift
 230          * path (i.e. being moved to alternate location), when collision occurs
 231          * on addition of a key to an already full primary bucket.
 232          * Used to test perf of lookup on keys that are on the shift path.
 233          */
 234         tbl_rwc_test_param.keys_shift_path = rte_malloc(NULL, sizeof(uint32_t) *
 235                                                         TOTAL_INSERT, 0);
 236         if (tbl_rwc_test_param.keys_shift_path == NULL) {
 237                 printf("RTE_MALLOC failed\n");
 238                 goto err;
 239         }
 240
 241         /*
 242          * This consist of keys which are never on the shift
 243          * path (i.e. being moved to alternate location), when collision occurs
 244          * on addition of a key to an already full primary bucket.
 245          * Used to test perf of lookup on keys that are not on the shift path.
 246          */
 247         keys_non_shift_path = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_INSERT,
 248                                          0);
 249         if (keys_non_shift_path == NULL) {
 250                 printf("RTE_MALLOC failed\n");
 251                 goto err;
 252         }
 253
 254
 255         hash_sig_t sig;
 256         uint32_t prim_bucket_idx;
 257         int ret;
 258         uint32_t num_buckets;
 259         uint32_t bucket_bitmask;
 260         num_buckets  = rte_align32pow2(TOTAL_ENTRY) / 8;
 261         bucket_bitmask = num_buckets - 1;
 262
 263         /*
 264          * Used to mark bkts in which at least one key was shifted to its
 265          * alternate location
 266          */
 267         scanned_bkts = rte_malloc(NULL, sizeof(uint8_t) * num_buckets, 0);
 268         if (scanned_bkts == NULL) {
 269                 printf("RTE_MALLOC failed\n");
 270                 goto err;
 271         }
 272
 273         tbl_rwc_test_param.keys = keys;
 274         tbl_rwc_test_param.keys_no_ks = keys_no_ks;
 275         tbl_rwc_test_param.keys_ks = keys_ks;
 276         tbl_rwc_test_param.keys_absent = keys_absent;
 277         tbl_rwc_test_param.keys_non_shift_path = keys_non_shift_path;
 278         /* Generate keys by adding previous two keys, neglect overflow */
 279         printf("Generating keys...\n");
 280         keys[0] = 0;
 281         keys[1] = 1;
 282         for (i = 2; i < TOTAL_INSERT; i++)
 283                 keys[i] = keys[i-1] + keys[i-2];
 284
 285         /* Segregate keys into keys_no_ks and keys_ks */
 286         for (i = 0; i < TOTAL_INSERT; i++) {
 287                 /* Check if primary bucket has space.*/
 288                 sig = rte_hash_hash(tbl_rwc_test_param.h,
 289                                         tbl_rwc_test_param.keys+i);
 290                 prim_bucket_idx = sig & bucket_bitmask;
 291                 ret = check_bucket(prim_bucket_idx, keys[i]);
 292                 if (ret < 0) {
 293                         /*
 294                          * Primary bucket is full, this key will result in
 295                          * shifting of the keys to their alternate locations.
 296                          */
 297                         keys_ks[count_keys_ks] = keys[i];
 298                         count_keys_ks++;
 299                 } else if (ret == 0) {
 300                         /*
 301                          * Primary bucket has space, this key will not result in
 302                          * shifting of the keys. Hence, add key to the table.
 303                          */
 304                         ret = rte_hash_add_key_data(tbl_rwc_test_param.h,
 305                                                         keys+i,
 306                                                         (void *)((uintptr_t)i));
 307                         if (ret < 0) {
 308                                 printf("writer failed %"PRIu32"\n", i);
 309                                 break;
 310                         }
 311                         keys_no_ks[count_keys_no_ks] = keys[i];
 312                         count_keys_no_ks++;
 313                 }
 314         }
 315
 316         for (i = 0; i < count_keys_no_ks; i++) {
 317                 /*
 318                  * Identify keys in keys_no_ks with value less than
 319                  * 4M (HTM enabled) OR 5K (HTM disabled)
 320                  */
 321                 if (keys_no_ks[i] < TOTAL_INSERT)
 322                         found[keys_no_ks[i]]++;
 323         }
 324
 325         for (i = 0; i < count_keys_ks; i++) {
 326                 /*
 327                  * Identify keys in keys_ks with value less than
 328                  * 4M (HTM enabled) OR 5K (HTM disabled)
 329                  */
 330                 if (keys_ks[i] < TOTAL_INSERT)
 331                         found[keys_ks[i]]++;
 332         }
 333
 334         uint32_t count_keys_absent = 0;
 335         for (i = 0; i < TOTAL_INSERT; i++) {
 336                 /*
 337                  * Identify missing keys between 0 and
 338                  * 4M (HTM enabled) OR 5K (HTM disabled)
 339                  */
 340                 if (found[i] == 0)
 341                         keys_absent[count_keys_absent++] = i;
 342         }
 343
 344         /* Find keys that will not be on the shift path */
 345         uint32_t iter;
 346         const void *next_key;
 347         void *next_data;
 348         uint32_t count = 0;
 349         for (i = 0; i < num_buckets; i++) {
 350                 /* Check bucket for no keys shifted to alternate locations */
 351                 if (scanned_bkts[i] == 0) {
 352                         iter = i * 8;
 353                         while (rte_hash_iterate(tbl_rwc_test_param.h,
 354                                 &next_key, &next_data, &iter) >= 0) {
 355
 356                                 /* Check if key belongs to the current bucket */
 357                                 if (i >= (iter-1)/8)
 358                                         keys_non_shift_path[count++]
 359                                                 = *(const uint32_t *)next_key;
 360                                 else
 361                                         break;
 362                         }
 363                 }
 364         }
 365
 366         tbl_rwc_test_param.count_keys_no_ks = count_keys_no_ks;
 367         tbl_rwc_test_param.count_keys_ks = count_keys_ks;
 368         tbl_rwc_test_param.count_keys_absent = count_keys_absent;
 369         tbl_rwc_test_param.count_keys_non_shift_path = count;
 370
 371         printf("\nCount of keys NOT causing shifting of existing keys to "
 372         "alternate location: %d\n", tbl_rwc_test_param.count_keys_no_ks);
 373         printf("\nCount of keys causing shifting of existing keys to alternate "
 374                 "locations: %d\n\n", tbl_rwc_test_param.count_keys_ks);
 375         printf("Count of absent keys that will never be added to the hash "
 376                 "table: %d\n\n", tbl_rwc_test_param.count_keys_absent);
 377         printf("Count of keys likely to be on the shift path: %d\n\n",
 378                tbl_rwc_test_param.count_keys_shift_path);
 379         printf("Count of keys not likely to be on the shift path: %d\n\n",
 380                tbl_rwc_test_param.count_keys_non_shift_path);
 381
 382         rte_free(found);
 383         rte_hash_free(tbl_rwc_test_param.h);
 384         return 0;
 385
 386 err:
 387         rte_free(keys);
 388         rte_free(keys_no_ks);
 389         rte_free(keys_ks);
 390         rte_free(keys_absent);
 391         rte_free(found);
 392         rte_free(tbl_rwc_test_param.keys_shift_path);
 393         rte_free(scanned_bkts);
 394         return -1;
 395 }
 396
 397 static int
 398 init_params(int rwc_lf, int use_jhash, int htm)
 399 {
 400         struct rte_hash *handle;
 401
 402         struct rte_hash_parameters hash_params = {
 403                 .entries = TOTAL_ENTRY,
 404                 .key_len = sizeof(uint32_t),
 405                 .hash_func_init_val = 0,
 406                 .socket_id = rte_socket_id(),
 407         };
 408
 409         if (use_jhash)
 410                 hash_params.hash_func = rte_jhash;
 411         else
 412                 hash_params.hash_func = rte_hash_crc;
 413
 414         if (rwc_lf)
 415                 hash_params.extra_flag =
 416                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
 417                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 418         else if (htm)
 419                 hash_params.extra_flag =
 420                         RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
 421                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 422                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 423         else
 424                 hash_params.extra_flag =
 425                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
 426                         RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
 427
 428         hash_params.name = "tests";
 429
 430         handle = rte_hash_create(&hash_params);
 431         if (handle == NULL) {
 432                 printf("hash creation failed");
 433                 return -1;
 434         }
 435
 436         tbl_rwc_test_param.h = handle;
 437         return 0;
 438 }
 439
 440 static int
 441 test_rwc_reader(__attribute__((unused)) void *arg)
 442 {
 443         uint32_t i, j;
 444         int ret;
 445         uint64_t begin, cycles;
 446         uint32_t loop_cnt = 0;
 447         uint8_t read_type = (uint8_t)((uintptr_t)arg);
 448         uint32_t read_cnt;
 449         uint32_t *keys;
 450         uint32_t extra_keys;
 451         int32_t *pos;
 452         void *temp_a[BULK_LOOKUP_SIZE];
 453
 454         /* Used to identify keys not inserted in the hash table */
 455         pos = rte_zmalloc(NULL, sizeof(uint32_t) * BULK_LOOKUP_SIZE, 0);
 456         if (pos == NULL) {
 457                 printf("RTE_MALLOC failed\n");
 458                 return -1;
 459         }
 460
 461         if (read_type & READ_FAIL) {
 462                 keys = tbl_rwc_test_param.keys_absent;
 463                 read_cnt = tbl_rwc_test_param.count_keys_absent;
 464         } else if (read_type & READ_PASS_NO_KEY_SHIFTS) {
 465                 keys = tbl_rwc_test_param.keys_no_ks;
 466                 read_cnt = tbl_rwc_test_param.count_keys_no_ks;
 467         } else if (read_type & READ_PASS_SHIFT_PATH) {
 468                 keys = tbl_rwc_test_param.keys_shift_path;
 469                 read_cnt = tbl_rwc_test_param.count_keys_shift_path;
 470         } else {
 471                 keys = tbl_rwc_test_param.keys_non_shift_path;
 472                 read_cnt = tbl_rwc_test_param.count_keys_non_shift_path;
 473         }
 474
 475         extra_keys = read_cnt & (BULK_LOOKUP_SIZE - 1);
 476
 477         begin = rte_rdtsc_precise();
 478         do {
 479                 if (read_type & BULK_LOOKUP) {
 480                         for (i = 0; i < (read_cnt - extra_keys);
 481                              i += BULK_LOOKUP_SIZE) {
 482                                 /* Array of  pointer to the list of keys */
 483                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 484                                         temp_a[j] = keys + i + j;
 485
 486                                 rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 487                                                    (const void **)
 488                                                    ((uintptr_t)temp_a),
 489                                                    BULK_LOOKUP_SIZE, pos);
 490                                 /* Validate lookup result */
 491                                 for (j = 0; j < BULK_LOOKUP_SIZE; j++)
 492                                         if ((read_type & READ_FAIL &&
 493                                              pos[j] != -ENOENT) ||
 494                                             (!(read_type & READ_FAIL) &&
 495                                              pos[j] == -ENOENT)) {
 496                                                 printf("lookup failed!"
 497                                                        "%"PRIu32"\n",
 498                                                        keys[i + j]);
 499                                                 return -1;
 500                                         }
 501                         }
 502                         for (j = 0; j < extra_keys; j++)
 503                                 temp_a[j] = keys + i + j;
 504
 505                         rte_hash_lookup_bulk(tbl_rwc_test_param.h,
 506                                            (const void **)
 507                                            ((uintptr_t)temp_a),
 508                                            extra_keys, pos);
 509                         for (j = 0; j < extra_keys; j++)
 510                                 if ((read_type & READ_FAIL &&
 511                                      pos[j] != -ENOENT) ||
 512                                     (!(read_type & READ_FAIL) &&
 513                                      pos[j] == -ENOENT)) {
 514                                         printf("lookup failed! %"PRIu32"\n",
 515                                                keys[i + j]);
 516                                         return -1;
 517                                 }
 518                 } else {
 519                         for (i = 0; i < read_cnt; i++) {
 520                                 ret = rte_hash_lookup
 521                                         (tbl_rwc_test_param.h, keys + i);
 522                                 if (((read_type & READ_FAIL) &&
 523                                      (ret != -ENOENT)) ||
 524                                     (!(read_type & READ_FAIL) &&
 525                                         ret == -ENOENT)) {
 526                                         printf("lookup failed! %"PRIu32"\n",
 527                                                keys[i]);
 528                                         return -1;
 529                                 }
 530                         }
 531                 }
 532                 loop_cnt++;
 533         } while (!writer_done);
 534
 535         cycles = rte_rdtsc_precise() - begin;
 536         rte_atomic64_add(&gread_cycles, cycles);
 537         rte_atomic64_add(&greads, read_cnt*loop_cnt);
 538         return 0;
 539 }
 540
 541 static int
 542 write_keys(uint8_t key_shift)
 543 {
 544         uint32_t i;
 545         int ret;
 546         uint32_t key_cnt;
 547         uint32_t *keys;
 548         if (key_shift) {
 549                 key_cnt = tbl_rwc_test_param.count_keys_ks;
 550                 keys = tbl_rwc_test_param.keys_ks;
 551         } else {
 552                 key_cnt = tbl_rwc_test_param.count_keys_no_ks;
 553                 keys = tbl_rwc_test_param.keys_no_ks;
 554         }
 555         for (i = 0; i < key_cnt; i++) {
 556                 ret = rte_hash_add_key(tbl_rwc_test_param.h, keys + i);
 557                 if (!key_shift && ret < 0) {
 558                         printf("writer failed %"PRIu32"\n", i);
 559                         return -1;
 560                 }
 561         }
 562         return 0;
 563 }
 564
 565 static int
 566 test_rwc_multi_writer(__attribute__((unused)) void *arg)
 567 {
 568         uint32_t i, offset;
 569         uint32_t pos_core = (uint32_t)((uintptr_t)arg);
 570         offset = pos_core * tbl_rwc_test_param.single_insert;
 571         for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++)
 572                 rte_hash_add_key(tbl_rwc_test_param.h,
 573                                  tbl_rwc_test_param.keys_ks + i);
 574         multi_writer_done[pos_core] = 1;
 575         return 0;
 576 }
 577
 578 /*
 579  * Test lookup perf:
 580  * Reader(s) lookup keys present in the table.
 581  */
 582 static int
 583 test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
 584                                 int htm)
 585 {
 586         unsigned int n, m;
 587         uint64_t i;
 588         int use_jhash = 0;
 589         uint8_t key_shift = 0;
 590         uint8_t read_type = READ_PASS_NO_KEY_SHIFTS;
 591
 592         rte_atomic64_init(&greads);
 593         rte_atomic64_init(&gread_cycles);
 594
 595         if (init_params(rwc_lf, use_jhash, htm) != 0)
 596                 goto err;
 597         printf("\nTest: Hash add - no key-shifts, read - hit\n");
 598         for (m = 0; m < 2; m++) {
 599                 if (m == 1) {
 600                         printf("\n** With bulk-lookup **\n");
 601                         read_type |= BULK_LOOKUP;
 602                 }
 603                 for (n = 0; n < NUM_TEST; n++) {
 604                         unsigned int tot_lcore = rte_lcore_count();
 605                         if (tot_lcore < rwc_core_cnt[n] + 1)
 606                                 goto finish;
 607
 608                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 609
 610                         rte_atomic64_clear(&greads);
 611                         rte_atomic64_clear(&gread_cycles);
 612
 613                         rte_hash_reset(tbl_rwc_test_param.h);
 614                         writer_done = 0;
 615                         if (write_keys(key_shift) < 0)
 616                                 goto err;
 617                         writer_done = 1;
 618                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 619                                 rte_eal_remote_launch(test_rwc_reader,
 620                                                 (void *)(uintptr_t)read_type,
 621                                                         enabled_core_ids[i]);
 622                         rte_eal_mp_wait_lcore();
 623
 624                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 625                                 if (lcore_config[i].ret < 0)
 626                                         goto err;
 627
 628                         unsigned long long cycles_per_lookup =
 629                                 rte_atomic64_read(&gread_cycles) /
 630                                 rte_atomic64_read(&greads);
 631                         rwc_perf_results->w_no_ks_r_hit[m][n]
 632                                                 = cycles_per_lookup;
 633                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 634                 }
 635         }
 636
 637 finish:
 638         rte_hash_free(tbl_rwc_test_param.h);
 639         return 0;
 640
 641 err:
 642         rte_hash_free(tbl_rwc_test_param.h);
 643         return -1;
 644 }
 645
 646 /*
 647  * Test lookup perf:
 648  * Reader(s) lookup keys absent in the table while
 649  * 'Main' thread adds with no key-shifts.
 650  */
 651 static int
 652 test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
 653                                 int htm)
 654 {
 655         unsigned int n, m;
 656         uint64_t i;
 657         int use_jhash = 0;
 658         uint8_t key_shift = 0;
 659         uint8_t read_type = READ_FAIL;
 660         int ret;
 661
 662         rte_atomic64_init(&greads);
 663         rte_atomic64_init(&gread_cycles);
 664
 665         if (init_params(rwc_lf, use_jhash, htm) != 0)
 666                 goto err;
 667         printf("\nTest: Hash add - no key-shifts, Hash lookup - miss\n");
 668         for (m = 0; m < 2; m++) {
 669                 if (m == 1) {
 670                         printf("\n** With bulk-lookup **\n");
 671                         read_type |= BULK_LOOKUP;
 672                 }
 673                 for (n = 0; n < NUM_TEST; n++) {
 674                         unsigned int tot_lcore = rte_lcore_count();
 675                         if (tot_lcore < rwc_core_cnt[n] + 1)
 676                                 goto finish;
 677
 678                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 679
 680                         rte_atomic64_clear(&greads);
 681                         rte_atomic64_clear(&gread_cycles);
 682
 683                         rte_hash_reset(tbl_rwc_test_param.h);
 684                         writer_done = 0;
 685
 686                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 687                                 rte_eal_remote_launch(test_rwc_reader,
 688                                                 (void *)(uintptr_t)read_type,
 689                                                         enabled_core_ids[i]);
 690                         ret = write_keys(key_shift);
 691                         writer_done = 1;
 692                         rte_eal_mp_wait_lcore();
 693
 694                         if (ret < 0)
 695                                 goto err;
 696                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 697                                 if (lcore_config[i].ret < 0)
 698                                         goto err;
 699
 700                         unsigned long long cycles_per_lookup =
 701                                 rte_atomic64_read(&gread_cycles) /
 702                                 rte_atomic64_read(&greads);
 703                         rwc_perf_results->w_no_ks_r_miss[m][n]
 704                                                 = cycles_per_lookup;
 705                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 706                 }
 707         }
 708
 709 finish:
 710         rte_hash_free(tbl_rwc_test_param.h);
 711         return 0;
 712
 713 err:
 714         rte_hash_free(tbl_rwc_test_param.h);
 715         return -1;
 716 }
 717
 718 /*
 719  * Test lookup perf:
 720  * Reader(s) lookup keys present in the table and not likely to be on the
 721  * shift path  while 'Main' thread adds keys causing key-shifts.
 722  */
 723 static int
 724 test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
 725                                     int rwc_lf, int htm)
 726 {
 727         unsigned int n, m;
 728         uint64_t i;
 729         int use_jhash = 0;
 730         int ret;
 731         uint8_t key_shift;
 732         uint8_t read_type = READ_PASS_NON_SHIFT_PATH;
 733
 734         rte_atomic64_init(&greads);
 735         rte_atomic64_init(&gread_cycles);
 736
 737         if (init_params(rwc_lf, use_jhash, htm) != 0)
 738                 goto err;
 739         printf("\nTest: Hash add - key shift, Hash lookup - hit"
 740                " (non-shift-path)\n");
 741         for (m = 0; m < 2; m++) {
 742                 if (m == 1) {
 743                         printf("\n** With bulk-lookup **\n");
 744                         read_type |= BULK_LOOKUP;
 745                 }
 746                 for (n = 0; n < NUM_TEST; n++) {
 747                         unsigned int tot_lcore = rte_lcore_count();
 748                         if (tot_lcore < rwc_core_cnt[n] + 1)
 749                                 goto finish;
 750
 751                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 752
 753                         rte_atomic64_clear(&greads);
 754                         rte_atomic64_clear(&gread_cycles);
 755
 756                         rte_hash_reset(tbl_rwc_test_param.h);
 757                         writer_done = 0;
 758                         key_shift = 0;
 759                         if (write_keys(key_shift) < 0)
 760                                 goto err;
 761                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 762                                 rte_eal_remote_launch(test_rwc_reader,
 763                                                 (void *)(uintptr_t)read_type,
 764                                                         enabled_core_ids[i]);
 765                         key_shift = 1;
 766                         ret = write_keys(key_shift);
 767                         writer_done = 1;
 768                         rte_eal_mp_wait_lcore();
 769
 770                         if (ret < 0)
 771                                 goto err;
 772                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 773                                 if (lcore_config[i].ret < 0)
 774                                         goto err;
 775
 776                         unsigned long long cycles_per_lookup =
 777                                 rte_atomic64_read(&gread_cycles) /
 778                                 rte_atomic64_read(&greads);
 779                         rwc_perf_results->w_ks_r_hit_nsp[m][n]
 780                                                 = cycles_per_lookup;
 781                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 782                 }
 783         }
 784
 785 finish:
 786         rte_hash_free(tbl_rwc_test_param.h);
 787         return 0;
 788
 789 err:
 790         rte_hash_free(tbl_rwc_test_param.h);
 791         return -1;
 792 }
 793
 794 /*
 795  * Test lookup perf:
 796  * Reader(s) lookup keys present in the table and likely on the shift-path while
 797  * 'Main' thread adds keys causing key-shifts.
 798  */
 799 static int
 800 test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
 801                                 int htm)
 802 {
 803         unsigned int n, m;
 804         uint64_t i;
 805         int use_jhash = 0;
 806         int ret;
 807         uint8_t key_shift;
 808         uint8_t read_type = READ_PASS_SHIFT_PATH;
 809
 810         rte_atomic64_init(&greads);
 811         rte_atomic64_init(&gread_cycles);
 812
 813         if (init_params(rwc_lf, use_jhash, htm) != 0)
 814                 goto err;
 815         printf("\nTest: Hash add - key shift, Hash lookup - hit (shift-path)"
 816                "\n");
 817
 818         for (m = 0; m < 2; m++) {
 819                 if (m == 1) {
 820                         printf("\n** With bulk-lookup **\n");
 821                         read_type |= BULK_LOOKUP;
 822                 }
 823                 for (n = 0; n < NUM_TEST; n++) {
 824                         unsigned int tot_lcore = rte_lcore_count();
 825                         if (tot_lcore < rwc_core_cnt[n])
 826                                 goto finish;
 827
 828                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 829                         rte_atomic64_clear(&greads);
 830                         rte_atomic64_clear(&gread_cycles);
 831
 832                         rte_hash_reset(tbl_rwc_test_param.h);
 833                         writer_done = 0;
 834                         key_shift = 0;
 835                         if (write_keys(key_shift) < 0)
 836                                 goto err;
 837                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 838                                 rte_eal_remote_launch(test_rwc_reader,
 839                                                 (void *)(uintptr_t)read_type,
 840                                                 enabled_core_ids[i]);
 841                         key_shift = 1;
 842                         ret = write_keys(key_shift);
 843                         writer_done = 1;
 844                         rte_eal_mp_wait_lcore();
 845
 846                         if (ret < 0)
 847                                 goto err;
 848                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 849                                 if (lcore_config[i].ret < 0)
 850                                         goto err;
 851
 852                         unsigned long long cycles_per_lookup =
 853                                 rte_atomic64_read(&gread_cycles) /
 854                                 rte_atomic64_read(&greads);
 855                         rwc_perf_results->w_ks_r_hit_sp[m][n]
 856                                                 = cycles_per_lookup;
 857                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 858                 }
 859         }
 860
 861 finish:
 862         rte_hash_free(tbl_rwc_test_param.h);
 863         return 0;
 864
 865 err:
 866         rte_hash_free(tbl_rwc_test_param.h);
 867         return -1;
 868 }
 869
 870 /*
 871  * Test lookup perf:
 872  * Reader(s) lookup keys absent in the table while
 873  * 'Main' thread adds keys causing key-shifts.
 874  */
 875 static int
 876 test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
 877                              htm)
 878 {
 879         unsigned int n, m;
 880         uint64_t i;
 881         int use_jhash = 0;
 882         int ret;
 883         uint8_t key_shift;
 884         uint8_t read_type = READ_FAIL;
 885
 886         rte_atomic64_init(&greads);
 887         rte_atomic64_init(&gread_cycles);
 888
 889         if (init_params(rwc_lf, use_jhash, htm) != 0)
 890                 goto err;
 891         printf("\nTest: Hash add - key shift, Hash lookup - miss\n");
 892         for (m = 0; m < 2; m++) {
 893                 if (m == 1) {
 894                         printf("\n** With bulk-lookup **\n");
 895                         read_type |= BULK_LOOKUP;
 896                 }
 897                 for (n = 0; n < NUM_TEST; n++) {
 898                         unsigned int tot_lcore = rte_lcore_count();
 899                         if (tot_lcore < rwc_core_cnt[n] + 1)
 900                                 goto finish;
 901
 902                         printf("\nNumber of readers: %u\n", rwc_core_cnt[n]);
 903
 904                         rte_atomic64_clear(&greads);
 905                         rte_atomic64_clear(&gread_cycles);
 906
 907                         rte_hash_reset(tbl_rwc_test_param.h);
 908                         writer_done = 0;
 909                         key_shift = 0;
 910                         if (write_keys(key_shift) < 0)
 911                                 goto err;
 912                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 913                                 rte_eal_remote_launch(test_rwc_reader,
 914                                                 (void *)(uintptr_t)read_type,
 915                                                         enabled_core_ids[i]);
 916                         key_shift = 1;
 917                         ret = write_keys(key_shift);
 918                         writer_done = 1;
 919                         rte_eal_mp_wait_lcore();
 920
 921                         if (ret < 0)
 922                                 goto err;
 923                         for (i = 1; i <= rwc_core_cnt[n]; i++)
 924                                 if (lcore_config[i].ret < 0)
 925                                         goto err;
 926
 927                         unsigned long long cycles_per_lookup =
 928                                 rte_atomic64_read(&gread_cycles) /
 929                                 rte_atomic64_read(&greads);
 930                         rwc_perf_results->w_ks_r_miss[m][n] = cycles_per_lookup;
 931                         printf("Cycles per lookup: %llu\n", cycles_per_lookup);
 932                 }
 933         }
 934
 935 finish:
 936         rte_hash_free(tbl_rwc_test_param.h);
 937         return 0;
 938
 939 err:
 940         rte_hash_free(tbl_rwc_test_param.h);
 941         return -1;
 942 }
 943
 944 /*
 945  * Test lookup perf for multi-writer:
 946  * Reader(s) lookup keys present in the table and likely on the shift-path while
 947  * Writers add keys causing key-shiftsi.
 948  * Writers are running in parallel, on different data plane cores.
 949  */
 950 static int
 951 test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
 952                            int htm)
 953 {
 954         unsigned int n, m, k;
 955         uint64_t i;
 956         int use_jhash = 0;
 957         uint8_t key_shift;
 958         uint8_t read_type = READ_PASS_SHIFT_PATH;
 959
 960         rte_atomic64_init(&greads);
 961         rte_atomic64_init(&gread_cycles);
 962
 963         if (init_params(rwc_lf, use_jhash, htm) != 0)
 964                 goto err;
 965         printf("\nTest: Multi-add-lookup\n");
 966         uint8_t pos_core;
 967         for (m = 1; m < NUM_TEST; m++) {
 968                 /* Calculate keys added by each writer */
 969                 tbl_rwc_test_param.single_insert =
 970                         tbl_rwc_test_param.count_keys_ks / rwc_core_cnt[m];
 971                 for (k = 0; k < 2; k++) {
 972                         if (k == 1) {
 973                                 printf("\n** With bulk-lookup **\n");
 974                                 read_type |= BULK_LOOKUP;
 975                         }
 976                         for (n = 0; n < NUM_TEST; n++) {
 977                                 unsigned int tot_lcore  = rte_lcore_count();
 978                                 if (tot_lcore < (rwc_core_cnt[n] +
 979                                      rwc_core_cnt[m] + 1))
 980                                         goto finish;
 981
 982                                 printf("\nNumber of writers: %u",
 983                                        rwc_core_cnt[m]);
 984                                 printf("\nNumber of readers: %u\n",
 985                                        rwc_core_cnt[n]);
 986
 987                                 rte_atomic64_clear(&greads);
 988                                 rte_atomic64_clear(&gread_cycles);
 989
 990                                 rte_hash_reset(tbl_rwc_test_param.h);
 991                                 writer_done = 0;
 992                                 for (i = 0; i < 4; i++)
 993                                         multi_writer_done[i] = 0;
 994                                 key_shift = 0;
 995                                 if (write_keys(key_shift) < 0)
 996                                         goto err;
 997
 998                                 /* Launch reader(s) */
 999                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1000                                         rte_eal_remote_launch(test_rwc_reader,
1001                                                 (void *)(uintptr_t)read_type,
1002                                                 enabled_core_ids[i]);
1003                                 key_shift = 1;
1004                                 pos_core = 0;
1005
1006                                 /* Launch writers */
1007                                 for (; i <= rwc_core_cnt[m]
1008                                      + rwc_core_cnt[n]; i++) {
1009                                         rte_eal_remote_launch
1010                                                 (test_rwc_multi_writer,
1011                                                 (void *)(uintptr_t)pos_core,
1012                                                 enabled_core_ids[i]);
1013                                         pos_core++;
1014                                 }
1015
1016                                 /* Wait for writers to complete */
1017                                 for (i = 0; i < rwc_core_cnt[m]; i++)
1018                                         while
1019                                                 (multi_writer_done[i] == 0);
1020                                 writer_done = 1;
1021
1022                                 rte_eal_mp_wait_lcore();
1023
1024                                 for (i = 1; i <= rwc_core_cnt[n]; i++)
1025                                         if (lcore_config[i].ret < 0)
1026                                                 goto err;
1027
1028                                 unsigned long long cycles_per_lookup =
1029                                         rte_atomic64_read(&gread_cycles)
1030                                         / rte_atomic64_read(&greads);
1031                                 rwc_perf_results->multi_rw[m][k][n]
1032                                         = cycles_per_lookup;
1033                                 printf("Cycles per lookup: %llu\n",
1034                                        cycles_per_lookup);
1035                         }
1036                 }
1037         }
1038
1039 finish:
1040         rte_hash_free(tbl_rwc_test_param.h);
1041         return 0;
1042
1043 err:
1044         rte_hash_free(tbl_rwc_test_param.h);
1045         return -1;
1046 }
1047
1048 static int
1049 test_hash_readwrite_lf_main(void)
1050 {
1051         /*
1052          * Variables used to choose different tests.
1053          * rwc_lf indicates if read-write concurrency lock-free support is
1054          * enabled.
1055          * htm indicates if Hardware transactional memory support is enabled.
1056          */
1057         int rwc_lf = 0;
1058         int htm;
1059         int use_jhash = 0;
1060         if (rte_lcore_count() == 1) {
1061                 printf("More than one lcore is required "
1062                         "to do read write lock-free concurrency test\n");
1063                 return -1;
1064         }
1065
1066         setlocale(LC_NUMERIC, "");
1067
1068         if (rte_tm_supported())
1069                 htm = 1;
1070         else
1071                 htm = 0;
1072
1073         if (init_params(rwc_lf, use_jhash, htm) != 0)
1074                 return -1;
1075         if (generate_keys() != 0)
1076                 return -1;
1077         if (get_enabled_cores_list() != 0)
1078                 return -1;
1079
1080         if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
1081                 rwc_lf = 1;
1082                 printf("Test lookup with read-write concurrency lock free support"
1083                        " enabled\n");
1084                 if (test_hash_add_no_ks_lookup_hit(&rwc_lf_results, rwc_lf,
1085                                                         htm) < 0)
1086                         return -1;
1087                 if (test_hash_add_no_ks_lookup_miss(&rwc_lf_results, rwc_lf,
1088                                                         htm) < 0)
1089                         return -1;
1090                 if (test_hash_add_ks_lookup_hit_non_sp(&rwc_lf_results, rwc_lf,
1091                                                         htm) < 0)
1092                         return -1;
1093                 if (test_hash_add_ks_lookup_hit_sp(&rwc_lf_results, rwc_lf,
1094                                                         htm) < 0)
1095                         return -1;
1096                 if (test_hash_add_ks_lookup_miss(&rwc_lf_results, rwc_lf, htm)
1097                                                         < 0)
1098                         return -1;
1099                 if (test_hash_multi_add_lookup(&rwc_lf_results, rwc_lf, htm)
1100                                                         < 0)
1101                         return -1;
1102         }
1103         printf("\nTest lookup with read-write concurrency lock free support"
1104                " disabled\n");
1105         rwc_lf = 0;
1106         if (!htm) {
1107                 printf("With HTM Disabled\n");
1108                 if (!RUN_WITH_HTM_DISABLED) {
1109                         printf("Enable RUN_WITH_HTM_DISABLED to test with"
1110                                " lock-free disabled");
1111                         goto results;
1112                 }
1113         } else
1114                 printf("With HTM Enabled\n");
1115         if (test_hash_add_no_ks_lookup_hit(&rwc_non_lf_results, rwc_lf, htm)
1116                                                 < 0)
1117                 return -1;
1118         if (test_hash_add_no_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm)
1119                                                 < 0)
1120                 return -1;
1121         if (test_hash_add_ks_lookup_hit_non_sp(&rwc_non_lf_results, rwc_lf,
1122                                                 htm) < 0)
1123                 return -1;
1124         if (test_hash_add_ks_lookup_hit_sp(&rwc_non_lf_results, rwc_lf, htm)
1125                                                 < 0)
1126                 return -1;
1127         if (test_hash_add_ks_lookup_miss(&rwc_non_lf_results, rwc_lf, htm) < 0)
1128                 return -1;
1129         if (test_hash_multi_add_lookup(&rwc_non_lf_results, rwc_lf, htm) < 0)
1130                 return -1;
1131 results:
1132         printf("\n\t\t\t\t\t\t********** Results summary **********\n\n");
1133         int i, j, k;
1134         for (j = 0; j < 2; j++) {
1135                 if (j == 1)
1136                         printf("\n\t\t\t\t\t#######********** Bulk Lookup "
1137                                "**********#######\n\n");
1138                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t"
1139                         "\t\t\t\t_________________\n");
1140                 printf("Writers\t\tReaders\t\tLock-free\tHTM\t\tTest-case\t\t\t"
1141                        "\t\t\tCycles per lookup\n");
1142                 printf("_______\t\t_______\t\t_________\t___\t\t_________\t\t\t"
1143                        "\t\t\t_________________\n");
1144                 for (i = 0; i < NUM_TEST; i++) {
1145                         printf("%u\t\t%u\t\t", 1, rwc_core_cnt[i]);
1146                         printf("Enabled\t\t");
1147                         printf("N/A\t\t");
1148                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1149                                 "%u\n\t\t\t\t\t\t\t\t",
1150                                 rwc_lf_results.w_no_ks_r_hit[j][i]);
1151                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1152                                 "%u\n\t\t\t\t\t\t\t\t",
1153                                 rwc_lf_results.w_no_ks_r_miss[j][i]);
1154                         printf("Hash add - key-shifts, lookup - hit"
1155                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1156                                rwc_lf_results.w_ks_r_hit_nsp[j][i]);
1157                         printf("Hash add - key-shifts, lookup - hit "
1158                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1159                                rwc_lf_results.w_ks_r_hit_sp[j][i]);
1160                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1161                                 "%u\n\n\t\t\t\t",
1162                                 rwc_lf_results.w_ks_r_miss[j][i]);
1163
1164                         printf("Disabled\t");
1165                         if (htm)
1166                                 printf("Enabled\t\t");
1167                         else
1168                                 printf("Disabled\t");
1169                         printf("Hash add - no key-shifts, lookup - hit\t\t\t\t"
1170                                 "%u\n\t\t\t\t\t\t\t\t",
1171                                 rwc_non_lf_results.w_no_ks_r_hit[j][i]);
1172                         printf("Hash add - no key-shifts, lookup - miss\t\t\t\t"
1173                                 "%u\n\t\t\t\t\t\t\t\t",
1174                                 rwc_non_lf_results.w_no_ks_r_miss[j][i]);
1175                         printf("Hash add - key-shifts, lookup - hit "
1176                                "(non-shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1177                                rwc_non_lf_results.w_ks_r_hit_nsp[j][i]);
1178                         printf("Hash add - key-shifts, lookup - hit "
1179                                "(shift-path)\t\t%u\n\t\t\t\t\t\t\t\t",
1180                                rwc_non_lf_results.w_ks_r_hit_sp[j][i]);
1181                         printf("Hash add - key-shifts, Hash lookup miss\t\t\t\t"
1182                                "%u\n", rwc_non_lf_results.w_ks_r_miss[j][i]);
1183
1184                         printf("_______\t\t_______\t\t_________\t___\t\t"
1185                                "_________\t\t\t\t\t\t_________________\n");
1186                 }
1187
1188                 for (i = 1; i < NUM_TEST; i++) {
1189                         for (k = 0; k < NUM_TEST; k++) {
1190                                 printf("%u", rwc_core_cnt[i]);
1191                                 printf("\t\t%u\t\t", rwc_core_cnt[k]);
1192                                 printf("Enabled\t\t");
1193                                 printf("N/A\t\t");
1194                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n\n\t\t"
1195                                        "\t\t",
1196                                        rwc_lf_results.multi_rw[i][j][k]);
1197                                 printf("Disabled\t");
1198                                 if (htm)
1199                                         printf("Enabled\t\t");
1200                                 else
1201                                         printf("Disabled\t");
1202                                 printf("Multi-add-lookup\t\t\t\t\t\t%u\n",
1203                                        rwc_non_lf_results.multi_rw[i][j][k]);
1204
1205                                 printf("_______\t\t_______\t\t_________\t___"
1206                                        "\t\t_________\t\t\t\t\t\t"
1207                                        "_________________\n");
1208                         }
1209                 }
1210         }
1211         rte_free(tbl_rwc_test_param.keys);
1212         rte_free(tbl_rwc_test_param.keys_no_ks);
1213         rte_free(tbl_rwc_test_param.keys_ks);
1214         rte_free(tbl_rwc_test_param.keys_absent);
1215         rte_free(tbl_rwc_test_param.keys_shift_path);
1216         rte_free(scanned_bkts);
1217         return 0;
1218 }
1219
1220 REGISTER_TEST_COMMAND(hash_readwrite_lf_autotest, test_hash_readwrite_lf_main);