drivers/net/mlx5/mlx5_utils.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_hash_crc.h>
   7
   8 #include <mlx5_malloc.h>
   9
  10 #include "mlx5_utils.h"
  11
  12 /********************* Hash List **********************/
  13
  14 static struct mlx5_hlist_entry *
  15 mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
  16                              void *ctx __rte_unused)
  17 {
  18         return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
  19 }
  20
  21 static void
  22 mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
  23                              struct mlx5_hlist_entry *entry)
  24 {
  25         mlx5_free(entry);
  26 }
  27
  28 static int
  29 mlx5_hlist_default_match_cb(struct mlx5_hlist *h __rte_unused,
  30                             struct mlx5_hlist_entry *entry,
  31                             uint64_t key, void *ctx __rte_unused)
  32 {
  33         return entry->key != key;
  34 }
  35
  36 struct mlx5_hlist *
  37 mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
  38                   uint32_t flags, mlx5_hlist_create_cb cb_create,
  39                   mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
  40 {
  41         struct mlx5_hlist *h;
  42         uint32_t act_size;
  43         uint32_t alloc_size;
  44         uint32_t i;
  45
  46         if (!size || (!cb_create ^ !cb_remove))
  47                 return NULL;
  48         /* Align to the next power of 2, 32bits integer is enough now. */
  49         if (!rte_is_power_of_2(size)) {
  50                 act_size = rte_align32pow2(size);
  51                 DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
  52                         "be aligned to 0x%" PRIX32 ".", size, act_size);
  53         } else {
  54                 act_size = size;
  55         }
  56         alloc_size = sizeof(struct mlx5_hlist) +
  57                      sizeof(struct mlx5_hlist_bucket) * act_size;
  58         /* Using zmalloc, then no need to initialize the heads. */
  59         h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
  60                         SOCKET_ID_ANY);
  61         if (!h) {
  62                 DRV_LOG(ERR, "No memory for hash list %s creation",
  63                         name ? name : "None");
  64                 return NULL;
  65         }
  66         if (name)
  67                 snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
  68         h->table_sz = act_size;
  69         h->mask = act_size - 1;
  70         h->entry_sz = entry_size;
  71         h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
  72         h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
  73         h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
  74         h->cb_match = cb_match ? cb_match : mlx5_hlist_default_match_cb;
  75         h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
  76         for (i = 0; i < act_size; i++)
  77                 rte_rwlock_init(&h->buckets[i].lock);
  78         DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
  79                 h->name, act_size);
  80         return h;
  81 }
  82
  83 static struct mlx5_hlist_entry *
  84 __hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
  85                void *ctx, bool reuse)
  86 {
  87         struct mlx5_hlist_head *first;
  88         struct mlx5_hlist_entry *node;
  89
  90         MLX5_ASSERT(h);
  91         first = &h->buckets[idx].head;
  92         LIST_FOREACH(node, first, next) {
  93                 if (!h->cb_match(h, node, key, ctx)) {
  94                         if (reuse) {
  95                                 __atomic_add_fetch(&node->ref_cnt, 1,
  96                                                    __ATOMIC_RELAXED);
  97                                 DRV_LOG(DEBUG, "Hash list %s entry %p "
  98                                         "reuse: %u.",
  99                                         h->name, (void *)node, node->ref_cnt);
 100                         }
 101                         break;
 102                 }
 103         }
 104         return node;
 105 }
 106
 107 static struct mlx5_hlist_entry *
 108 hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
 109              void *ctx, bool reuse)
 110 {
 111         struct mlx5_hlist_entry *node;
 112
 113         MLX5_ASSERT(h);
 114         rte_rwlock_read_lock(&h->buckets[idx].lock);
 115         node = __hlist_lookup(h, key, idx, ctx, reuse);
 116         rte_rwlock_read_unlock(&h->buckets[idx].lock);
 117         return node;
 118 }
 119
 120 struct mlx5_hlist_entry *
 121 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 122 {
 123         uint32_t idx;
 124
 125         if (h->direct_key)
 126                 idx = (uint32_t)(key & h->mask);
 127         else
 128                 idx = rte_hash_crc_8byte(key, 0) & h->mask;
 129         return hlist_lookup(h, key, idx, ctx, false);
 130 }
 131
 132 struct mlx5_hlist_entry*
 133 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 134 {
 135         uint32_t idx;
 136         struct mlx5_hlist_head *first;
 137         struct mlx5_hlist_bucket *b;
 138         struct mlx5_hlist_entry *entry;
 139         uint32_t prev_gen_cnt = 0;
 140
 141         if (h->direct_key)
 142                 idx = (uint32_t)(key & h->mask);
 143         else
 144                 idx = rte_hash_crc_8byte(key, 0) & h->mask;
 145         MLX5_ASSERT(h);
 146         b = &h->buckets[idx];
 147         /* Use write lock directly for write-most list. */
 148         if (!h->write_most) {
 149                 prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
 150                 entry = hlist_lookup(h, key, idx, ctx, true);
 151                 if (entry)
 152                         return entry;
 153         }
 154         rte_rwlock_write_lock(&b->lock);
 155         /* Check if the list changed by other threads. */
 156         if (h->write_most ||
 157             prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
 158                 entry = __hlist_lookup(h, key, idx, ctx, true);
 159                 if (entry)
 160                         goto done;
 161         }
 162         first = &b->head;
 163         entry = h->cb_create(h, key, ctx);
 164         if (!entry) {
 165                 rte_errno = ENOMEM;
 166                 DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
 167                 goto done;
 168         }
 169         entry->key = key;
 170         entry->ref_cnt = 1;
 171         LIST_INSERT_HEAD(first, entry, next);
 172         __atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
 173         DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
 174                 h->name, (void *)entry, entry->ref_cnt);
 175 done:
 176         rte_rwlock_write_unlock(&b->lock);
 177         return entry;
 178 }
 179
 180 int
 181 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
 182 {
 183         uint32_t idx;
 184
 185         if (h->direct_key)
 186                 idx = (uint32_t)(entry->key & h->mask);
 187         else
 188                 idx = rte_hash_crc_8byte(entry->key, 0) & h->mask;
 189
 190         rte_rwlock_write_lock(&h->buckets[idx].lock);
 191         MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
 192         DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
 193                 h->name, (void *)entry, entry->ref_cnt);
 194         if (--entry->ref_cnt) {
 195                 rte_rwlock_write_unlock(&h->buckets[idx].lock);
 196                 return 1;
 197         }
 198         LIST_REMOVE(entry, next);
 199         /* Set to NULL to get rid of removing action for more than once. */
 200         entry->next.le_prev = NULL;
 201         h->cb_remove(h, entry);
 202         rte_rwlock_write_unlock(&h->buckets[idx].lock);
 203         DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
 204                 h->name, (void *)entry);
 205         return 0;
 206 }
 207
 208 void
 209 mlx5_hlist_destroy(struct mlx5_hlist *h)
 210 {
 211         uint32_t idx;
 212         struct mlx5_hlist_entry *entry;
 213
 214         MLX5_ASSERT(h);
 215         for (idx = 0; idx < h->table_sz; ++idx) {
 216                 /* No LIST_FOREACH_SAFE, using while instead. */
 217                 while (!LIST_EMPTY(&h->buckets[idx].head)) {
 218                         entry = LIST_FIRST(&h->buckets[idx].head);
 219                         LIST_REMOVE(entry, next);
 220                         /*
 221                          * The owner of whole element which contains data entry
 222                          * is the user, so it's the user's duty to do the clean
 223                          * up and the free work because someone may not put the
 224                          * hlist entry at the beginning(suggested to locate at
 225                          * the beginning). Or else the default free function
 226                          * will be used.
 227                          */
 228                         h->cb_remove(h, entry);
 229                 }
 230         }
 231         mlx5_free(h);
 232 }
 233
 234 /********************* Cache list ************************/
 235
 236 static struct mlx5_cache_entry *
 237 mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
 238                              struct mlx5_cache_entry *entry __rte_unused,
 239                              void *ctx __rte_unused)
 240 {
 241         return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
 242 }
 243
 244 static void
 245 mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
 246                              struct mlx5_cache_entry *entry)
 247 {
 248         mlx5_free(entry);
 249 }
 250
 251 int
 252 mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
 253                      uint32_t entry_size, void *ctx,
 254                      mlx5_cache_create_cb cb_create,
 255                      mlx5_cache_match_cb cb_match,
 256                      mlx5_cache_remove_cb cb_remove)
 257 {
 258         MLX5_ASSERT(list);
 259         if (!cb_match || (!cb_create ^ !cb_remove))
 260                 return -1;
 261         if (name)
 262                 snprintf(list->name, sizeof(list->name), "%s", name);
 263         list->entry_sz = entry_size;
 264         list->ctx = ctx;
 265         list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
 266         list->cb_match = cb_match;
 267         list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
 268         rte_rwlock_init(&list->lock);
 269         DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
 270         LIST_INIT(&list->head);
 271         return 0;
 272 }
 273
 274 static struct mlx5_cache_entry *
 275 __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 276 {
 277         struct mlx5_cache_entry *entry;
 278
 279         LIST_FOREACH(entry, &list->head, next) {
 280                 if (list->cb_match(list, entry, ctx))
 281                         continue;
 282                 if (reuse) {
 283                         __atomic_add_fetch(&entry->ref_cnt, 1,
 284                                            __ATOMIC_RELAXED);
 285                         DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
 286                                 list->name, (void *)entry, entry->ref_cnt);
 287                 }
 288                 break;
 289         }
 290         return entry;
 291 }
 292
 293 static struct mlx5_cache_entry *
 294 cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 295 {
 296         struct mlx5_cache_entry *entry;
 297
 298         rte_rwlock_read_lock(&list->lock);
 299         entry = __cache_lookup(list, ctx, reuse);
 300         rte_rwlock_read_unlock(&list->lock);
 301         return entry;
 302 }
 303
 304 struct mlx5_cache_entry *
 305 mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
 306 {
 307         return cache_lookup(list, ctx, false);
 308 }
 309
 310 struct mlx5_cache_entry *
 311 mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 312 {
 313         struct mlx5_cache_entry *entry;
 314         uint32_t prev_gen_cnt = 0;
 315
 316         MLX5_ASSERT(list);
 317         prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
 318         /* Lookup with read lock, reuse if found. */
 319         entry = cache_lookup(list, ctx, true);
 320         if (entry)
 321                 return entry;
 322         /* Not found, append with write lock - block read from other threads. */
 323         rte_rwlock_write_lock(&list->lock);
 324         /* If list changed by other threads before lock, search again. */
 325         if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 326                 /* Lookup and reuse w/o read lock. */
 327                 entry = __cache_lookup(list, ctx, true);
 328                 if (entry)
 329                         goto done;
 330         }
 331         entry = list->cb_create(list, entry, ctx);
 332         if (!entry) {
 333                 DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
 334                         list->name, (void *)entry);
 335                 goto done;
 336         }
 337         entry->ref_cnt = 1;
 338         LIST_INSERT_HEAD(&list->head, entry, next);
 339         __atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
 340         __atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 341         DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
 342                 list->name, (void *)entry, entry->ref_cnt);
 343 done:
 344         rte_rwlock_write_unlock(&list->lock);
 345         return entry;
 346 }
 347
 348 int
 349 mlx5_cache_unregister(struct mlx5_cache_list *list,
 350                       struct mlx5_cache_entry *entry)
 351 {
 352         rte_rwlock_write_lock(&list->lock);
 353         MLX5_ASSERT(entry && entry->next.le_prev);
 354         DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
 355                 list->name, (void *)entry, entry->ref_cnt);
 356         if (--entry->ref_cnt) {
 357                 rte_rwlock_write_unlock(&list->lock);
 358                 return 1;
 359         }
 360         __atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
 361         __atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 362         LIST_REMOVE(entry, next);
 363         list->cb_remove(list, entry);
 364         rte_rwlock_write_unlock(&list->lock);
 365         DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
 366                 list->name, (void *)entry);
 367         return 0;
 368 }
 369
 370 void
 371 mlx5_cache_list_destroy(struct mlx5_cache_list *list)
 372 {
 373         struct mlx5_cache_entry *entry;
 374
 375         MLX5_ASSERT(list);
 376         /* no LIST_FOREACH_SAFE, using while instead */
 377         while (!LIST_EMPTY(&list->head)) {
 378                 entry = LIST_FIRST(&list->head);
 379                 LIST_REMOVE(entry, next);
 380                 list->cb_remove(list, entry);
 381                 DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
 382                         list->name, (void *)entry);
 383         }
 384         memset(list, 0, sizeof(*list));
 385 }
 386
 387 uint32_t
 388 mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
 389 {
 390         MLX5_ASSERT(list);
 391         return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
 392 }
 393
 394 /********************* Indexed pool **********************/
 395
 396 static inline void
 397 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 398 {
 399         if (pool->cfg.need_lock)
 400                 rte_spinlock_lock(&pool->lock);
 401 }
 402
 403 static inline void
 404 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 405 {
 406         if (pool->cfg.need_lock)
 407                 rte_spinlock_unlock(&pool->lock);
 408 }
 409
 410 static inline uint32_t
 411 mlx5_trunk_idx_get(struct mlx5_indexed_pool *pool, uint32_t entry_idx)
 412 {
 413         struct mlx5_indexed_pool_config *cfg = &pool->cfg;
 414         uint32_t trunk_idx = 0;
 415         uint32_t i;
 416
 417         if (!cfg->grow_trunk)
 418                 return entry_idx / cfg->trunk_size;
 419         if (entry_idx >= pool->grow_tbl[cfg->grow_trunk - 1]) {
 420                 trunk_idx = (entry_idx - pool->grow_tbl[cfg->grow_trunk - 1]) /
 421                             (cfg->trunk_size << (cfg->grow_shift *
 422                             cfg->grow_trunk)) + cfg->grow_trunk;
 423         } else {
 424                 for (i = 0; i < cfg->grow_trunk; i++) {
 425                         if (entry_idx < pool->grow_tbl[i])
 426                                 break;
 427                 }
 428                 trunk_idx = i;
 429         }
 430         return trunk_idx;
 431 }
 432
 433 static inline uint32_t
 434 mlx5_trunk_size_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx)
 435 {
 436         struct mlx5_indexed_pool_config *cfg = &pool->cfg;
 437
 438         return cfg->trunk_size << (cfg->grow_shift *
 439                (trunk_idx > cfg->grow_trunk ? cfg->grow_trunk : trunk_idx));
 440 }
 441
 442 static inline uint32_t
 443 mlx5_trunk_idx_offset_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx)
 444 {
 445         struct mlx5_indexed_pool_config *cfg = &pool->cfg;
 446         uint32_t offset = 0;
 447
 448         if (!trunk_idx)
 449                 return 0;
 450         if (!cfg->grow_trunk)
 451                 return cfg->trunk_size * trunk_idx;
 452         if (trunk_idx < cfg->grow_trunk)
 453                 offset = pool->grow_tbl[trunk_idx - 1];
 454         else
 455                 offset = pool->grow_tbl[cfg->grow_trunk - 1] +
 456                          (cfg->trunk_size << (cfg->grow_shift *
 457                          cfg->grow_trunk)) * (trunk_idx - cfg->grow_trunk);
 458         return offset;
 459 }
 460
 461 struct mlx5_indexed_pool *
 462 mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 463 {
 464         struct mlx5_indexed_pool *pool;
 465         uint32_t i;
 466
 467         if (!cfg || (!cfg->malloc ^ !cfg->free) ||
 468             (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 469             ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 470                 return NULL;
 471         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool) + cfg->grow_trunk *
 472                            sizeof(pool->grow_tbl[0]), RTE_CACHE_LINE_SIZE,
 473                            SOCKET_ID_ANY);
 474         if (!pool)
 475                 return NULL;
 476         pool->cfg = *cfg;
 477         if (!pool->cfg.trunk_size)
 478                 pool->cfg.trunk_size = MLX5_IPOOL_DEFAULT_TRUNK_SIZE;
 479         if (!cfg->malloc && !cfg->free) {
 480                 pool->cfg.malloc = mlx5_malloc;
 481                 pool->cfg.free = mlx5_free;
 482         }
 483         pool->free_list = TRUNK_INVALID;
 484         if (pool->cfg.need_lock)
 485                 rte_spinlock_init(&pool->lock);
 486         /*
 487          * Initialize the dynamic grow trunk size lookup table to have a quick
 488          * lookup for the trunk entry index offset.
 489          */
 490         for (i = 0; i < cfg->grow_trunk; i++) {
 491                 pool->grow_tbl[i] = cfg->trunk_size << (cfg->grow_shift * i);
 492                 if (i > 0)
 493                         pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 494         }
 495         return pool;
 496 }
 497
 498 static int
 499 mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 500 {
 501         struct mlx5_indexed_trunk *trunk;
 502         struct mlx5_indexed_trunk **trunk_tmp;
 503         struct mlx5_indexed_trunk **p;
 504         size_t trunk_size = 0;
 505         size_t data_size;
 506         size_t bmp_size;
 507         uint32_t idx;
 508
 509         if (pool->n_trunk_valid == TRUNK_MAX_IDX)
 510                 return -ENOMEM;
 511         if (pool->n_trunk_valid == pool->n_trunk) {
 512                 /* No free trunk flags, expand trunk list. */
 513                 int n_grow = pool->n_trunk_valid ? pool->n_trunk :
 514                              RTE_CACHE_LINE_SIZE / sizeof(void *);
 515
 516                 p = pool->cfg.malloc(0, (pool->n_trunk_valid + n_grow) *
 517                                      sizeof(struct mlx5_indexed_trunk *),
 518                                      RTE_CACHE_LINE_SIZE, rte_socket_id());
 519                 if (!p)
 520                         return -ENOMEM;
 521                 if (pool->trunks)
 522                         memcpy(p, pool->trunks, pool->n_trunk_valid *
 523                                sizeof(struct mlx5_indexed_trunk *));
 524                 memset(RTE_PTR_ADD(p, pool->n_trunk_valid * sizeof(void *)), 0,
 525                        n_grow * sizeof(void *));
 526                 trunk_tmp = pool->trunks;
 527                 pool->trunks = p;
 528                 if (trunk_tmp)
 529                         pool->cfg.free(trunk_tmp);
 530                 pool->n_trunk += n_grow;
 531         }
 532         if (!pool->cfg.release_mem_en) {
 533                 idx = pool->n_trunk_valid;
 534         } else {
 535                 /* Find the first available slot in trunk list */
 536                 for (idx = 0; idx < pool->n_trunk; idx++)
 537                         if (pool->trunks[idx] == NULL)
 538                                 break;
 539         }
 540         trunk_size += sizeof(*trunk);
 541         data_size = mlx5_trunk_size_get(pool, idx);
 542         bmp_size = rte_bitmap_get_memory_footprint(data_size);
 543         /* rte_bitmap requires memory cacheline aligned. */
 544         trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
 545         trunk_size += bmp_size;
 546         trunk = pool->cfg.malloc(0, trunk_size,
 547                                  RTE_CACHE_LINE_SIZE, rte_socket_id());
 548         if (!trunk)
 549                 return -ENOMEM;
 550         pool->trunks[idx] = trunk;
 551         trunk->idx = idx;
 552         trunk->free = data_size;
 553         trunk->prev = TRUNK_INVALID;
 554         trunk->next = TRUNK_INVALID;
 555         MLX5_ASSERT(pool->free_list == TRUNK_INVALID);
 556         pool->free_list = idx;
 557         /* Mark all entries as available. */
 558         trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 559                      [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 560                      bmp_size);
 561         MLX5_ASSERT(trunk->bmp);
 562         pool->n_trunk_valid++;
 563 #ifdef POOL_DEBUG
 564         pool->trunk_new++;
 565         pool->trunk_avail++;
 566 #endif
 567         return 0;
 568 }
 569
 570 void *
 571 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 572 {
 573         struct mlx5_indexed_trunk *trunk;
 574         uint64_t slab = 0;
 575         uint32_t iidx = 0;
 576         void *p;
 577
 578         mlx5_ipool_lock(pool);
 579         if (pool->free_list == TRUNK_INVALID) {
 580                 /* If no available trunks, grow new. */
 581                 if (mlx5_ipool_grow(pool)) {
 582                         mlx5_ipool_unlock(pool);
 583                         return NULL;
 584                 }
 585         }
 586         MLX5_ASSERT(pool->free_list != TRUNK_INVALID);
 587         trunk = pool->trunks[pool->free_list];
 588         MLX5_ASSERT(trunk->free);
 589         if (!rte_bitmap_scan(trunk->bmp, &iidx, &slab)) {
 590                 mlx5_ipool_unlock(pool);
 591                 return NULL;
 592         }
 593         MLX5_ASSERT(slab);
 594         iidx += __builtin_ctzll(slab);
 595         MLX5_ASSERT(iidx != UINT32_MAX);
 596         MLX5_ASSERT(iidx < mlx5_trunk_size_get(pool, trunk->idx));
 597         rte_bitmap_clear(trunk->bmp, iidx);
 598         p = &trunk->data[iidx * pool->cfg.size];
 599         /*
 600          * The ipool index should grow continually from small to big,
 601          * some features as metering only accept limited bits of index.
 602          * Random index with MSB set may be rejected.
 603          */
 604         iidx += mlx5_trunk_idx_offset_get(pool, trunk->idx);
 605         iidx += 1; /* non-zero index. */
 606         trunk->free--;
 607 #ifdef POOL_DEBUG
 608         pool->n_entry++;
 609 #endif
 610         if (!trunk->free) {
 611                 /* Full trunk will be removed from free list in imalloc. */
 612                 MLX5_ASSERT(pool->free_list == trunk->idx);
 613                 pool->free_list = trunk->next;
 614                 if (trunk->next != TRUNK_INVALID)
 615                         pool->trunks[trunk->next]->prev = TRUNK_INVALID;
 616                 trunk->prev = TRUNK_INVALID;
 617                 trunk->next = TRUNK_INVALID;
 618 #ifdef POOL_DEBUG
 619                 pool->trunk_empty++;
 620                 pool->trunk_avail--;
 621 #endif
 622         }
 623         *idx = iidx;
 624         mlx5_ipool_unlock(pool);
 625         return p;
 626 }
 627
 628 void *
 629 mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 630 {
 631         void *entry = mlx5_ipool_malloc(pool, idx);
 632
 633         if (entry && pool->cfg.size)
 634                 memset(entry, 0, pool->cfg.size);
 635         return entry;
 636 }
 637
 638 void
 639 mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 640 {
 641         struct mlx5_indexed_trunk *trunk;
 642         uint32_t trunk_idx;
 643         uint32_t entry_idx;
 644
 645         if (!idx)
 646                 return;
 647         idx -= 1;
 648         mlx5_ipool_lock(pool);
 649         trunk_idx = mlx5_trunk_idx_get(pool, idx);
 650         if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) ||
 651             (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk))
 652                 goto out;
 653         trunk = pool->trunks[trunk_idx];
 654         if (!trunk)
 655                 goto out;
 656         entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx);
 657         if (trunk_idx != trunk->idx ||
 658             rte_bitmap_get(trunk->bmp, entry_idx))
 659                 goto out;
 660         rte_bitmap_set(trunk->bmp, entry_idx);
 661         trunk->free++;
 662         if (pool->cfg.release_mem_en && trunk->free == mlx5_trunk_size_get
 663            (pool, trunk->idx)) {
 664                 if (pool->free_list == trunk->idx)
 665                         pool->free_list = trunk->next;
 666                 if (trunk->next != TRUNK_INVALID)
 667                         pool->trunks[trunk->next]->prev = trunk->prev;
 668                 if (trunk->prev != TRUNK_INVALID)
 669                         pool->trunks[trunk->prev]->next = trunk->next;
 670                 pool->cfg.free(trunk);
 671                 pool->trunks[trunk_idx] = NULL;
 672                 pool->n_trunk_valid--;
 673 #ifdef POOL_DEBUG
 674                 pool->trunk_avail--;
 675                 pool->trunk_free++;
 676 #endif
 677                 if (pool->n_trunk_valid == 0) {
 678                         pool->cfg.free(pool->trunks);
 679                         pool->trunks = NULL;
 680                         pool->n_trunk = 0;
 681                 }
 682         } else if (trunk->free == 1) {
 683                 /* Put into free trunk list head. */
 684                 MLX5_ASSERT(pool->free_list != trunk->idx);
 685                 trunk->next = pool->free_list;
 686                 trunk->prev = TRUNK_INVALID;
 687                 if (pool->free_list != TRUNK_INVALID)
 688                         pool->trunks[pool->free_list]->prev = trunk->idx;
 689                 pool->free_list = trunk->idx;
 690 #ifdef POOL_DEBUG
 691                 pool->trunk_empty--;
 692                 pool->trunk_avail++;
 693 #endif
 694         }
 695 #ifdef POOL_DEBUG
 696         pool->n_entry--;
 697 #endif
 698 out:
 699         mlx5_ipool_unlock(pool);
 700 }
 701
 702 void *
 703 mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 704 {
 705         struct mlx5_indexed_trunk *trunk;
 706         void *p = NULL;
 707         uint32_t trunk_idx;
 708         uint32_t entry_idx;
 709
 710         if (!idx)
 711                 return NULL;
 712         idx -= 1;
 713         mlx5_ipool_lock(pool);
 714         trunk_idx = mlx5_trunk_idx_get(pool, idx);
 715         if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) ||
 716             (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk))
 717                 goto out;
 718         trunk = pool->trunks[trunk_idx];
 719         if (!trunk)
 720                 goto out;
 721         entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx);
 722         if (trunk_idx != trunk->idx ||
 723             rte_bitmap_get(trunk->bmp, entry_idx))
 724                 goto out;
 725         p = &trunk->data[entry_idx * pool->cfg.size];
 726 out:
 727         mlx5_ipool_unlock(pool);
 728         return p;
 729 }
 730
 731 int
 732 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 733 {
 734         struct mlx5_indexed_trunk **trunks;
 735         uint32_t i;
 736
 737         MLX5_ASSERT(pool);
 738         mlx5_ipool_lock(pool);
 739         trunks = pool->trunks;
 740         for (i = 0; i < pool->n_trunk; i++) {
 741                 if (trunks[i])
 742                         pool->cfg.free(trunks[i]);
 743         }
 744         if (!pool->trunks)
 745                 pool->cfg.free(pool->trunks);
 746         mlx5_ipool_unlock(pool);
 747         mlx5_free(pool);
 748         return 0;
 749 }
 750
 751 void
 752 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 753 {
 754         printf("Pool %s entry size %u, trunks %u, %d entry per trunk, "
 755                "total: %d\n",
 756                pool->cfg.type, pool->cfg.size, pool->n_trunk_valid,
 757                pool->cfg.trunk_size, pool->n_trunk_valid);
 758 #ifdef POOL_DEBUG
 759         printf("Pool %s entry %u, trunk alloc %u, empty: %u, "
 760                "available %u free %u\n",
 761                pool->cfg.type, pool->n_entry, pool->trunk_new,
 762                pool->trunk_empty, pool->trunk_avail, pool->trunk_free);
 763 #endif
 764 }
 765
 766 struct mlx5_l3t_tbl *
 767 mlx5_l3t_create(enum mlx5_l3t_type type)
 768 {
 769         struct mlx5_l3t_tbl *tbl;
 770         struct mlx5_indexed_pool_config l3t_ip_cfg = {
 771                 .trunk_size = 16,
 772                 .grow_trunk = 6,
 773                 .grow_shift = 1,
 774                 .need_lock = 0,
 775                 .release_mem_en = 1,
 776                 .malloc = mlx5_malloc,
 777                 .free = mlx5_free,
 778         };
 779
 780         if (type >= MLX5_L3T_TYPE_MAX) {
 781                 rte_errno = EINVAL;
 782                 return NULL;
 783         }
 784         tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_l3t_tbl), 1,
 785                           SOCKET_ID_ANY);
 786         if (!tbl) {
 787                 rte_errno = ENOMEM;
 788                 return NULL;
 789         }
 790         tbl->type = type;
 791         switch (type) {
 792         case MLX5_L3T_TYPE_WORD:
 793                 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_word);
 794                 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_w";
 795                 break;
 796         case MLX5_L3T_TYPE_DWORD:
 797                 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_dword);
 798                 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_dw";
 799                 break;
 800         case MLX5_L3T_TYPE_QWORD:
 801                 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_qword);
 802                 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_qw";
 803                 break;
 804         default:
 805                 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_ptr);
 806                 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_tpr";
 807                 break;
 808         }
 809         rte_spinlock_init(&tbl->sl);
 810         tbl->eip = mlx5_ipool_create(&l3t_ip_cfg);
 811         if (!tbl->eip) {
 812                 rte_errno = ENOMEM;
 813                 mlx5_free(tbl);
 814                 tbl = NULL;
 815         }
 816         return tbl;
 817 }
 818
 819 void
 820 mlx5_l3t_destroy(struct mlx5_l3t_tbl *tbl)
 821 {
 822         struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
 823         uint32_t i, j;
 824
 825         if (!tbl)
 826                 return;
 827         g_tbl = tbl->tbl;
 828         if (g_tbl) {
 829                 for (i = 0; i < MLX5_L3T_GT_SIZE; i++) {
 830                         m_tbl = g_tbl->tbl[i];
 831                         if (!m_tbl)
 832                                 continue;
 833                         for (j = 0; j < MLX5_L3T_MT_SIZE; j++) {
 834                                 if (!m_tbl->tbl[j])
 835                                         continue;
 836                                 MLX5_ASSERT(!((struct mlx5_l3t_entry_word *)
 837                                             m_tbl->tbl[j])->ref_cnt);
 838                                 mlx5_ipool_free(tbl->eip,
 839                                                 ((struct mlx5_l3t_entry_word *)
 840                                                 m_tbl->tbl[j])->idx);
 841                                 m_tbl->tbl[j] = 0;
 842                                 if (!(--m_tbl->ref_cnt))
 843                                         break;
 844                         }
 845                         MLX5_ASSERT(!m_tbl->ref_cnt);
 846                         mlx5_free(g_tbl->tbl[i]);
 847                         g_tbl->tbl[i] = 0;
 848                         if (!(--g_tbl->ref_cnt))
 849                                 break;
 850                 }
 851                 MLX5_ASSERT(!g_tbl->ref_cnt);
 852                 mlx5_free(tbl->tbl);
 853                 tbl->tbl = 0;
 854         }
 855         mlx5_ipool_destroy(tbl->eip);
 856         mlx5_free(tbl);
 857 }
 858
 859 static int32_t
 860 __l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 861                 union mlx5_l3t_data *data)
 862 {
 863         struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
 864         struct mlx5_l3t_entry_word *w_e_tbl;
 865         struct mlx5_l3t_entry_dword *dw_e_tbl;
 866         struct mlx5_l3t_entry_qword *qw_e_tbl;
 867         struct mlx5_l3t_entry_ptr *ptr_e_tbl;
 868         void *e_tbl;
 869         uint32_t entry_idx;
 870
 871         g_tbl = tbl->tbl;
 872         if (!g_tbl)
 873                 return -1;
 874         m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK];
 875         if (!m_tbl)
 876                 return -1;
 877         e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK];
 878         if (!e_tbl)
 879                 return -1;
 880         entry_idx = idx & MLX5_L3T_ET_MASK;
 881         switch (tbl->type) {
 882         case MLX5_L3T_TYPE_WORD:
 883                 w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
 884                 data->word = w_e_tbl->entry[entry_idx].data;
 885                 if (w_e_tbl->entry[entry_idx].data)
 886                         w_e_tbl->entry[entry_idx].ref_cnt++;
 887                 break;
 888         case MLX5_L3T_TYPE_DWORD:
 889                 dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
 890                 data->dword = dw_e_tbl->entry[entry_idx].data;
 891                 if (dw_e_tbl->entry[entry_idx].data)
 892                         dw_e_tbl->entry[entry_idx].ref_cnt++;
 893                 break;
 894         case MLX5_L3T_TYPE_QWORD:
 895                 qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
 896                 data->qword = qw_e_tbl->entry[entry_idx].data;
 897                 if (qw_e_tbl->entry[entry_idx].data)
 898                         qw_e_tbl->entry[entry_idx].ref_cnt++;
 899                 break;
 900         default:
 901                 ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
 902                 data->ptr = ptr_e_tbl->entry[entry_idx].data;
 903                 if (ptr_e_tbl->entry[entry_idx].data)
 904                         ptr_e_tbl->entry[entry_idx].ref_cnt++;
 905                 break;
 906         }
 907         return 0;
 908 }
 909
 910 int32_t
 911 mlx5_l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 912                    union mlx5_l3t_data *data)
 913 {
 914         int ret;
 915
 916         rte_spinlock_lock(&tbl->sl);
 917         ret = __l3t_get_entry(tbl, idx, data);
 918         rte_spinlock_unlock(&tbl->sl);
 919         return ret;
 920 }
 921
 922 int32_t
 923 mlx5_l3t_clear_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx)
 924 {
 925         struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
 926         struct mlx5_l3t_entry_word *w_e_tbl;
 927         struct mlx5_l3t_entry_dword *dw_e_tbl;
 928         struct mlx5_l3t_entry_qword *qw_e_tbl;
 929         struct mlx5_l3t_entry_ptr *ptr_e_tbl;
 930         void *e_tbl;
 931         uint32_t entry_idx;
 932         uint64_t ref_cnt;
 933         int32_t ret = -1;
 934
 935         rte_spinlock_lock(&tbl->sl);
 936         g_tbl = tbl->tbl;
 937         if (!g_tbl)
 938                 goto out;
 939         m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK];
 940         if (!m_tbl)
 941                 goto out;
 942         e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK];
 943         if (!e_tbl)
 944                 goto out;
 945         entry_idx = idx & MLX5_L3T_ET_MASK;
 946         switch (tbl->type) {
 947         case MLX5_L3T_TYPE_WORD:
 948                 w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
 949                 MLX5_ASSERT(w_e_tbl->entry[entry_idx].ref_cnt);
 950                 ret = --w_e_tbl->entry[entry_idx].ref_cnt;
 951                 if (ret)
 952                         goto out;
 953                 w_e_tbl->entry[entry_idx].data = 0;
 954                 ref_cnt = --w_e_tbl->ref_cnt;
 955                 break;
 956         case MLX5_L3T_TYPE_DWORD:
 957                 dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
 958                 MLX5_ASSERT(dw_e_tbl->entry[entry_idx].ref_cnt);
 959                 ret = --dw_e_tbl->entry[entry_idx].ref_cnt;
 960                 if (ret)
 961                         goto out;
 962                 dw_e_tbl->entry[entry_idx].data = 0;
 963                 ref_cnt = --dw_e_tbl->ref_cnt;
 964                 break;
 965         case MLX5_L3T_TYPE_QWORD:
 966                 qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
 967                 MLX5_ASSERT(qw_e_tbl->entry[entry_idx].ref_cnt);
 968                 ret = --qw_e_tbl->entry[entry_idx].ref_cnt;
 969                 if (ret)
 970                         goto out;
 971                 qw_e_tbl->entry[entry_idx].data = 0;
 972                 ref_cnt = --qw_e_tbl->ref_cnt;
 973                 break;
 974         default:
 975                 ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
 976                 MLX5_ASSERT(ptr_e_tbl->entry[entry_idx].ref_cnt);
 977                 ret = --ptr_e_tbl->entry[entry_idx].ref_cnt;
 978                 if (ret)
 979                         goto out;
 980                 ptr_e_tbl->entry[entry_idx].data = NULL;
 981                 ref_cnt = --ptr_e_tbl->ref_cnt;
 982                 break;
 983         }
 984         if (!ref_cnt) {
 985                 mlx5_ipool_free(tbl->eip,
 986                                 ((struct mlx5_l3t_entry_word *)e_tbl)->idx);
 987                 m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK] =
 988                                                                         NULL;
 989                 if (!(--m_tbl->ref_cnt)) {
 990                         mlx5_free(m_tbl);
 991                         g_tbl->tbl
 992                         [(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK] = NULL;
 993                         if (!(--g_tbl->ref_cnt)) {
 994                                 mlx5_free(g_tbl);
 995                                 tbl->tbl = 0;
 996                         }
 997                 }
 998         }
 999 out:
1000         rte_spinlock_unlock(&tbl->sl);
1001         return ret;
1002 }
1003
1004 static int32_t
1005 __l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
1006                 union mlx5_l3t_data *data)
1007 {
1008         struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
1009         struct mlx5_l3t_entry_word *w_e_tbl;
1010         struct mlx5_l3t_entry_dword *dw_e_tbl;
1011         struct mlx5_l3t_entry_qword *qw_e_tbl;
1012         struct mlx5_l3t_entry_ptr *ptr_e_tbl;
1013         void *e_tbl;
1014         uint32_t entry_idx, tbl_idx = 0;
1015
1016         /* Check the global table, create it if empty. */
1017         g_tbl = tbl->tbl;
1018         if (!g_tbl) {
1019                 g_tbl = mlx5_malloc(MLX5_MEM_ZERO,
1020                                     sizeof(struct mlx5_l3t_level_tbl) +
1021                                     sizeof(void *) * MLX5_L3T_GT_SIZE, 1,
1022                                     SOCKET_ID_ANY);
1023                 if (!g_tbl) {
1024                         rte_errno = ENOMEM;
1025                         return -1;
1026                 }
1027                 tbl->tbl = g_tbl;
1028         }
1029         /*
1030          * Check the middle table, create it if empty. Ref_cnt will be
1031          * increased if new sub table created.
1032          */
1033         m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK];
1034         if (!m_tbl) {
1035                 m_tbl = mlx5_malloc(MLX5_MEM_ZERO,
1036                                     sizeof(struct mlx5_l3t_level_tbl) +
1037                                     sizeof(void *) * MLX5_L3T_MT_SIZE, 1,
1038                                     SOCKET_ID_ANY);
1039                 if (!m_tbl) {
1040                         rte_errno = ENOMEM;
1041                         return -1;
1042                 }
1043                 g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK] =
1044                                                                         m_tbl;
1045                 g_tbl->ref_cnt++;
1046         }
1047         /*
1048          * Check the entry table, create it if empty. Ref_cnt will be
1049          * increased if new sub entry table created.
1050          */
1051         e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK];
1052         if (!e_tbl) {
1053                 e_tbl = mlx5_ipool_zmalloc(tbl->eip, &tbl_idx);
1054                 if (!e_tbl) {
1055                         rte_errno = ENOMEM;
1056                         return -1;
1057                 }
1058                 ((struct mlx5_l3t_entry_word *)e_tbl)->idx = tbl_idx;
1059                 m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK] =
1060                                                                         e_tbl;
1061                 m_tbl->ref_cnt++;
1062         }
1063         entry_idx = idx & MLX5_L3T_ET_MASK;
1064         switch (tbl->type) {
1065         case MLX5_L3T_TYPE_WORD:
1066                 w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
1067                 if (w_e_tbl->entry[entry_idx].data) {
1068                         data->word = w_e_tbl->entry[entry_idx].data;
1069                         w_e_tbl->entry[entry_idx].ref_cnt++;
1070                         rte_errno = EEXIST;
1071                         return -1;
1072                 }
1073                 w_e_tbl->entry[entry_idx].data = data->word;
1074                 w_e_tbl->entry[entry_idx].ref_cnt = 1;
1075                 w_e_tbl->ref_cnt++;
1076                 break;
1077         case MLX5_L3T_TYPE_DWORD:
1078                 dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
1079                 if (dw_e_tbl->entry[entry_idx].data) {
1080                         data->dword = dw_e_tbl->entry[entry_idx].data;
1081                         dw_e_tbl->entry[entry_idx].ref_cnt++;
1082                         rte_errno = EEXIST;
1083                         return -1;
1084                 }
1085                 dw_e_tbl->entry[entry_idx].data = data->dword;
1086                 dw_e_tbl->entry[entry_idx].ref_cnt = 1;
1087                 dw_e_tbl->ref_cnt++;
1088                 break;
1089         case MLX5_L3T_TYPE_QWORD:
1090                 qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
1091                 if (qw_e_tbl->entry[entry_idx].data) {
1092                         data->qword = qw_e_tbl->entry[entry_idx].data;
1093                         qw_e_tbl->entry[entry_idx].ref_cnt++;
1094                         rte_errno = EEXIST;
1095                         return -1;
1096                 }
1097                 qw_e_tbl->entry[entry_idx].data = data->qword;
1098                 qw_e_tbl->entry[entry_idx].ref_cnt = 1;
1099                 qw_e_tbl->ref_cnt++;
1100                 break;
1101         default:
1102                 ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
1103                 if (ptr_e_tbl->entry[entry_idx].data) {
1104                         data->ptr = ptr_e_tbl->entry[entry_idx].data;
1105                         ptr_e_tbl->entry[entry_idx].ref_cnt++;
1106                         rte_errno = EEXIST;
1107                         return -1;
1108                 }
1109                 ptr_e_tbl->entry[entry_idx].data = data->ptr;
1110                 ptr_e_tbl->entry[entry_idx].ref_cnt = 1;
1111                 ptr_e_tbl->ref_cnt++;
1112                 break;
1113         }
1114         return 0;
1115 }
1116
1117 int32_t
1118 mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
1119                    union mlx5_l3t_data *data)
1120 {
1121         int ret;
1122
1123         rte_spinlock_lock(&tbl->sl);
1124         ret = __l3t_set_entry(tbl, idx, data);
1125         rte_spinlock_unlock(&tbl->sl);
1126         return ret;
1127 }
1128
1129 int32_t
1130 mlx5_l3t_prepare_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
1131                        union mlx5_l3t_data *data,
1132                        mlx5_l3t_alloc_callback_fn cb, void *ctx)
1133 {
1134         int32_t ret;
1135
1136         rte_spinlock_lock(&tbl->sl);
1137         /* Check if entry data is ready. */
1138         ret = __l3t_get_entry(tbl, idx, data);
1139         if (!ret) {
1140                 switch (tbl->type) {
1141                 case MLX5_L3T_TYPE_WORD:
1142                         if (data->word)
1143                                 goto out;
1144                         break;
1145                 case MLX5_L3T_TYPE_DWORD:
1146                         if (data->dword)
1147                                 goto out;
1148                         break;
1149                 case MLX5_L3T_TYPE_QWORD:
1150                         if (data->qword)
1151                                 goto out;
1152                         break;
1153                 default:
1154                         if (data->ptr)
1155                                 goto out;
1156                         break;
1157                 }
1158         }
1159         /* Entry data is not ready, use user callback to create it. */
1160         ret = cb(ctx, data);
1161         if (ret)
1162                 goto out;
1163         /* Save the new allocated data to entry. */
1164         ret = __l3t_set_entry(tbl, idx, data);
1165 out:
1166         rte_spinlock_unlock(&tbl->sl);
1167         return ret;
1168 }