drivers/net/mlx5/mlx5_utils.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <rte_malloc.h>
   6
   7 #include <mlx5_malloc.h>
   8
   9 #include "mlx5_utils.h"
  10
  11
  12 /********************* Cache list ************************/
  13
  14 static struct mlx5_cache_entry *
  15 mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
  16                              struct mlx5_cache_entry *entry __rte_unused,
  17                              void *ctx __rte_unused)
  18 {
  19         return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
  20 }
  21
  22 static void
  23 mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
  24                              struct mlx5_cache_entry *entry)
  25 {
  26         mlx5_free(entry);
  27 }
  28
  29 int
  30 mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
  31                      uint32_t entry_size, void *ctx,
  32                      mlx5_cache_create_cb cb_create,
  33                      mlx5_cache_match_cb cb_match,
  34                      mlx5_cache_remove_cb cb_remove)
  35 {
  36         MLX5_ASSERT(list);
  37         if (!cb_match || (!cb_create ^ !cb_remove))
  38                 return -1;
  39         if (name)
  40                 snprintf(list->name, sizeof(list->name), "%s", name);
  41         list->entry_sz = entry_size;
  42         list->ctx = ctx;
  43         list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
  44         list->cb_match = cb_match;
  45         list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
  46         rte_rwlock_init(&list->lock);
  47         DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
  48         LIST_INIT(&list->head);
  49         return 0;
  50 }
  51
  52 static struct mlx5_cache_entry *
  53 __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
  54 {
  55         struct mlx5_cache_entry *entry;
  56
  57         LIST_FOREACH(entry, &list->head, next) {
  58                 if (list->cb_match(list, entry, ctx))
  59                         continue;
  60                 if (reuse) {
  61                         __atomic_add_fetch(&entry->ref_cnt, 1,
  62                                            __ATOMIC_RELAXED);
  63                         DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
  64                                 list->name, (void *)entry, entry->ref_cnt);
  65                 }
  66                 break;
  67         }
  68         return entry;
  69 }
  70
  71 static struct mlx5_cache_entry *
  72 cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
  73 {
  74         struct mlx5_cache_entry *entry;
  75
  76         rte_rwlock_read_lock(&list->lock);
  77         entry = __cache_lookup(list, ctx, reuse);
  78         rte_rwlock_read_unlock(&list->lock);
  79         return entry;
  80 }
  81
  82 struct mlx5_cache_entry *
  83 mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
  84 {
  85         return cache_lookup(list, ctx, false);
  86 }
  87
  88 struct mlx5_cache_entry *
  89 mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
  90 {
  91         struct mlx5_cache_entry *entry;
  92         uint32_t prev_gen_cnt = 0;
  93
  94         MLX5_ASSERT(list);
  95         prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
  96         /* Lookup with read lock, reuse if found. */
  97         entry = cache_lookup(list, ctx, true);
  98         if (entry)
  99                 return entry;
 100         /* Not found, append with write lock - block read from other threads. */
 101         rte_rwlock_write_lock(&list->lock);
 102         /* If list changed by other threads before lock, search again. */
 103         if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 104                 /* Lookup and reuse w/o read lock. */
 105                 entry = __cache_lookup(list, ctx, true);
 106                 if (entry)
 107                         goto done;
 108         }
 109         entry = list->cb_create(list, entry, ctx);
 110         if (!entry) {
 111                 DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
 112                         list->name, (void *)entry);
 113                 goto done;
 114         }
 115         entry->ref_cnt = 1;
 116         LIST_INSERT_HEAD(&list->head, entry, next);
 117         __atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
 118         __atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 119         DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
 120                 list->name, (void *)entry, entry->ref_cnt);
 121 done:
 122         rte_rwlock_write_unlock(&list->lock);
 123         return entry;
 124 }
 125
 126 int
 127 mlx5_cache_unregister(struct mlx5_cache_list *list,
 128                       struct mlx5_cache_entry *entry)
 129 {
 130         rte_rwlock_write_lock(&list->lock);
 131         MLX5_ASSERT(entry && entry->next.le_prev);
 132         DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
 133                 list->name, (void *)entry, entry->ref_cnt);
 134         if (--entry->ref_cnt) {
 135                 rte_rwlock_write_unlock(&list->lock);
 136                 return 1;
 137         }
 138         __atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
 139         __atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 140         LIST_REMOVE(entry, next);
 141         list->cb_remove(list, entry);
 142         rte_rwlock_write_unlock(&list->lock);
 143         DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
 144                 list->name, (void *)entry);
 145         return 0;
 146 }
 147
 148 void
 149 mlx5_cache_list_destroy(struct mlx5_cache_list *list)
 150 {
 151         struct mlx5_cache_entry *entry;
 152
 153         MLX5_ASSERT(list);
 154         /* no LIST_FOREACH_SAFE, using while instead */
 155         while (!LIST_EMPTY(&list->head)) {
 156                 entry = LIST_FIRST(&list->head);
 157                 LIST_REMOVE(entry, next);
 158                 list->cb_remove(list, entry);
 159                 DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
 160                         list->name, (void *)entry);
 161         }
 162         memset(list, 0, sizeof(*list));
 163 }
 164
 165 uint32_t
 166 mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
 167 {
 168         MLX5_ASSERT(list);
 169         return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
 170 }
 171
 172 /********************* Indexed pool **********************/
 173
 174 static inline void
 175 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 176 {
 177         if (pool->cfg.need_lock)
 178                 rte_spinlock_lock(&pool->lock);
 179 }
 180
 181 static inline void
 182 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 183 {
 184         if (pool->cfg.need_lock)
 185                 rte_spinlock_unlock(&pool->lock);
 186 }
 187
 188 static inline uint32_t
 189 mlx5_trunk_idx_get(struct mlx5_indexed_pool *pool, uint32_t entry_idx)
 190 {
 191         struct mlx5_indexed_pool_config *cfg = &pool->cfg;
 192         uint32_t trunk_idx = 0;
 193         uint32_t i;
 194
 195         if (!cfg->grow_trunk)
 196                 return entry_idx / cfg->trunk_size;
 197         if (entry_idx >= pool->grow_tbl[cfg->grow_trunk - 1]) {
 198                 trunk_idx = (entry_idx - pool->grow_tbl[cfg->grow_trunk - 1]) /
 199                             (cfg->trunk_size << (cfg->grow_shift *
 200                             cfg->grow_trunk)) + cfg->grow_trunk;
 201         } else {
 202                 for (i = 0; i < cfg->grow_trunk; i++) {
 203                         if (entry_idx < pool->grow_tbl[i])
 204                                 break;
 205                 }
 206                 trunk_idx = i;
 207         }
 208         return trunk_idx;
 209 }
 210
 211 static inline uint32_t
 212 mlx5_trunk_size_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx)
 213 {
 214         struct mlx5_indexed_pool_config *cfg = &pool->cfg;
 215
 216         return cfg->trunk_size << (cfg->grow_shift *
 217                (trunk_idx > cfg->grow_trunk ? cfg->grow_trunk : trunk_idx));
 218 }
 219
 220 static inline uint32_t
 221 mlx5_trunk_idx_offset_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx)
 222 {
 223         struct mlx5_indexed_pool_config *cfg = &pool->cfg;
 224         uint32_t offset = 0;
 225
 226         if (!trunk_idx)
 227                 return 0;
 228         if (!cfg->grow_trunk)
 229                 return cfg->trunk_size * trunk_idx;
 230         if (trunk_idx < cfg->grow_trunk)
 231                 offset = pool->grow_tbl[trunk_idx - 1];
 232         else
 233                 offset = pool->grow_tbl[cfg->grow_trunk - 1] +
 234                          (cfg->trunk_size << (cfg->grow_shift *
 235                          cfg->grow_trunk)) * (trunk_idx - cfg->grow_trunk);
 236         return offset;
 237 }
 238
 239 struct mlx5_indexed_pool *
 240 mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 241 {
 242         struct mlx5_indexed_pool *pool;
 243         uint32_t i;
 244
 245         if (!cfg || (!cfg->malloc ^ !cfg->free) ||
 246             (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 247             ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 248                 return NULL;
 249         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool) + cfg->grow_trunk *
 250                            sizeof(pool->grow_tbl[0]), RTE_CACHE_LINE_SIZE,
 251                            SOCKET_ID_ANY);
 252         if (!pool)
 253                 return NULL;
 254         pool->cfg = *cfg;
 255         if (!pool->cfg.trunk_size)
 256                 pool->cfg.trunk_size = MLX5_IPOOL_DEFAULT_TRUNK_SIZE;
 257         if (!cfg->malloc && !cfg->free) {
 258                 pool->cfg.malloc = mlx5_malloc;
 259                 pool->cfg.free = mlx5_free;
 260         }
 261         pool->free_list = TRUNK_INVALID;
 262         if (pool->cfg.need_lock)
 263                 rte_spinlock_init(&pool->lock);
 264         /*
 265          * Initialize the dynamic grow trunk size lookup table to have a quick
 266          * lookup for the trunk entry index offset.
 267          */
 268         for (i = 0; i < cfg->grow_trunk; i++) {
 269                 pool->grow_tbl[i] = cfg->trunk_size << (cfg->grow_shift * i);
 270                 if (i > 0)
 271                         pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 272         }
 273         return pool;
 274 }
 275
 276 static int
 277 mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 278 {
 279         struct mlx5_indexed_trunk *trunk;
 280         struct mlx5_indexed_trunk **trunk_tmp;
 281         struct mlx5_indexed_trunk **p;
 282         size_t trunk_size = 0;
 283         size_t data_size;
 284         size_t bmp_size;
 285         uint32_t idx;
 286
 287         if (pool->n_trunk_valid == TRUNK_MAX_IDX)
 288                 return -ENOMEM;
 289         if (pool->n_trunk_valid == pool->n_trunk) {
 290                 /* No free trunk flags, expand trunk list. */
 291                 int n_grow = pool->n_trunk_valid ? pool->n_trunk :
 292                              RTE_CACHE_LINE_SIZE / sizeof(void *);
 293
 294                 p = pool->cfg.malloc(0, (pool->n_trunk_valid + n_grow) *
 295                                      sizeof(struct mlx5_indexed_trunk *),
 296                                      RTE_CACHE_LINE_SIZE, rte_socket_id());
 297                 if (!p)
 298                         return -ENOMEM;
 299                 if (pool->trunks)
 300                         memcpy(p, pool->trunks, pool->n_trunk_valid *
 301                                sizeof(struct mlx5_indexed_trunk *));
 302                 memset(RTE_PTR_ADD(p, pool->n_trunk_valid * sizeof(void *)), 0,
 303                        n_grow * sizeof(void *));
 304                 trunk_tmp = pool->trunks;
 305                 pool->trunks = p;
 306                 if (trunk_tmp)
 307                         pool->cfg.free(trunk_tmp);
 308                 pool->n_trunk += n_grow;
 309         }
 310         if (!pool->cfg.release_mem_en) {
 311                 idx = pool->n_trunk_valid;
 312         } else {
 313                 /* Find the first available slot in trunk list */
 314                 for (idx = 0; idx < pool->n_trunk; idx++)
 315                         if (pool->trunks[idx] == NULL)
 316                                 break;
 317         }
 318         trunk_size += sizeof(*trunk);
 319         data_size = mlx5_trunk_size_get(pool, idx);
 320         bmp_size = rte_bitmap_get_memory_footprint(data_size);
 321         /* rte_bitmap requires memory cacheline aligned. */
 322         trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
 323         trunk_size += bmp_size;
 324         trunk = pool->cfg.malloc(0, trunk_size,
 325                                  RTE_CACHE_LINE_SIZE, rte_socket_id());
 326         if (!trunk)
 327                 return -ENOMEM;
 328         pool->trunks[idx] = trunk;
 329         trunk->idx = idx;
 330         trunk->free = data_size;
 331         trunk->prev = TRUNK_INVALID;
 332         trunk->next = TRUNK_INVALID;
 333         MLX5_ASSERT(pool->free_list == TRUNK_INVALID);
 334         pool->free_list = idx;
 335         /* Mark all entries as available. */
 336         trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 337                      [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 338                      bmp_size);
 339         MLX5_ASSERT(trunk->bmp);
 340         pool->n_trunk_valid++;
 341 #ifdef POOL_DEBUG
 342         pool->trunk_new++;
 343         pool->trunk_avail++;
 344 #endif
 345         return 0;
 346 }
 347
 348 void *
 349 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 350 {
 351         struct mlx5_indexed_trunk *trunk;
 352         uint64_t slab = 0;
 353         uint32_t iidx = 0;
 354         void *p;
 355
 356         mlx5_ipool_lock(pool);
 357         if (pool->free_list == TRUNK_INVALID) {
 358                 /* If no available trunks, grow new. */
 359                 if (mlx5_ipool_grow(pool)) {
 360                         mlx5_ipool_unlock(pool);
 361                         return NULL;
 362                 }
 363         }
 364         MLX5_ASSERT(pool->free_list != TRUNK_INVALID);
 365         trunk = pool->trunks[pool->free_list];
 366         MLX5_ASSERT(trunk->free);
 367         if (!rte_bitmap_scan(trunk->bmp, &iidx, &slab)) {
 368                 mlx5_ipool_unlock(pool);
 369                 return NULL;
 370         }
 371         MLX5_ASSERT(slab);
 372         iidx += __builtin_ctzll(slab);
 373         MLX5_ASSERT(iidx != UINT32_MAX);
 374         MLX5_ASSERT(iidx < mlx5_trunk_size_get(pool, trunk->idx));
 375         rte_bitmap_clear(trunk->bmp, iidx);
 376         p = &trunk->data[iidx * pool->cfg.size];
 377         /*
 378          * The ipool index should grow continually from small to big,
 379          * some features as metering only accept limited bits of index.
 380          * Random index with MSB set may be rejected.
 381          */
 382         iidx += mlx5_trunk_idx_offset_get(pool, trunk->idx);
 383         iidx += 1; /* non-zero index. */
 384         trunk->free--;
 385 #ifdef POOL_DEBUG
 386         pool->n_entry++;
 387 #endif
 388         if (!trunk->free) {
 389                 /* Full trunk will be removed from free list in imalloc. */
 390                 MLX5_ASSERT(pool->free_list == trunk->idx);
 391                 pool->free_list = trunk->next;
 392                 if (trunk->next != TRUNK_INVALID)
 393                         pool->trunks[trunk->next]->prev = TRUNK_INVALID;
 394                 trunk->prev = TRUNK_INVALID;
 395                 trunk->next = TRUNK_INVALID;
 396 #ifdef POOL_DEBUG
 397                 pool->trunk_empty++;
 398                 pool->trunk_avail--;
 399 #endif
 400         }
 401         *idx = iidx;
 402         mlx5_ipool_unlock(pool);
 403         return p;
 404 }
 405
 406 void *
 407 mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 408 {
 409         void *entry = mlx5_ipool_malloc(pool, idx);
 410
 411         if (entry && pool->cfg.size)
 412                 memset(entry, 0, pool->cfg.size);
 413         return entry;
 414 }
 415
 416 void
 417 mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 418 {
 419         struct mlx5_indexed_trunk *trunk;
 420         uint32_t trunk_idx;
 421         uint32_t entry_idx;
 422
 423         if (!idx)
 424                 return;
 425         idx -= 1;
 426         mlx5_ipool_lock(pool);
 427         trunk_idx = mlx5_trunk_idx_get(pool, idx);
 428         if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) ||
 429             (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk))
 430                 goto out;
 431         trunk = pool->trunks[trunk_idx];
 432         if (!trunk)
 433                 goto out;
 434         entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx);
 435         if (trunk_idx != trunk->idx ||
 436             rte_bitmap_get(trunk->bmp, entry_idx))
 437                 goto out;
 438         rte_bitmap_set(trunk->bmp, entry_idx);
 439         trunk->free++;
 440         if (pool->cfg.release_mem_en && trunk->free == mlx5_trunk_size_get
 441            (pool, trunk->idx)) {
 442                 if (pool->free_list == trunk->idx)
 443                         pool->free_list = trunk->next;
 444                 if (trunk->next != TRUNK_INVALID)
 445                         pool->trunks[trunk->next]->prev = trunk->prev;
 446                 if (trunk->prev != TRUNK_INVALID)
 447                         pool->trunks[trunk->prev]->next = trunk->next;
 448                 pool->cfg.free(trunk);
 449                 pool->trunks[trunk_idx] = NULL;
 450                 pool->n_trunk_valid--;
 451 #ifdef POOL_DEBUG
 452                 pool->trunk_avail--;
 453                 pool->trunk_free++;
 454 #endif
 455                 if (pool->n_trunk_valid == 0) {
 456                         pool->cfg.free(pool->trunks);
 457                         pool->trunks = NULL;
 458                         pool->n_trunk = 0;
 459                 }
 460         } else if (trunk->free == 1) {
 461                 /* Put into free trunk list head. */
 462                 MLX5_ASSERT(pool->free_list != trunk->idx);
 463                 trunk->next = pool->free_list;
 464                 trunk->prev = TRUNK_INVALID;
 465                 if (pool->free_list != TRUNK_INVALID)
 466                         pool->trunks[pool->free_list]->prev = trunk->idx;
 467                 pool->free_list = trunk->idx;
 468 #ifdef POOL_DEBUG
 469                 pool->trunk_empty--;
 470                 pool->trunk_avail++;
 471 #endif
 472         }
 473 #ifdef POOL_DEBUG
 474         pool->n_entry--;
 475 #endif
 476 out:
 477         mlx5_ipool_unlock(pool);
 478 }
 479
 480 void *
 481 mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 482 {
 483         struct mlx5_indexed_trunk *trunk;
 484         void *p = NULL;
 485         uint32_t trunk_idx;
 486         uint32_t entry_idx;
 487
 488         if (!idx)
 489                 return NULL;
 490         idx -= 1;
 491         mlx5_ipool_lock(pool);
 492         trunk_idx = mlx5_trunk_idx_get(pool, idx);
 493         if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) ||
 494             (pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk))
 495                 goto out;
 496         trunk = pool->trunks[trunk_idx];
 497         if (!trunk)
 498                 goto out;
 499         entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx);
 500         if (trunk_idx != trunk->idx ||
 501             rte_bitmap_get(trunk->bmp, entry_idx))
 502                 goto out;
 503         p = &trunk->data[entry_idx * pool->cfg.size];
 504 out:
 505         mlx5_ipool_unlock(pool);
 506         return p;
 507 }
 508
 509 int
 510 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 511 {
 512         struct mlx5_indexed_trunk **trunks;
 513         uint32_t i;
 514
 515         MLX5_ASSERT(pool);
 516         mlx5_ipool_lock(pool);
 517         trunks = pool->trunks;
 518         for (i = 0; i < pool->n_trunk; i++) {
 519                 if (trunks[i])
 520                         pool->cfg.free(trunks[i]);
 521         }
 522         if (!pool->trunks)
 523                 pool->cfg.free(pool->trunks);
 524         mlx5_ipool_unlock(pool);
 525         mlx5_free(pool);
 526         return 0;
 527 }
 528
 529 void
 530 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 531 {
 532         printf("Pool %s entry size %u, trunks %u, %d entry per trunk, "
 533                "total: %d\n",
 534                pool->cfg.type, pool->cfg.size, pool->n_trunk_valid,
 535                pool->cfg.trunk_size, pool->n_trunk_valid);
 536 #ifdef POOL_DEBUG
 537         printf("Pool %s entry %u, trunk alloc %u, empty: %u, "
 538                "available %u free %u\n",
 539                pool->cfg.type, pool->n_entry, pool->trunk_new,
 540                pool->trunk_empty, pool->trunk_avail, pool->trunk_free);
 541 #endif
 542 }
 543
 544 struct mlx5_l3t_tbl *
 545 mlx5_l3t_create(enum mlx5_l3t_type type)
 546 {
 547         struct mlx5_l3t_tbl *tbl;
 548         struct mlx5_indexed_pool_config l3t_ip_cfg = {
 549                 .trunk_size = 16,
 550                 .grow_trunk = 6,
 551                 .grow_shift = 1,
 552                 .need_lock = 0,
 553                 .release_mem_en = 1,
 554                 .malloc = mlx5_malloc,
 555                 .free = mlx5_free,
 556         };
 557
 558         if (type >= MLX5_L3T_TYPE_MAX) {
 559                 rte_errno = EINVAL;
 560                 return NULL;
 561         }
 562         tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_l3t_tbl), 1,
 563                           SOCKET_ID_ANY);
 564         if (!tbl) {
 565                 rte_errno = ENOMEM;
 566                 return NULL;
 567         }
 568         tbl->type = type;
 569         switch (type) {
 570         case MLX5_L3T_TYPE_WORD:
 571                 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_word);
 572                 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_w";
 573                 break;
 574         case MLX5_L3T_TYPE_DWORD:
 575                 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_dword);
 576                 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_dw";
 577                 break;
 578         case MLX5_L3T_TYPE_QWORD:
 579                 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_qword);
 580                 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_qw";
 581                 break;
 582         default:
 583                 l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_ptr);
 584                 l3t_ip_cfg.type = "mlx5_l3t_e_tbl_tpr";
 585                 break;
 586         }
 587         rte_spinlock_init(&tbl->sl);
 588         tbl->eip = mlx5_ipool_create(&l3t_ip_cfg);
 589         if (!tbl->eip) {
 590                 rte_errno = ENOMEM;
 591                 mlx5_free(tbl);
 592                 tbl = NULL;
 593         }
 594         return tbl;
 595 }
 596
 597 void
 598 mlx5_l3t_destroy(struct mlx5_l3t_tbl *tbl)
 599 {
 600         struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
 601         uint32_t i, j;
 602
 603         if (!tbl)
 604                 return;
 605         g_tbl = tbl->tbl;
 606         if (g_tbl) {
 607                 for (i = 0; i < MLX5_L3T_GT_SIZE; i++) {
 608                         m_tbl = g_tbl->tbl[i];
 609                         if (!m_tbl)
 610                                 continue;
 611                         for (j = 0; j < MLX5_L3T_MT_SIZE; j++) {
 612                                 if (!m_tbl->tbl[j])
 613                                         continue;
 614                                 MLX5_ASSERT(!((struct mlx5_l3t_entry_word *)
 615                                             m_tbl->tbl[j])->ref_cnt);
 616                                 mlx5_ipool_free(tbl->eip,
 617                                                 ((struct mlx5_l3t_entry_word *)
 618                                                 m_tbl->tbl[j])->idx);
 619                                 m_tbl->tbl[j] = 0;
 620                                 if (!(--m_tbl->ref_cnt))
 621                                         break;
 622                         }
 623                         MLX5_ASSERT(!m_tbl->ref_cnt);
 624                         mlx5_free(g_tbl->tbl[i]);
 625                         g_tbl->tbl[i] = 0;
 626                         if (!(--g_tbl->ref_cnt))
 627                                 break;
 628                 }
 629                 MLX5_ASSERT(!g_tbl->ref_cnt);
 630                 mlx5_free(tbl->tbl);
 631                 tbl->tbl = 0;
 632         }
 633         mlx5_ipool_destroy(tbl->eip);
 634         mlx5_free(tbl);
 635 }
 636
 637 static int32_t
 638 __l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 639                 union mlx5_l3t_data *data)
 640 {
 641         struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
 642         struct mlx5_l3t_entry_word *w_e_tbl;
 643         struct mlx5_l3t_entry_dword *dw_e_tbl;
 644         struct mlx5_l3t_entry_qword *qw_e_tbl;
 645         struct mlx5_l3t_entry_ptr *ptr_e_tbl;
 646         void *e_tbl;
 647         uint32_t entry_idx;
 648
 649         g_tbl = tbl->tbl;
 650         if (!g_tbl)
 651                 return -1;
 652         m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK];
 653         if (!m_tbl)
 654                 return -1;
 655         e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK];
 656         if (!e_tbl)
 657                 return -1;
 658         entry_idx = idx & MLX5_L3T_ET_MASK;
 659         switch (tbl->type) {
 660         case MLX5_L3T_TYPE_WORD:
 661                 w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
 662                 data->word = w_e_tbl->entry[entry_idx].data;
 663                 if (w_e_tbl->entry[entry_idx].data)
 664                         w_e_tbl->entry[entry_idx].ref_cnt++;
 665                 break;
 666         case MLX5_L3T_TYPE_DWORD:
 667                 dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
 668                 data->dword = dw_e_tbl->entry[entry_idx].data;
 669                 if (dw_e_tbl->entry[entry_idx].data)
 670                         dw_e_tbl->entry[entry_idx].ref_cnt++;
 671                 break;
 672         case MLX5_L3T_TYPE_QWORD:
 673                 qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
 674                 data->qword = qw_e_tbl->entry[entry_idx].data;
 675                 if (qw_e_tbl->entry[entry_idx].data)
 676                         qw_e_tbl->entry[entry_idx].ref_cnt++;
 677                 break;
 678         default:
 679                 ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
 680                 data->ptr = ptr_e_tbl->entry[entry_idx].data;
 681                 if (ptr_e_tbl->entry[entry_idx].data)
 682                         ptr_e_tbl->entry[entry_idx].ref_cnt++;
 683                 break;
 684         }
 685         return 0;
 686 }
 687
 688 int32_t
 689 mlx5_l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 690                    union mlx5_l3t_data *data)
 691 {
 692         int ret;
 693
 694         rte_spinlock_lock(&tbl->sl);
 695         ret = __l3t_get_entry(tbl, idx, data);
 696         rte_spinlock_unlock(&tbl->sl);
 697         return ret;
 698 }
 699
 700 int32_t
 701 mlx5_l3t_clear_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx)
 702 {
 703         struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
 704         struct mlx5_l3t_entry_word *w_e_tbl;
 705         struct mlx5_l3t_entry_dword *dw_e_tbl;
 706         struct mlx5_l3t_entry_qword *qw_e_tbl;
 707         struct mlx5_l3t_entry_ptr *ptr_e_tbl;
 708         void *e_tbl;
 709         uint32_t entry_idx;
 710         uint64_t ref_cnt;
 711         int32_t ret = -1;
 712
 713         rte_spinlock_lock(&tbl->sl);
 714         g_tbl = tbl->tbl;
 715         if (!g_tbl)
 716                 goto out;
 717         m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK];
 718         if (!m_tbl)
 719                 goto out;
 720         e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK];
 721         if (!e_tbl)
 722                 goto out;
 723         entry_idx = idx & MLX5_L3T_ET_MASK;
 724         switch (tbl->type) {
 725         case MLX5_L3T_TYPE_WORD:
 726                 w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
 727                 MLX5_ASSERT(w_e_tbl->entry[entry_idx].ref_cnt);
 728                 ret = --w_e_tbl->entry[entry_idx].ref_cnt;
 729                 if (ret)
 730                         goto out;
 731                 w_e_tbl->entry[entry_idx].data = 0;
 732                 ref_cnt = --w_e_tbl->ref_cnt;
 733                 break;
 734         case MLX5_L3T_TYPE_DWORD:
 735                 dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
 736                 MLX5_ASSERT(dw_e_tbl->entry[entry_idx].ref_cnt);
 737                 ret = --dw_e_tbl->entry[entry_idx].ref_cnt;
 738                 if (ret)
 739                         goto out;
 740                 dw_e_tbl->entry[entry_idx].data = 0;
 741                 ref_cnt = --dw_e_tbl->ref_cnt;
 742                 break;
 743         case MLX5_L3T_TYPE_QWORD:
 744                 qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
 745                 MLX5_ASSERT(qw_e_tbl->entry[entry_idx].ref_cnt);
 746                 ret = --qw_e_tbl->entry[entry_idx].ref_cnt;
 747                 if (ret)
 748                         goto out;
 749                 qw_e_tbl->entry[entry_idx].data = 0;
 750                 ref_cnt = --qw_e_tbl->ref_cnt;
 751                 break;
 752         default:
 753                 ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
 754                 MLX5_ASSERT(ptr_e_tbl->entry[entry_idx].ref_cnt);
 755                 ret = --ptr_e_tbl->entry[entry_idx].ref_cnt;
 756                 if (ret)
 757                         goto out;
 758                 ptr_e_tbl->entry[entry_idx].data = NULL;
 759                 ref_cnt = --ptr_e_tbl->ref_cnt;
 760                 break;
 761         }
 762         if (!ref_cnt) {
 763                 mlx5_ipool_free(tbl->eip,
 764                                 ((struct mlx5_l3t_entry_word *)e_tbl)->idx);
 765                 m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK] =
 766                                                                         NULL;
 767                 if (!(--m_tbl->ref_cnt)) {
 768                         mlx5_free(m_tbl);
 769                         g_tbl->tbl
 770                         [(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK] = NULL;
 771                         if (!(--g_tbl->ref_cnt)) {
 772                                 mlx5_free(g_tbl);
 773                                 tbl->tbl = 0;
 774                         }
 775                 }
 776         }
 777 out:
 778         rte_spinlock_unlock(&tbl->sl);
 779         return ret;
 780 }
 781
 782 static int32_t
 783 __l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 784                 union mlx5_l3t_data *data)
 785 {
 786         struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
 787         struct mlx5_l3t_entry_word *w_e_tbl;
 788         struct mlx5_l3t_entry_dword *dw_e_tbl;
 789         struct mlx5_l3t_entry_qword *qw_e_tbl;
 790         struct mlx5_l3t_entry_ptr *ptr_e_tbl;
 791         void *e_tbl;
 792         uint32_t entry_idx, tbl_idx = 0;
 793
 794         /* Check the global table, create it if empty. */
 795         g_tbl = tbl->tbl;
 796         if (!g_tbl) {
 797                 g_tbl = mlx5_malloc(MLX5_MEM_ZERO,
 798                                     sizeof(struct mlx5_l3t_level_tbl) +
 799                                     sizeof(void *) * MLX5_L3T_GT_SIZE, 1,
 800                                     SOCKET_ID_ANY);
 801                 if (!g_tbl) {
 802                         rte_errno = ENOMEM;
 803                         return -1;
 804                 }
 805                 tbl->tbl = g_tbl;
 806         }
 807         /*
 808          * Check the middle table, create it if empty. Ref_cnt will be
 809          * increased if new sub table created.
 810          */
 811         m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK];
 812         if (!m_tbl) {
 813                 m_tbl = mlx5_malloc(MLX5_MEM_ZERO,
 814                                     sizeof(struct mlx5_l3t_level_tbl) +
 815                                     sizeof(void *) * MLX5_L3T_MT_SIZE, 1,
 816                                     SOCKET_ID_ANY);
 817                 if (!m_tbl) {
 818                         rte_errno = ENOMEM;
 819                         return -1;
 820                 }
 821                 g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK] =
 822                                                                         m_tbl;
 823                 g_tbl->ref_cnt++;
 824         }
 825         /*
 826          * Check the entry table, create it if empty. Ref_cnt will be
 827          * increased if new sub entry table created.
 828          */
 829         e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK];
 830         if (!e_tbl) {
 831                 e_tbl = mlx5_ipool_zmalloc(tbl->eip, &tbl_idx);
 832                 if (!e_tbl) {
 833                         rte_errno = ENOMEM;
 834                         return -1;
 835                 }
 836                 ((struct mlx5_l3t_entry_word *)e_tbl)->idx = tbl_idx;
 837                 m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK] =
 838                                                                         e_tbl;
 839                 m_tbl->ref_cnt++;
 840         }
 841         entry_idx = idx & MLX5_L3T_ET_MASK;
 842         switch (tbl->type) {
 843         case MLX5_L3T_TYPE_WORD:
 844                 w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
 845                 if (w_e_tbl->entry[entry_idx].data) {
 846                         data->word = w_e_tbl->entry[entry_idx].data;
 847                         w_e_tbl->entry[entry_idx].ref_cnt++;
 848                         rte_errno = EEXIST;
 849                         return -1;
 850                 }
 851                 w_e_tbl->entry[entry_idx].data = data->word;
 852                 w_e_tbl->entry[entry_idx].ref_cnt = 1;
 853                 w_e_tbl->ref_cnt++;
 854                 break;
 855         case MLX5_L3T_TYPE_DWORD:
 856                 dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
 857                 if (dw_e_tbl->entry[entry_idx].data) {
 858                         data->dword = dw_e_tbl->entry[entry_idx].data;
 859                         dw_e_tbl->entry[entry_idx].ref_cnt++;
 860                         rte_errno = EEXIST;
 861                         return -1;
 862                 }
 863                 dw_e_tbl->entry[entry_idx].data = data->dword;
 864                 dw_e_tbl->entry[entry_idx].ref_cnt = 1;
 865                 dw_e_tbl->ref_cnt++;
 866                 break;
 867         case MLX5_L3T_TYPE_QWORD:
 868                 qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
 869                 if (qw_e_tbl->entry[entry_idx].data) {
 870                         data->qword = qw_e_tbl->entry[entry_idx].data;
 871                         qw_e_tbl->entry[entry_idx].ref_cnt++;
 872                         rte_errno = EEXIST;
 873                         return -1;
 874                 }
 875                 qw_e_tbl->entry[entry_idx].data = data->qword;
 876                 qw_e_tbl->entry[entry_idx].ref_cnt = 1;
 877                 qw_e_tbl->ref_cnt++;
 878                 break;
 879         default:
 880                 ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
 881                 if (ptr_e_tbl->entry[entry_idx].data) {
 882                         data->ptr = ptr_e_tbl->entry[entry_idx].data;
 883                         ptr_e_tbl->entry[entry_idx].ref_cnt++;
 884                         rte_errno = EEXIST;
 885                         return -1;
 886                 }
 887                 ptr_e_tbl->entry[entry_idx].data = data->ptr;
 888                 ptr_e_tbl->entry[entry_idx].ref_cnt = 1;
 889                 ptr_e_tbl->ref_cnt++;
 890                 break;
 891         }
 892         return 0;
 893 }
 894
 895 int32_t
 896 mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 897                    union mlx5_l3t_data *data)
 898 {
 899         int ret;
 900
 901         rte_spinlock_lock(&tbl->sl);
 902         ret = __l3t_set_entry(tbl, idx, data);
 903         rte_spinlock_unlock(&tbl->sl);
 904         return ret;
 905 }
 906
 907 int32_t
 908 mlx5_l3t_prepare_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 909                        union mlx5_l3t_data *data,
 910                        mlx5_l3t_alloc_callback_fn cb, void *ctx)
 911 {
 912         int32_t ret;
 913
 914         rte_spinlock_lock(&tbl->sl);
 915         /* Check if entry data is ready. */
 916         ret = __l3t_get_entry(tbl, idx, data);
 917         if (!ret) {
 918                 switch (tbl->type) {
 919                 case MLX5_L3T_TYPE_WORD:
 920                         if (data->word)
 921                                 goto out;
 922                         break;
 923                 case MLX5_L3T_TYPE_DWORD:
 924                         if (data->dword)
 925                                 goto out;
 926                         break;
 927                 case MLX5_L3T_TYPE_QWORD:
 928                         if (data->qword)
 929                                 goto out;
 930                         break;
 931                 default:
 932                         if (data->ptr)
 933                                 goto out;
 934                         break;
 935                 }
 936         }
 937         /* Entry data is not ready, use user callback to create it. */
 938         ret = cb(ctx, data);
 939         if (ret)
 940                 goto out;
 941         /* Save the new allocated data to entry. */
 942         ret = __l3t_set_entry(tbl, idx, data);
 943 out:
 944         rte_spinlock_unlock(&tbl->sl);
 945         return ret;
 946 }