drivers/net/bnxt/tf_core/tf_tbl.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 /* Truflow Table APIs and supporting code */
   7
   8 #include <stdio.h>
   9 #include <string.h>
  10 #include <stdbool.h>
  11 #include <math.h>
  12 #include <sys/param.h>
  13 #include <rte_common.h>
  14 #include <rte_errno.h>
  15 #include "hsi_struct_def_dpdk.h"
  16
  17 #include "tf_core.h"
  18 #include "tf_util.h"
  19 #include "tf_em.h"
  20 #include "tf_msg.h"
  21 #include "tfp.h"
  22 #include "hwrm_tf.h"
  23 #include "bnxt.h"
  24 #include "tf_resources.h"
  25 #include "tf_rm.h"
  26 #include "stack.h"
  27 #include "tf_common.h"
  28
  29 #define PTU_PTE_VALID          0x1UL
  30 #define PTU_PTE_LAST           0x2UL
  31 #define PTU_PTE_NEXT_TO_LAST   0x4UL
  32
  33 /* Number of pointers per page_size */
  34 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  35
  36 #define TF_EM_PG_SZ_4K        (1 << 12)
  37 #define TF_EM_PG_SZ_8K        (1 << 13)
  38 #define TF_EM_PG_SZ_64K       (1 << 16)
  39 #define TF_EM_PG_SZ_256K      (1 << 18)
  40 #define TF_EM_PG_SZ_1M        (1 << 20)
  41 #define TF_EM_PG_SZ_2M        (1 << 21)
  42 #define TF_EM_PG_SZ_4M        (1 << 22)
  43 #define TF_EM_PG_SZ_1G        (1 << 30)
  44
  45 #define TF_EM_CTX_ID_INVALID   0xFFFF
  46
  47 #define TF_EM_MIN_ENTRIES     (1 << 15) /* 32K */
  48 #define TF_EM_MAX_ENTRIES     (1 << 27) /* 128M */
  49
  50 /**
  51  * Function to free a page table
  52  *
  53  * [in] tp
  54  *   Pointer to the page table to free
  55  */
  56 static void
  57 tf_em_free_pg_tbl(struct hcapi_cfa_em_page_tbl *tp)
  58 {
  59         uint32_t i;
  60
  61         for (i = 0; i < tp->pg_count; i++) {
  62                 if (!tp->pg_va_tbl[i]) {
  63                         TFP_DRV_LOG(WARNING,
  64                                     "No mapping for page: %d table: %016" PRIu64 "\n",
  65                                     i,
  66                                     (uint64_t)(uintptr_t)tp);
  67                         continue;
  68                 }
  69
  70                 tfp_free(tp->pg_va_tbl[i]);
  71                 tp->pg_va_tbl[i] = NULL;
  72         }
  73
  74         tp->pg_count = 0;
  75         tfp_free(tp->pg_va_tbl);
  76         tp->pg_va_tbl = NULL;
  77         tfp_free(tp->pg_pa_tbl);
  78         tp->pg_pa_tbl = NULL;
  79 }
  80
  81 /**
  82  * Function to free an EM table
  83  *
  84  * [in] tbl
  85  *   Pointer to the EM table to free
  86  */
  87 static void
  88 tf_em_free_page_table(struct hcapi_cfa_em_table *tbl)
  89 {
  90         struct hcapi_cfa_em_page_tbl *tp;
  91         int i;
  92
  93         for (i = 0; i < tbl->num_lvl; i++) {
  94                 tp = &tbl->pg_tbl[i];
  95                 TFP_DRV_LOG(INFO,
  96                            "EEM: Freeing page table: size %u lvl %d cnt %u\n",
  97                            TF_EM_PAGE_SIZE,
  98                             i,
  99                             tp->pg_count);
 100
 101                 tf_em_free_pg_tbl(tp);
 102         }
 103
 104         tbl->l0_addr = NULL;
 105         tbl->l0_dma_addr = 0;
 106         tbl->num_lvl = 0;
 107         tbl->num_data_pages = 0;
 108 }
 109
 110 /**
 111  * Allocation of page tables
 112  *
 113  * [in] tfp
 114  *   Pointer to a TruFlow handle
 115  *
 116  * [in] pg_count
 117  *   Page count to allocate
 118  *
 119  * [in] pg_size
 120  *   Size of each page
 121  *
 122  * Returns:
 123  *   0       - Success
 124  *   -ENOMEM - Out of memory
 125  */
 126 static int
 127 tf_em_alloc_pg_tbl(struct hcapi_cfa_em_page_tbl *tp,
 128                    uint32_t pg_count,
 129                    uint32_t pg_size)
 130 {
 131         uint32_t i;
 132         struct tfp_calloc_parms parms;
 133
 134         parms.nitems = pg_count;
 135         parms.size = sizeof(void *);
 136         parms.alignment = 0;
 137
 138         if (tfp_calloc(&parms) != 0)
 139                 return -ENOMEM;
 140
 141         tp->pg_va_tbl = parms.mem_va;
 142
 143         if (tfp_calloc(&parms) != 0) {
 144                 tfp_free(tp->pg_va_tbl);
 145                 return -ENOMEM;
 146         }
 147
 148         tp->pg_pa_tbl = parms.mem_va;
 149
 150         tp->pg_count = 0;
 151         tp->pg_size = pg_size;
 152
 153         for (i = 0; i < pg_count; i++) {
 154                 parms.nitems = 1;
 155                 parms.size = pg_size;
 156                 parms.alignment = TF_EM_PAGE_ALIGNMENT;
 157
 158                 if (tfp_calloc(&parms) != 0)
 159                         goto cleanup;
 160
 161                 tp->pg_pa_tbl[i] = (uintptr_t)parms.mem_pa;
 162                 tp->pg_va_tbl[i] = parms.mem_va;
 163
 164                 memset(tp->pg_va_tbl[i], 0, pg_size);
 165                 tp->pg_count++;
 166         }
 167
 168         return 0;
 169
 170 cleanup:
 171         tf_em_free_pg_tbl(tp);
 172         return -ENOMEM;
 173 }
 174
 175 /**
 176  * Allocates EM page tables
 177  *
 178  * [in] tbl
 179  *   Table to allocate pages for
 180  *
 181  * Returns:
 182  *   0       - Success
 183  *   -ENOMEM - Out of memory
 184  */
 185 static int
 186 tf_em_alloc_page_table(struct hcapi_cfa_em_table *tbl)
 187 {
 188         struct hcapi_cfa_em_page_tbl *tp;
 189         int rc = 0;
 190         int i;
 191         uint32_t j;
 192
 193         for (i = 0; i < tbl->num_lvl; i++) {
 194                 tp = &tbl->pg_tbl[i];
 195
 196                 rc = tf_em_alloc_pg_tbl(tp,
 197                                         tbl->page_cnt[i],
 198                                         TF_EM_PAGE_SIZE);
 199                 if (rc) {
 200                         TFP_DRV_LOG(WARNING,
 201                                 "Failed to allocate page table: lvl: %d, rc:%s\n",
 202                                 i,
 203                                 strerror(-rc));
 204                         goto cleanup;
 205                 }
 206
 207                 for (j = 0; j < tp->pg_count; j++) {
 208                         TFP_DRV_LOG(INFO,
 209                                 "EEM: Allocated page table: size %u lvl %d cnt"
 210                                 " %u VA:%p PA:%p\n",
 211                                 TF_EM_PAGE_SIZE,
 212                                 i,
 213                                 tp->pg_count,
 214                                 (uint32_t *)tp->pg_va_tbl[j],
 215                                 (uint32_t *)(uintptr_t)tp->pg_pa_tbl[j]);
 216                 }
 217         }
 218         return rc;
 219
 220 cleanup:
 221         tf_em_free_page_table(tbl);
 222         return rc;
 223 }
 224
 225 /**
 226  * Links EM page tables
 227  *
 228  * [in] tp
 229  *   Pointer to page table
 230  *
 231  * [in] tp_next
 232  *   Pointer to the next page table
 233  *
 234  * [in] set_pte_last
 235  *   Flag controlling if the page table is last
 236  */
 237 static void
 238 tf_em_link_page_table(struct hcapi_cfa_em_page_tbl *tp,
 239                       struct hcapi_cfa_em_page_tbl *tp_next,
 240                       bool set_pte_last)
 241 {
 242         uint64_t *pg_pa = tp_next->pg_pa_tbl;
 243         uint64_t *pg_va;
 244         uint64_t valid;
 245         uint32_t k = 0;
 246         uint32_t i;
 247         uint32_t j;
 248
 249         for (i = 0; i < tp->pg_count; i++) {
 250                 pg_va = tp->pg_va_tbl[i];
 251
 252                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
 253                         if (k == tp_next->pg_count - 2 && set_pte_last)
 254                                 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
 255                         else if (k == tp_next->pg_count - 1 && set_pte_last)
 256                                 valid = PTU_PTE_LAST | PTU_PTE_VALID;
 257                         else
 258                                 valid = PTU_PTE_VALID;
 259
 260                         pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
 261                         if (++k >= tp_next->pg_count)
 262                                 return;
 263                 }
 264         }
 265 }
 266
 267 /**
 268  * Setup a EM page table
 269  *
 270  * [in] tbl
 271  *   Pointer to EM page table
 272  */
 273 static void
 274 tf_em_setup_page_table(struct hcapi_cfa_em_table *tbl)
 275 {
 276         struct hcapi_cfa_em_page_tbl *tp_next;
 277         struct hcapi_cfa_em_page_tbl *tp;
 278         bool set_pte_last = 0;
 279         int i;
 280
 281         for (i = 0; i < tbl->num_lvl - 1; i++) {
 282                 tp = &tbl->pg_tbl[i];
 283                 tp_next = &tbl->pg_tbl[i + 1];
 284                 if (i == tbl->num_lvl - 2)
 285                         set_pte_last = 1;
 286                 tf_em_link_page_table(tp, tp_next, set_pte_last);
 287         }
 288
 289         tbl->l0_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_va_tbl[0];
 290         tbl->l0_dma_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_pa_tbl[0];
 291 }
 292
 293 /**
 294  * Given the page size, size of each data item (entry size),
 295  * and the total number of entries needed, determine the number
 296  * of page table levels and the number of data pages required.
 297  *
 298  * [in] page_size
 299  *   Page size
 300  *
 301  * [in] entry_size
 302  *   Entry size
 303  *
 304  * [in] num_entries
 305  *   Number of entries needed
 306  *
 307  * [out] num_data_pages
 308  *   Number of pages required
 309  *
 310  * Returns:
 311  *   Success  - Number of EM page levels required
 312  *   -ENOMEM  - Out of memory
 313  */
 314 static int
 315 tf_em_size_page_tbl_lvl(uint32_t page_size,
 316                         uint32_t entry_size,
 317                         uint32_t num_entries,
 318                         uint64_t *num_data_pages)
 319 {
 320         uint64_t lvl_data_size = page_size;
 321         int lvl = TF_PT_LVL_0;
 322         uint64_t data_size;
 323
 324         *num_data_pages = 0;
 325         data_size = (uint64_t)num_entries * entry_size;
 326
 327         while (lvl_data_size < data_size) {
 328                 lvl++;
 329
 330                 if (lvl == TF_PT_LVL_1)
 331                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 332                                 page_size;
 333                 else if (lvl == TF_PT_LVL_2)
 334                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 335                                 MAX_PAGE_PTRS(page_size) * page_size;
 336                 else
 337                         return -ENOMEM;
 338         }
 339
 340         *num_data_pages = roundup(data_size, page_size) / page_size;
 341
 342         return lvl;
 343 }
 344
 345 /**
 346  * Return the number of page table pages needed to
 347  * reference the given number of next level pages.
 348  *
 349  * [in] num_pages
 350  *   Number of EM pages
 351  *
 352  * [in] page_size
 353  *   Size of each EM page
 354  *
 355  * Returns:
 356  *   Number of EM page table pages
 357  */
 358 static uint32_t
 359 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 360                      uint32_t page_size)
 361 {
 362         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 363                        MAX_PAGE_PTRS(page_size);
 364         return 0;
 365 }
 366
 367 /**
 368  * Given the number of data pages, page_size and the maximum
 369  * number of page table levels (already determined), size
 370  * the number of page table pages required at each level.
 371  *
 372  * [in] max_lvl
 373  *   Max number of levels
 374  *
 375  * [in] num_data_pages
 376  *   Number of EM data pages
 377  *
 378  * [in] page_size
 379  *   Size of an EM page
 380  *
 381  * [out] *page_cnt
 382  *   EM page count
 383  */
 384 static void
 385 tf_em_size_page_tbls(int max_lvl,
 386                      uint64_t num_data_pages,
 387                      uint32_t page_size,
 388                      uint32_t *page_cnt)
 389 {
 390         if (max_lvl == TF_PT_LVL_0) {
 391                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 392         } else if (max_lvl == TF_PT_LVL_1) {
 393                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 394                 page_cnt[TF_PT_LVL_0] =
 395                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 396         } else if (max_lvl == TF_PT_LVL_2) {
 397                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 398                 page_cnt[TF_PT_LVL_1] =
 399                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 400                 page_cnt[TF_PT_LVL_0] =
 401                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 402         } else {
 403                 return;
 404         }
 405 }
 406
 407 /**
 408  * Size the EM table based on capabilities
 409  *
 410  * [in] tbl
 411  *   EM table to size
 412  *
 413  * Returns:
 414  *   0        - Success
 415  *   - EINVAL - Parameter error
 416  *   - ENOMEM - Out of memory
 417  */
 418 static int
 419 tf_em_size_table(struct hcapi_cfa_em_table *tbl)
 420 {
 421         uint64_t num_data_pages;
 422         uint32_t *page_cnt;
 423         int max_lvl;
 424         uint32_t num_entries;
 425         uint32_t cnt = TF_EM_MIN_ENTRIES;
 426
 427         /* Ignore entry if both size and number are zero */
 428         if (!tbl->entry_size && !tbl->num_entries)
 429                 return 0;
 430
 431         /* If only one is set then error */
 432         if (!tbl->entry_size || !tbl->num_entries)
 433                 return -EINVAL;
 434
 435         /* Determine number of page table levels and the number
 436          * of data pages needed to process the given eem table.
 437          */
 438         if (tbl->type == TF_RECORD_TABLE) {
 439                 /*
 440                  * For action records just a memory size is provided. Work
 441                  * backwards to resolve to number of entries
 442                  */
 443                 num_entries = tbl->num_entries / tbl->entry_size;
 444                 if (num_entries < TF_EM_MIN_ENTRIES) {
 445                         num_entries = TF_EM_MIN_ENTRIES;
 446                 } else {
 447                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 448                                 cnt *= 2;
 449                         num_entries = cnt;
 450                 }
 451         } else {
 452                 num_entries = tbl->num_entries;
 453         }
 454
 455         max_lvl = tf_em_size_page_tbl_lvl(TF_EM_PAGE_SIZE,
 456                                           tbl->entry_size,
 457                                           tbl->num_entries,
 458                                           &num_data_pages);
 459         if (max_lvl < 0) {
 460                 TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 461                 TFP_DRV_LOG(WARNING,
 462                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 463                             tbl->type, (uint64_t)num_entries * tbl->entry_size,
 464                             TF_EM_PAGE_SIZE);
 465                 return -ENOMEM;
 466         }
 467
 468         tbl->num_lvl = max_lvl + 1;
 469         tbl->num_data_pages = num_data_pages;
 470
 471         /* Determine the number of pages needed at each level */
 472         page_cnt = tbl->page_cnt;
 473         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 474         tf_em_size_page_tbls(max_lvl, num_data_pages, TF_EM_PAGE_SIZE,
 475                                 page_cnt);
 476
 477         TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 478         TFP_DRV_LOG(INFO,
 479                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 " l0: %u l1: %u l2: %u\n",
 480                     max_lvl + 1,
 481                     (uint64_t)num_data_pages * TF_EM_PAGE_SIZE,
 482                     num_data_pages,
 483                     page_cnt[TF_PT_LVL_0],
 484                     page_cnt[TF_PT_LVL_1],
 485                     page_cnt[TF_PT_LVL_2]);
 486
 487         return 0;
 488 }
 489
 490 /**
 491  * Unregisters EM Ctx in Firmware
 492  *
 493  * [in] tfp
 494  *   Pointer to a TruFlow handle
 495  *
 496  * [in] tbl_scope_cb
 497  *   Pointer to a table scope control block
 498  *
 499  * [in] dir
 500  *   Receive or transmit direction
 501  */
 502 static void
 503 tf_em_ctx_unreg(struct tf *tfp,
 504                 struct tf_tbl_scope_cb *tbl_scope_cb,
 505                 int dir)
 506 {
 507         struct hcapi_cfa_em_ctx_mem_info *ctxp =
 508                 &tbl_scope_cb->em_ctx_info[dir];
 509         struct hcapi_cfa_em_table *tbl;
 510         int i;
 511
 512         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 513                 tbl = &ctxp->em_tables[i];
 514
 515                 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
 516                         tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
 517                         tf_em_free_page_table(tbl);
 518                 }
 519         }
 520 }
 521
 522 /**
 523  * Registers EM Ctx in Firmware
 524  *
 525  * [in] tfp
 526  *   Pointer to a TruFlow handle
 527  *
 528  * [in] tbl_scope_cb
 529  *   Pointer to a table scope control block
 530  *
 531  * [in] dir
 532  *   Receive or transmit direction
 533  *
 534  * Returns:
 535  *   0       - Success
 536  *   -ENOMEM - Out of Memory
 537  */
 538 static int
 539 tf_em_ctx_reg(struct tf *tfp,
 540               struct tf_tbl_scope_cb *tbl_scope_cb,
 541               int dir)
 542 {
 543         struct hcapi_cfa_em_ctx_mem_info *ctxp =
 544                 &tbl_scope_cb->em_ctx_info[dir];
 545         struct hcapi_cfa_em_table *tbl;
 546         int rc = 0;
 547         int i;
 548
 549         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 550                 tbl = &ctxp->em_tables[i];
 551
 552                 if (tbl->num_entries && tbl->entry_size) {
 553                         rc = tf_em_size_table(tbl);
 554
 555                         if (rc)
 556                                 goto cleanup;
 557
 558                         rc = tf_em_alloc_page_table(tbl);
 559                         if (rc)
 560                                 goto cleanup;
 561
 562                         tf_em_setup_page_table(tbl);
 563                         rc = tf_msg_em_mem_rgtr(tfp,
 564                                                 tbl->num_lvl - 1,
 565                                                 TF_EM_PAGE_SIZE_ENUM,
 566                                                 tbl->l0_dma_addr,
 567                                                 &tbl->ctx_id);
 568                         if (rc)
 569                                 goto cleanup;
 570                 }
 571         }
 572         return rc;
 573
 574 cleanup:
 575         tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
 576         return rc;
 577 }
 578
 579 /**
 580  * Validates EM number of entries requested
 581  *
 582  * [in] tbl_scope_cb
 583  *   Pointer to table scope control block to be populated
 584  *
 585  * [in] parms
 586  *   Pointer to input parameters
 587  *
 588  * Returns:
 589  *   0       - Success
 590  *   -EINVAL - Parameter error
 591  */
 592 static int
 593 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 594                            struct tf_alloc_tbl_scope_parms *parms)
 595 {
 596         uint32_t cnt;
 597
 598         if (parms->rx_mem_size_in_mb != 0) {
 599                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 600                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 601                                      + 1);
 602                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 603                                         TF_MEGABYTE) / (key_b + action_b);
 604
 605                 if (num_entries < TF_EM_MIN_ENTRIES) {
 606                         TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 607                                     "%uMB\n",
 608                                     parms->rx_mem_size_in_mb);
 609                         return -EINVAL;
 610                 }
 611
 612                 cnt = TF_EM_MIN_ENTRIES;
 613                 while (num_entries > cnt &&
 614                        cnt <= TF_EM_MAX_ENTRIES)
 615                         cnt *= 2;
 616
 617                 if (cnt > TF_EM_MAX_ENTRIES) {
 618                         TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 619                                     "%u\n",
 620                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 621                         return -EINVAL;
 622                 }
 623
 624                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 625         } else {
 626                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 627                     TF_EM_MIN_ENTRIES ||
 628                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 629                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 630                         TFP_DRV_LOG(ERR,
 631                                     "EEM: Invalid number of Rx flows "
 632                                     "requested:%u max:%u\n",
 633                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 634                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 635                         return -EINVAL;
 636                 }
 637
 638                 /* must be a power-of-2 supported value
 639                  * in the range 32K - 128M
 640                  */
 641                 cnt = TF_EM_MIN_ENTRIES;
 642                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 643                        cnt <= TF_EM_MAX_ENTRIES)
 644                         cnt *= 2;
 645
 646                 if (cnt > TF_EM_MAX_ENTRIES) {
 647                         TFP_DRV_LOG(ERR,
 648                                     "EEM: Invalid number of Rx requested: %u\n",
 649                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 650                         return -EINVAL;
 651                 }
 652         }
 653
 654         if (parms->tx_mem_size_in_mb != 0) {
 655                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 656                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 657                                      + 1);
 658                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 659                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 660                         (key_b + action_b);
 661
 662                 if (num_entries < TF_EM_MIN_ENTRIES) {
 663                         TFP_DRV_LOG(ERR,
 664                                     "EEM: Insufficient memory requested:%uMB\n",
 665                                     parms->rx_mem_size_in_mb);
 666                         return -EINVAL;
 667                 }
 668
 669                 cnt = TF_EM_MIN_ENTRIES;
 670                 while (num_entries > cnt &&
 671                        cnt <= TF_EM_MAX_ENTRIES)
 672                         cnt *= 2;
 673
 674                 if (cnt > TF_EM_MAX_ENTRIES) {
 675                         TFP_DRV_LOG(ERR,
 676                                     "EEM: Invalid number of Tx requested: %u\n",
 677                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 678                         return -EINVAL;
 679                 }
 680
 681                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 682         } else {
 683                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 684                     TF_EM_MIN_ENTRIES ||
 685                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 686                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 687                         TFP_DRV_LOG(ERR,
 688                                     "EEM: Invalid number of Tx flows "
 689                                     "requested:%u max:%u\n",
 690                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 691                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 692                         return -EINVAL;
 693                 }
 694
 695                 cnt = TF_EM_MIN_ENTRIES;
 696                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 697                        cnt <= TF_EM_MAX_ENTRIES)
 698                         cnt *= 2;
 699
 700                 if (cnt > TF_EM_MAX_ENTRIES) {
 701                         TFP_DRV_LOG(ERR,
 702                                     "EEM: Invalid number of Tx requested: %u\n",
 703                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 704                         return -EINVAL;
 705                 }
 706         }
 707
 708         if (parms->rx_num_flows_in_k != 0 &&
 709             (parms->rx_max_key_sz_in_bits / 8 == 0)) {
 710                 TFP_DRV_LOG(ERR,
 711                             "EEM: Rx key size required: %u\n",
 712                             (parms->rx_max_key_sz_in_bits));
 713                 return -EINVAL;
 714         }
 715
 716         if (parms->tx_num_flows_in_k != 0 &&
 717             (parms->tx_max_key_sz_in_bits / 8 == 0)) {
 718                 TFP_DRV_LOG(ERR,
 719                             "EEM: Tx key size required: %u\n",
 720                             (parms->tx_max_key_sz_in_bits));
 721                 return -EINVAL;
 722         }
 723         /* Rx */
 724         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 725                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 726         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 727                 parms->rx_max_key_sz_in_bits / 8;
 728
 729         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 730                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 731         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 732                 parms->rx_max_key_sz_in_bits / 8;
 733
 734         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 735                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 736         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 737                 parms->rx_max_action_entry_sz_in_bits / 8;
 738
 739         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries =
 740                 0;
 741
 742         /* Tx */
 743         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 744                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 745         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 746                 parms->tx_max_key_sz_in_bits / 8;
 747
 748         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 749                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 750         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 751                 parms->tx_max_key_sz_in_bits / 8;
 752
 753         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 754                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 755         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 756                 parms->tx_max_action_entry_sz_in_bits / 8;
 757
 758         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries =
 759                 0;
 760
 761         return 0;
 762 }
 763
 764 /**
 765  * Internal function to set a Table Entry. Supports all internal Table Types
 766  *
 767  * [in] tfp
 768  *   Pointer to TruFlow handle
 769  *
 770  * [in] parms
 771  *   Pointer to input parameters
 772  *
 773  * Returns:
 774  *   0       - Success
 775  *   -EINVAL - Parameter error
 776  */
 777 static int
 778 tf_set_tbl_entry_internal(struct tf *tfp,
 779                           struct tf_set_tbl_entry_parms *parms)
 780 {
 781         int rc;
 782         int id;
 783         uint32_t index;
 784         struct bitalloc *session_pool;
 785         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 786
 787         /* Lookup the pool using the table type of the element */
 788         rc = tf_rm_lookup_tbl_type_pool(tfs,
 789                                         parms->dir,
 790                                         parms->type,
 791                                         &session_pool);
 792         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 793         if (rc)
 794                 return rc;
 795
 796         index = parms->idx;
 797
 798         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
 799             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
 800             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
 801                 TFP_DRV_LOG(ERR,
 802                             "%s, Type not supported, type:%d\n",
 803                             tf_dir_2_str(parms->dir),
 804                             parms->type);
 805                 return -EOPNOTSUPP;
 806         }
 807
 808         /* Adjust the returned index/offset as there is no guarantee
 809          * that the start is 0 at time of RM allocation
 810          */
 811         tf_rm_convert_index(tfs,
 812                             parms->dir,
 813                             parms->type,
 814                             TF_RM_CONVERT_RM_BASE,
 815                             parms->idx,
 816                             &index);
 817
 818         /* Verify that the entry has been previously allocated */
 819         id = ba_inuse(session_pool, index);
 820         if (id != 1) {
 821                 TFP_DRV_LOG(ERR,
 822                    "%s, Invalid or not allocated index, type:%d, idx:%d\n",
 823                    tf_dir_2_str(parms->dir),
 824                    parms->type,
 825                    index);
 826                 return -EINVAL;
 827         }
 828
 829         /* Set the entry */
 830         rc = tf_msg_set_tbl_entry(tfp,
 831                                   parms->dir,
 832                                   parms->type,
 833                                   parms->data_sz_in_bytes,
 834                                   parms->data,
 835                                   parms->idx);
 836         if (rc) {
 837                 TFP_DRV_LOG(ERR,
 838                             "%s, Set failed, type:%d, rc:%s\n",
 839                             tf_dir_2_str(parms->dir),
 840                             parms->type,
 841                             strerror(-rc));
 842         }
 843
 844         return rc;
 845 }
 846
 847 /**
 848  * Internal function to get a Table Entry. Supports all Table Types
 849  * except the TF_TBL_TYPE_EXT as that is handled as a table scope.
 850  *
 851  * [in] tfp
 852  *   Pointer to TruFlow handle
 853  *
 854  * [in] parms
 855  *   Pointer to input parameters
 856  *
 857  * Returns:
 858  *   0       - Success
 859  *   -EINVAL - Parameter error
 860  */
 861 static int
 862 tf_get_tbl_entry_internal(struct tf *tfp,
 863                           struct tf_get_tbl_entry_parms *parms)
 864 {
 865         int rc;
 866         int id;
 867         uint32_t index;
 868         struct bitalloc *session_pool;
 869         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 870
 871         /* Lookup the pool using the table type of the element */
 872         rc = tf_rm_lookup_tbl_type_pool(tfs,
 873                                         parms->dir,
 874                                         parms->type,
 875                                         &session_pool);
 876         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 877         if (rc)
 878                 return rc;
 879
 880         index = parms->idx;
 881
 882         /* Adjust the returned index/offset as there is no guarantee
 883          * that the start is 0 at time of RM allocation
 884          */
 885         tf_rm_convert_index(tfs,
 886                             parms->dir,
 887                             parms->type,
 888                             TF_RM_CONVERT_RM_BASE,
 889                             parms->idx,
 890                             &index);
 891
 892         /* Verify that the entry has been previously allocated */
 893         id = ba_inuse(session_pool, index);
 894         if (id != 1) {
 895                 TFP_DRV_LOG(ERR,
 896                    "%s, Invalid or not allocated index, type:%d, idx:%d\n",
 897                    tf_dir_2_str(parms->dir),
 898                    parms->type,
 899                    index);
 900                 return -EINVAL;
 901         }
 902
 903         /* Get the entry */
 904         rc = tf_msg_get_tbl_entry(tfp,
 905                                   parms->dir,
 906                                   parms->type,
 907                                   parms->data_sz_in_bytes,
 908                                   parms->data,
 909                                   parms->idx);
 910         if (rc) {
 911                 TFP_DRV_LOG(ERR,
 912                             "%s, Get failed, type:%d, rc:%s\n",
 913                             tf_dir_2_str(parms->dir),
 914                             parms->type,
 915                             strerror(-rc));
 916         }
 917
 918         return rc;
 919 }
 920
 921 /**
 922  * Internal function to get a Table Entry. Supports all Table Types
 923  * except the TF_TBL_TYPE_EXT as that is handled as a table scope.
 924  *
 925  * [in] tfp
 926  *   Pointer to TruFlow handle
 927  *
 928  * [in] parms
 929  *   Pointer to input parameters
 930  *
 931  * Returns:
 932  *   0       - Success
 933  *   -EINVAL - Parameter error
 934  */
 935 static int
 936 tf_bulk_get_tbl_entry_internal(struct tf *tfp,
 937                           struct tf_bulk_get_tbl_entry_parms *parms)
 938 {
 939         int rc;
 940         int id;
 941         uint32_t index;
 942         struct bitalloc *session_pool;
 943         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 944
 945         /* Lookup the pool using the table type of the element */
 946         rc = tf_rm_lookup_tbl_type_pool(tfs,
 947                                         parms->dir,
 948                                         parms->type,
 949                                         &session_pool);
 950         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 951         if (rc)
 952                 return rc;
 953
 954         index = parms->starting_idx;
 955
 956         /*
 957          * Adjust the returned index/offset as there is no guarantee
 958          * that the start is 0 at time of RM allocation
 959          */
 960         tf_rm_convert_index(tfs,
 961                             parms->dir,
 962                             parms->type,
 963                             TF_RM_CONVERT_RM_BASE,
 964                             parms->starting_idx,
 965                             &index);
 966
 967         /* Verify that the entry has been previously allocated */
 968         id = ba_inuse(session_pool, index);
 969         if (id != 1) {
 970                 TFP_DRV_LOG(ERR,
 971                    "%s, Invalid or not allocated index, type:%d, starting_idx:%d\n",
 972                    tf_dir_2_str(parms->dir),
 973                    parms->type,
 974                    index);
 975                 return -EINVAL;
 976         }
 977
 978         /* Get the entry */
 979         rc = tf_msg_bulk_get_tbl_entry(tfp, parms);
 980         if (rc) {
 981                 TFP_DRV_LOG(ERR,
 982                             "%s, Bulk get failed, type:%d, rc:%s\n",
 983                             tf_dir_2_str(parms->dir),
 984                             parms->type,
 985                             strerror(-rc));
 986         }
 987
 988         return rc;
 989 }
 990
 991 #if (TF_SHADOW == 1)
 992 /**
 993  * Allocate Tbl entry from the Shadow DB. Shadow DB is searched for
 994  * the requested entry. If found the ref count is incremente and
 995  * returned.
 996  *
 997  * [in] tfs
 998  *   Pointer to session
 999  * [in] parms
1000  *   Allocation parameters
1001  *
1002  * Return:
1003  *  0       - Success, entry found and ref count incremented
1004  *  -ENOENT - Failure, entry not found
1005  */
1006 static int
1007 tf_alloc_tbl_entry_shadow(struct tf_session *tfs __rte_unused,
1008                           struct tf_alloc_tbl_entry_parms *parms __rte_unused)
1009 {
1010         TFP_DRV_LOG(ERR,
1011                     "%s, Entry Alloc with search not supported\n",
1012                     tf_dir_2_str(parms->dir));
1013
1014         return -EOPNOTSUPP;
1015 }
1016
1017 /**
1018  * Free Tbl entry from the Shadow DB. Shadow DB is searched for
1019  * the requested entry. If found the ref count is decremente and
1020  * new ref_count returned.
1021  *
1022  * [in] tfs
1023  *   Pointer to session
1024  * [in] parms
1025  *   Allocation parameters
1026  *
1027  * Return:
1028  *  0       - Success, entry found and ref count decremented
1029  *  -ENOENT - Failure, entry not found
1030  */
1031 static int
1032 tf_free_tbl_entry_shadow(struct tf_session *tfs,
1033                          struct tf_free_tbl_entry_parms *parms)
1034 {
1035         TFP_DRV_LOG(ERR,
1036                     "%s, Entry Free with search not supported\n",
1037                     tf_dir_2_str(parms->dir));
1038
1039         return -EOPNOTSUPP;
1040 }
1041 #endif /* TF_SHADOW */
1042
1043 /**
1044  * Create External Tbl pool of memory indexes.
1045  *
1046  * [in] dir
1047  *   direction
1048  * [in] tbl_scope_cb
1049  *   pointer to the table scope
1050  * [in] num_entries
1051  *   number of entries to write
1052  * [in] entry_sz_bytes
1053  *   size of each entry
1054  *
1055  * Return:
1056  *  0       - Success, entry allocated - no search support
1057  *  -ENOMEM -EINVAL -EOPNOTSUPP
1058  *          - Failure, entry not allocated, out of resources
1059  */
1060 static int
1061 tf_create_tbl_pool_external(enum tf_dir dir,
1062                             struct tf_tbl_scope_cb *tbl_scope_cb,
1063                             uint32_t num_entries,
1064                             uint32_t entry_sz_bytes)
1065 {
1066         struct tfp_calloc_parms parms;
1067         uint32_t i;
1068         int32_t j;
1069         int rc = 0;
1070         struct stack *pool = &tbl_scope_cb->ext_act_pool[dir];
1071
1072         parms.nitems = num_entries;
1073         parms.size = sizeof(uint32_t);
1074         parms.alignment = 0;
1075
1076         if (tfp_calloc(&parms) != 0) {
1077                 TFP_DRV_LOG(ERR, "%s: TBL: external pool failure %s\n",
1078                             tf_dir_2_str(dir), strerror(ENOMEM));
1079                 return -ENOMEM;
1080         }
1081
1082         /* Create empty stack
1083          */
1084         rc = stack_init(num_entries, parms.mem_va, pool);
1085
1086         if (rc != 0) {
1087                 TFP_DRV_LOG(ERR, "%s: TBL: stack init failure %s\n",
1088                             tf_dir_2_str(dir), strerror(-rc));
1089                 goto cleanup;
1090         }
1091
1092         /* Save the  malloced memory address so that it can
1093          * be freed when the table scope is freed.
1094          */
1095         tbl_scope_cb->ext_act_pool_mem[dir] = (uint32_t *)parms.mem_va;
1096
1097         /* Fill pool with indexes in reverse
1098          */
1099         j = (num_entries - 1) * entry_sz_bytes;
1100
1101         for (i = 0; i < num_entries; i++) {
1102                 rc = stack_push(pool, j);
1103                 if (rc != 0) {
1104                         TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
1105                                     tf_dir_2_str(dir), strerror(-rc));
1106                         goto cleanup;
1107                 }
1108
1109                 if (j < 0) {
1110                         TFP_DRV_LOG(ERR, "%d TBL: invalid offset (%d)\n",
1111                                     dir, j);
1112                         goto cleanup;
1113                 }
1114                 j -= entry_sz_bytes;
1115         }
1116
1117         if (!stack_is_full(pool)) {
1118                 rc = -EINVAL;
1119                 TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
1120                             tf_dir_2_str(dir), strerror(-rc));
1121                 goto cleanup;
1122         }
1123         return 0;
1124 cleanup:
1125         tfp_free((void *)parms.mem_va);
1126         return rc;
1127 }
1128
1129 /**
1130  * Destroy External Tbl pool of memory indexes.
1131  *
1132  * [in] dir
1133  *   direction
1134  * [in] tbl_scope_cb
1135  *   pointer to the table scope
1136  *
1137  */
1138 static void
1139 tf_destroy_tbl_pool_external(enum tf_dir dir,
1140                              struct tf_tbl_scope_cb *tbl_scope_cb)
1141 {
1142         uint32_t *ext_act_pool_mem =
1143                 tbl_scope_cb->ext_act_pool_mem[dir];
1144
1145         tfp_free(ext_act_pool_mem);
1146 }
1147
1148 /**
1149  * Allocate External Tbl entry from the Session Pool.
1150  *
1151  * [in] tfp
1152  *   Pointer to Truflow Handle
1153  * [in] parms
1154  *   Allocation parameters
1155  *
1156  * Return:
1157  *  0       - Success, entry allocated - no search support
1158  *  -ENOMEM -EINVAL -EOPNOTSUPP
1159  *          - Failure, entry not allocated, out of resources
1160  */
1161 static int
1162 tf_alloc_tbl_entry_pool_external(struct tf *tfp,
1163                                  struct tf_alloc_tbl_entry_parms *parms)
1164 {
1165         int rc;
1166         uint32_t index;
1167         struct tf_session *tfs;
1168         struct tf_tbl_scope_cb *tbl_scope_cb;
1169         struct stack *pool;
1170
1171         TF_CHECK_PARMS_SESSION(tfp, parms);
1172
1173         tfs = (struct tf_session *)(tfp->session->core_data);
1174
1175         /* Get the pool info from the table scope
1176          */
1177         tbl_scope_cb = tbl_scope_cb_find(tfs, parms->tbl_scope_id);
1178
1179         if (tbl_scope_cb == NULL) {
1180                 TFP_DRV_LOG(ERR,
1181                             "%s, table scope not allocated\n",
1182                             tf_dir_2_str(parms->dir));
1183                 return -EINVAL;
1184         }
1185         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
1186
1187         /* Allocate an element
1188          */
1189         rc = stack_pop(pool, &index);
1190
1191         if (rc != 0) {
1192                 TFP_DRV_LOG(ERR,
1193                    "%s, Allocation failed, type:%d\n",
1194                    tf_dir_2_str(parms->dir),
1195                    parms->type);
1196                 return rc;
1197         }
1198         parms->idx = index;
1199         return rc;
1200 }
1201
1202 /**
1203  * Allocate Internal Tbl entry from the Session Pool.
1204  *
1205  * [in] tfp
1206  *   Pointer to Truflow Handle
1207  * [in] parms
1208  *   Allocation parameters
1209  *
1210  * Return:
1211  *  0       - Success, entry found and ref count decremented
1212  *  -ENOMEM - Failure, entry not allocated, out of resources
1213  */
1214 static int
1215 tf_alloc_tbl_entry_pool_internal(struct tf *tfp,
1216                                  struct tf_alloc_tbl_entry_parms *parms)
1217 {
1218         int rc;
1219         int id;
1220         int free_cnt;
1221         uint32_t index;
1222         struct bitalloc *session_pool;
1223         struct tf_session *tfs;
1224
1225         TF_CHECK_PARMS_SESSION(tfp, parms);
1226
1227         tfs = (struct tf_session *)(tfp->session->core_data);
1228
1229         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1230             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1231             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1232             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1233             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1234             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B &&
1235             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
1236                 TFP_DRV_LOG(ERR,
1237                             "%s, Type not supported, type:%d\n",
1238                             tf_dir_2_str(parms->dir),
1239                             parms->type);
1240                 return -EOPNOTSUPP;
1241         }
1242
1243         /* Lookup the pool using the table type of the element */
1244         rc = tf_rm_lookup_tbl_type_pool(tfs,
1245                                         parms->dir,
1246                                         parms->type,
1247                                         &session_pool);
1248         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1249         if (rc)
1250                 return rc;
1251
1252         id = ba_alloc(session_pool);
1253         if (id == -1) {
1254                 free_cnt = ba_free_count(session_pool);
1255
1256                 TFP_DRV_LOG(ERR,
1257                    "%s, Allocation failed, type:%d, free:%d\n",
1258                    tf_dir_2_str(parms->dir),
1259                    parms->type,
1260                    free_cnt);
1261                 return -ENOMEM;
1262         }
1263
1264         /* Adjust the returned index/offset as there is no guarantee
1265          * that the start is 0 at time of RM allocation
1266          */
1267         tf_rm_convert_index(tfs,
1268                             parms->dir,
1269                             parms->type,
1270                             TF_RM_CONVERT_ADD_BASE,
1271                             id,
1272                             &index);
1273         parms->idx = index;
1274         return rc;
1275 }
1276
1277 /**
1278  * Free External Tbl entry to the session pool.
1279  *
1280  * [in] tfp
1281  *   Pointer to Truflow Handle
1282  * [in] parms
1283  *   Allocation parameters
1284  *
1285  * Return:
1286  *  0       - Success, entry freed
1287  *
1288  * - Failure, entry not successfully freed for these reasons
1289  *  -ENOMEM
1290  *  -EOPNOTSUPP
1291  *  -EINVAL
1292  */
1293 static int
1294 tf_free_tbl_entry_pool_external(struct tf *tfp,
1295                                 struct tf_free_tbl_entry_parms *parms)
1296 {
1297         int rc = 0;
1298         struct tf_session *tfs;
1299         uint32_t index;
1300         struct tf_tbl_scope_cb *tbl_scope_cb;
1301         struct stack *pool;
1302
1303         TF_CHECK_PARMS_SESSION(tfp, parms);
1304
1305         tfs = (struct tf_session *)(tfp->session->core_data);
1306
1307         /* Get the pool info from the table scope
1308          */
1309         tbl_scope_cb = tbl_scope_cb_find(tfs, parms->tbl_scope_id);
1310
1311         if (tbl_scope_cb == NULL) {
1312                 TFP_DRV_LOG(ERR,
1313                             "%s, table scope error\n",
1314                             tf_dir_2_str(parms->dir));
1315                 return -EINVAL;
1316         }
1317         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
1318
1319         index = parms->idx;
1320
1321         rc = stack_push(pool, index);
1322
1323         if (rc != 0) {
1324                 TFP_DRV_LOG(ERR,
1325                    "%s, consistency error, stack full, type:%d, idx:%d\n",
1326                    tf_dir_2_str(parms->dir),
1327                    parms->type,
1328                    index);
1329         }
1330         return rc;
1331 }
1332
1333 /**
1334  * Free Internal Tbl entry from the Session Pool.
1335  *
1336  * [in] tfp
1337  *   Pointer to Truflow Handle
1338  * [in] parms
1339  *   Allocation parameters
1340  *
1341  * Return:
1342  *  0       - Success, entry found and ref count decremented
1343  *  -ENOMEM - Failure, entry not allocated, out of resources
1344  */
1345 static int
1346 tf_free_tbl_entry_pool_internal(struct tf *tfp,
1347                        struct tf_free_tbl_entry_parms *parms)
1348 {
1349         int rc = 0;
1350         int id;
1351         struct bitalloc *session_pool;
1352         struct tf_session *tfs;
1353         uint32_t index;
1354
1355         TF_CHECK_PARMS_SESSION(tfp, parms);
1356
1357         tfs = (struct tf_session *)(tfp->session->core_data);
1358
1359         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1360             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1361             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1362             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1363             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1364             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B &&
1365             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
1366                 TFP_DRV_LOG(ERR,
1367                             "%s, Type not supported, type:%d\n",
1368                             tf_dir_2_str(parms->dir),
1369                             parms->type);
1370                 return -EOPNOTSUPP;
1371         }
1372
1373         /* Lookup the pool using the table type of the element */
1374         rc = tf_rm_lookup_tbl_type_pool(tfs,
1375                                         parms->dir,
1376                                         parms->type,
1377                                         &session_pool);
1378         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1379         if (rc)
1380                 return rc;
1381
1382         index = parms->idx;
1383
1384         /* Adjust the returned index/offset as there is no guarantee
1385          * that the start is 0 at time of RM allocation
1386          */
1387         tf_rm_convert_index(tfs,
1388                             parms->dir,
1389                             parms->type,
1390                             TF_RM_CONVERT_RM_BASE,
1391                             parms->idx,
1392                             &index);
1393
1394         /* Check if element was indeed allocated */
1395         id = ba_inuse_free(session_pool, index);
1396         if (id == -1) {
1397                 TFP_DRV_LOG(ERR,
1398                    "%s, Element not previously alloc'ed, type:%d, idx:%d\n",
1399                    tf_dir_2_str(parms->dir),
1400                    parms->type,
1401                    index);
1402                 return -ENOMEM;
1403         }
1404
1405         return rc;
1406 }
1407
1408 /* API defined in tf_em.h */
1409 struct tf_tbl_scope_cb *
1410 tbl_scope_cb_find(struct tf_session *session,
1411                   uint32_t tbl_scope_id)
1412 {
1413         int i;
1414
1415         /* Check that id is valid */
1416         i = ba_inuse(session->tbl_scope_pool_rx, tbl_scope_id);
1417         if (i < 0)
1418                 return NULL;
1419
1420         for (i = 0; i < TF_NUM_TBL_SCOPE; i++) {
1421                 if (session->tbl_scopes[i].tbl_scope_id == tbl_scope_id)
1422                         return &session->tbl_scopes[i];
1423         }
1424
1425         return NULL;
1426 }
1427
1428 /* API defined in tf_core.h */
1429 int
1430 tf_free_eem_tbl_scope_cb(struct tf *tfp,
1431                          struct tf_free_tbl_scope_parms *parms)
1432 {
1433         int rc = 0;
1434         enum tf_dir  dir;
1435         struct tf_tbl_scope_cb *tbl_scope_cb;
1436         struct tf_session *session;
1437
1438         session = (struct tf_session *)(tfp->session->core_data);
1439
1440         tbl_scope_cb = tbl_scope_cb_find(session,
1441                                          parms->tbl_scope_id);
1442
1443         if (tbl_scope_cb == NULL) {
1444                 TFP_DRV_LOG(ERR, "Table scope error\n");
1445                 return -EINVAL;
1446         }
1447
1448         /* Free Table control block */
1449         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1450
1451         /* free table scope locks */
1452         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1453                 /* Free associated external pools
1454                  */
1455                 tf_destroy_tbl_pool_external(dir,
1456                                              tbl_scope_cb);
1457                 tf_msg_em_op(tfp,
1458                              dir,
1459                              HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
1460
1461                 /* free table scope and all associated resources */
1462                 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
1463         }
1464
1465         return rc;
1466 }
1467
1468 /* API defined in tf_em.h */
1469 int
1470 tf_alloc_eem_tbl_scope(struct tf *tfp,
1471                        struct tf_alloc_tbl_scope_parms *parms)
1472 {
1473         int rc;
1474         enum tf_dir dir;
1475         struct tf_tbl_scope_cb *tbl_scope_cb;
1476         struct hcapi_cfa_em_table *em_tables;
1477         int index;
1478         struct tf_session *session;
1479         struct tf_free_tbl_scope_parms free_parms;
1480
1481         session = (struct tf_session *)tfp->session->core_data;
1482
1483         /* Get Table Scope control block from the session pool */
1484         index = ba_alloc(session->tbl_scope_pool_rx);
1485         if (index == -1) {
1486                 TFP_DRV_LOG(ERR, "EEM: Unable to allocate table scope "
1487                             "Control Block\n");
1488                 return -ENOMEM;
1489         }
1490
1491         tbl_scope_cb = &session->tbl_scopes[index];
1492         tbl_scope_cb->index = index;
1493         tbl_scope_cb->tbl_scope_id = index;
1494         parms->tbl_scope_id = index;
1495
1496         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1497                 rc = tf_msg_em_qcaps(tfp,
1498                                      dir,
1499                                      &tbl_scope_cb->em_caps[dir]);
1500                 if (rc) {
1501                         TFP_DRV_LOG(ERR,
1502                                     "EEM: Unable to query for EEM capability,"
1503                                     " rc:%s\n",
1504                                     strerror(-rc));
1505                         goto cleanup;
1506                 }
1507         }
1508
1509         /*
1510          * Validate and setup table sizes
1511          */
1512         if (tf_em_validate_num_entries(tbl_scope_cb, parms))
1513                 goto cleanup;
1514
1515         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1516                 /*
1517                  * Allocate tables and signal configuration to FW
1518                  */
1519                 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
1520                 if (rc) {
1521                         TFP_DRV_LOG(ERR,
1522                                     "EEM: Unable to register for EEM ctx,"
1523                                     " rc:%s\n",
1524                                     strerror(-rc));
1525                         goto cleanup;
1526                 }
1527
1528                 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
1529                 rc = tf_msg_em_cfg(tfp,
1530                                    em_tables[TF_KEY0_TABLE].num_entries,
1531                                    em_tables[TF_KEY0_TABLE].ctx_id,
1532                                    em_tables[TF_KEY1_TABLE].ctx_id,
1533                                    em_tables[TF_RECORD_TABLE].ctx_id,
1534                                    em_tables[TF_EFC_TABLE].ctx_id,
1535                                    parms->hw_flow_cache_flush_timer,
1536                                    dir);
1537                 if (rc) {
1538                         TFP_DRV_LOG(ERR,
1539                                     "TBL: Unable to configure EEM in firmware"
1540                                     " rc:%s\n",
1541                                     strerror(-rc));
1542                         goto cleanup_full;
1543                 }
1544
1545                 rc = tf_msg_em_op(tfp,
1546                                   dir,
1547                                   HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
1548
1549                 if (rc) {
1550                         TFP_DRV_LOG(ERR,
1551                                     "EEM: Unable to enable EEM in firmware"
1552                                     " rc:%s\n",
1553                                     strerror(-rc));
1554                         goto cleanup_full;
1555                 }
1556
1557                 /* Allocate the pool of offsets of the external memory.
1558                  * Initially, this is a single fixed size pool for all external
1559                  * actions related to a single table scope.
1560                  */
1561                 rc = tf_create_tbl_pool_external(dir,
1562                                     tbl_scope_cb,
1563                                     em_tables[TF_RECORD_TABLE].num_entries,
1564                                     em_tables[TF_RECORD_TABLE].entry_size);
1565                 if (rc) {
1566                         TFP_DRV_LOG(ERR,
1567                                     "%s TBL: Unable to allocate idx pools %s\n",
1568                                     tf_dir_2_str(dir),
1569                                     strerror(-rc));
1570                         goto cleanup_full;
1571                 }
1572         }
1573
1574         return 0;
1575
1576 cleanup_full:
1577         free_parms.tbl_scope_id = index;
1578         tf_free_eem_tbl_scope_cb(tfp, &free_parms);
1579         return -EINVAL;
1580
1581 cleanup:
1582         /* Free Table control block */
1583         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1584         return -EINVAL;
1585 }
1586
1587 /* API defined in tf_core.h */
1588 int
1589 tf_set_tbl_entry(struct tf *tfp,
1590                  struct tf_set_tbl_entry_parms *parms)
1591 {
1592         int rc = 0;
1593         struct tf_tbl_scope_cb *tbl_scope_cb;
1594         struct tf_session *session;
1595
1596         TF_CHECK_PARMS_SESSION(tfp, parms);
1597
1598         if (parms->data == NULL) {
1599                 TFP_DRV_LOG(ERR,
1600                             "%s, invalid parms->data\n",
1601                             tf_dir_2_str(parms->dir));
1602                 return -EINVAL;
1603         }
1604
1605         if (parms->type == TF_TBL_TYPE_EXT) {
1606                 void *base_addr;
1607                 uint32_t offset = parms->idx;
1608                 uint32_t tbl_scope_id;
1609
1610                 session = (struct tf_session *)(tfp->session->core_data);
1611
1612                 tbl_scope_id = parms->tbl_scope_id;
1613
1614                 if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1615                         TFP_DRV_LOG(ERR,
1616                                     "%s, Table scope not allocated\n",
1617                                     tf_dir_2_str(parms->dir));
1618                         return -EINVAL;
1619                 }
1620
1621                 /* Get the table scope control block associated with the
1622                  * external pool
1623                  */
1624                 tbl_scope_cb = tbl_scope_cb_find(session, tbl_scope_id);
1625
1626                 if (tbl_scope_cb == NULL) {
1627                         TFP_DRV_LOG(ERR,
1628                                     "%s, table scope error\n",
1629                                     tf_dir_2_str(parms->dir));
1630                                 return -EINVAL;
1631                 }
1632
1633                 /* External table, implicitly the Action table */
1634                 base_addr = (void *)(uintptr_t)
1635                 hcapi_get_table_page(&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_RECORD_TABLE], offset);
1636
1637                 if (base_addr == NULL) {
1638                         TFP_DRV_LOG(ERR,
1639                                     "%s, Base address lookup failed\n",
1640                                     tf_dir_2_str(parms->dir));
1641                         return -EINVAL;
1642                 }
1643
1644                 offset %= TF_EM_PAGE_SIZE;
1645                 rte_memcpy((char *)base_addr + offset,
1646                            parms->data,
1647                            parms->data_sz_in_bytes);
1648         } else {
1649                 /* Internal table type processing */
1650                 rc = tf_set_tbl_entry_internal(tfp, parms);
1651                 if (rc) {
1652                         TFP_DRV_LOG(ERR,
1653                                     "%s, Set failed, type:%d, rc:%s\n",
1654                                     tf_dir_2_str(parms->dir),
1655                                     parms->type,
1656                                     strerror(-rc));
1657                 }
1658         }
1659
1660         return rc;
1661 }
1662
1663 /* API defined in tf_core.h */
1664 int
1665 tf_get_tbl_entry(struct tf *tfp,
1666                  struct tf_get_tbl_entry_parms *parms)
1667 {
1668         int rc = 0;
1669
1670         TF_CHECK_PARMS_SESSION(tfp, parms);
1671
1672         if (parms->type == TF_TBL_TYPE_EXT) {
1673                 /* Not supported, yet */
1674                 TFP_DRV_LOG(ERR,
1675                             "%s, External table type not supported\n",
1676                             tf_dir_2_str(parms->dir));
1677
1678                 rc = -EOPNOTSUPP;
1679         } else {
1680                 /* Internal table type processing */
1681                 rc = tf_get_tbl_entry_internal(tfp, parms);
1682                 if (rc)
1683                         TFP_DRV_LOG(ERR,
1684                                     "%s, Get failed, type:%d, rc:%s\n",
1685                                     tf_dir_2_str(parms->dir),
1686                                     parms->type,
1687                                     strerror(-rc));
1688         }
1689
1690         return rc;
1691 }
1692
1693 /* API defined in tf_core.h */
1694 int
1695 tf_bulk_get_tbl_entry(struct tf *tfp,
1696                  struct tf_bulk_get_tbl_entry_parms *parms)
1697 {
1698         int rc = 0;
1699
1700         TF_CHECK_PARMS_SESSION(tfp, parms);
1701
1702         if (parms->type == TF_TBL_TYPE_EXT) {
1703                 /* Not supported, yet */
1704                 TFP_DRV_LOG(ERR,
1705                             "%s, External table type not supported\n",
1706                             tf_dir_2_str(parms->dir));
1707
1708                 rc = -EOPNOTSUPP;
1709         } else {
1710                 /* Internal table type processing */
1711                 rc = tf_bulk_get_tbl_entry_internal(tfp, parms);
1712                 if (rc)
1713                         TFP_DRV_LOG(ERR,
1714                                     "%s, Bulk get failed, type:%d, rc:%s\n",
1715                                     tf_dir_2_str(parms->dir),
1716                                     parms->type,
1717                                     strerror(-rc));
1718         }
1719
1720         return rc;
1721 }
1722
1723 /* API defined in tf_core.h */
1724 int
1725 tf_alloc_tbl_scope(struct tf *tfp,
1726                    struct tf_alloc_tbl_scope_parms *parms)
1727 {
1728         int rc;
1729
1730         TF_CHECK_PARMS_SESSION_NO_DIR(tfp, parms);
1731
1732         rc = tf_alloc_eem_tbl_scope(tfp, parms);
1733
1734         return rc;
1735 }
1736
1737 /* API defined in tf_core.h */
1738 int
1739 tf_free_tbl_scope(struct tf *tfp,
1740                   struct tf_free_tbl_scope_parms *parms)
1741 {
1742         int rc;
1743
1744         TF_CHECK_PARMS_SESSION_NO_DIR(tfp, parms);
1745
1746         /* free table scope and all associated resources */
1747         rc = tf_free_eem_tbl_scope_cb(tfp, parms);
1748
1749         return rc;
1750 }
1751
1752 /* API defined in tf_core.h */
1753 int
1754 tf_alloc_tbl_entry(struct tf *tfp,
1755                    struct tf_alloc_tbl_entry_parms *parms)
1756 {
1757         int rc;
1758 #if (TF_SHADOW == 1)
1759         struct tf_session *tfs;
1760 #endif /* TF_SHADOW */
1761
1762         TF_CHECK_PARMS_SESSION(tfp, parms);
1763         /*
1764          * No shadow copy support for external tables, allocate and return
1765          */
1766         if (parms->type == TF_TBL_TYPE_EXT) {
1767                 rc = tf_alloc_tbl_entry_pool_external(tfp, parms);
1768                 return rc;
1769         }
1770
1771 #if (TF_SHADOW == 1)
1772         tfs = (struct tf_session *)(tfp->session->core_data);
1773
1774         /* Search the Shadow DB for requested element. If not found go
1775          * allocate one from the Session Pool
1776          */
1777         if (parms->search_enable && tfs->shadow_copy) {
1778                 rc = tf_alloc_tbl_entry_shadow(tfs, parms);
1779                 /* Entry found and parms populated with return data */
1780                 if (rc == 0)
1781                         return rc;
1782         }
1783 #endif /* TF_SHADOW */
1784
1785         rc = tf_alloc_tbl_entry_pool_internal(tfp, parms);
1786         if (rc)
1787                 TFP_DRV_LOG(ERR, "%s, Alloc failed, rc:%s\n",
1788                             tf_dir_2_str(parms->dir),
1789                             strerror(-rc));
1790
1791         return rc;
1792 }
1793
1794 /* API defined in tf_core.h */
1795 int
1796 tf_free_tbl_entry(struct tf *tfp,
1797                   struct tf_free_tbl_entry_parms *parms)
1798 {
1799         int rc;
1800 #if (TF_SHADOW == 1)
1801         struct tf_session *tfs;
1802 #endif /* TF_SHADOW */
1803
1804         TF_CHECK_PARMS_SESSION(tfp, parms);
1805
1806         /*
1807          * No shadow of external tables so just free the entry
1808          */
1809         if (parms->type == TF_TBL_TYPE_EXT) {
1810                 rc = tf_free_tbl_entry_pool_external(tfp, parms);
1811                 return rc;
1812         }
1813
1814 #if (TF_SHADOW == 1)
1815         tfs = (struct tf_session *)(tfp->session->core_data);
1816
1817         /* Search the Shadow DB for requested element. If not found go
1818          * allocate one from the Session Pool
1819          */
1820         if (parms->search_enable && tfs->shadow_copy) {
1821                 rc = tf_free_tbl_entry_shadow(tfs, parms);
1822                 /* Entry free'ed and parms populated with return data */
1823                 if (rc == 0)
1824                         return rc;
1825         }
1826 #endif /* TF_SHADOW */
1827
1828         rc = tf_free_tbl_entry_pool_internal(tfp, parms);
1829
1830         if (rc)
1831                 TFP_DRV_LOG(ERR, "%s, Alloc failed, rc:%s\n",
1832                             tf_dir_2_str(parms->dir),
1833                             strerror(-rc));
1834         return rc;
1835 }
1836
1837
1838 static void
1839 tf_dump_link_page_table(struct hcapi_cfa_em_page_tbl *tp,
1840                         struct hcapi_cfa_em_page_tbl *tp_next)
1841 {
1842         uint64_t *pg_va;
1843         uint32_t i;
1844         uint32_t j;
1845         uint32_t k = 0;
1846
1847         printf("pg_count:%d pg_size:0x%x\n",
1848                tp->pg_count,
1849                tp->pg_size);
1850         for (i = 0; i < tp->pg_count; i++) {
1851                 pg_va = tp->pg_va_tbl[i];
1852                 printf("\t%p\n", (void *)pg_va);
1853                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
1854                         printf("\t\t%p\n", (void *)(uintptr_t)pg_va[j]);
1855                         if (((pg_va[j] & 0x7) ==
1856                              tfp_cpu_to_le_64(PTU_PTE_LAST |
1857                                               PTU_PTE_VALID)))
1858                                 return;
1859
1860                         if (!(pg_va[j] & tfp_cpu_to_le_64(PTU_PTE_VALID))) {
1861                                 printf("** Invalid entry **\n");
1862                                 return;
1863                         }
1864
1865                         if (++k >= tp_next->pg_count) {
1866                                 printf("** Shouldn't get here **\n");
1867                                 return;
1868                         }
1869                 }
1870         }
1871 }
1872
1873 void tf_dump_dma(struct tf *tfp, uint32_t tbl_scope_id);
1874
1875 void tf_dump_dma(struct tf *tfp, uint32_t tbl_scope_id)
1876 {
1877         struct tf_session      *session;
1878         struct tf_tbl_scope_cb *tbl_scope_cb;
1879         struct hcapi_cfa_em_page_tbl *tp;
1880         struct hcapi_cfa_em_page_tbl *tp_next;
1881         struct hcapi_cfa_em_table *tbl;
1882         int i;
1883         int j;
1884         int dir;
1885
1886         printf("called %s\n", __func__);
1887
1888         /* find session struct */
1889         session = (struct tf_session *)tfp->session->core_data;
1890
1891         /* find control block for table scope */
1892         tbl_scope_cb = tbl_scope_cb_find(session,
1893                                          tbl_scope_id);
1894         if (tbl_scope_cb == NULL)
1895                 PMD_DRV_LOG(ERR, "No table scope\n");
1896
1897         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1898                 printf("Direction %s:\n", (dir == TF_DIR_RX ? "Rx" : "Tx"));
1899
1900                 for (j = TF_KEY0_TABLE; j < TF_MAX_TABLE; j++) {
1901                         tbl = &tbl_scope_cb->em_ctx_info[dir].em_tables[j];
1902                         printf
1903         ("Table: j:%d type:%d num_entries:%d entry_size:0x%x num_lvl:%d ",
1904                                j,
1905                                tbl->type,
1906                                tbl->num_entries,
1907                                tbl->entry_size,
1908                                tbl->num_lvl);
1909                         if (tbl->pg_tbl[0].pg_va_tbl &&
1910                             tbl->pg_tbl[0].pg_pa_tbl)
1911                                 printf("%p %p\n",
1912                                tbl->pg_tbl[0].pg_va_tbl[0],
1913                                (void *)(uintptr_t)tbl->pg_tbl[0].pg_pa_tbl[0]);
1914                         for (i = 0; i < tbl->num_lvl - 1; i++) {
1915                                 printf("Level:%d\n", i);
1916                                 tp = &tbl->pg_tbl[i];
1917                                 tp_next = &tbl->pg_tbl[i + 1];
1918                                 tf_dump_link_page_table(tp, tp_next);
1919                         }
1920                         printf("\n");
1921                 }
1922         }
1923 }