drivers/net/bnxt/tf_core/tf_tbl.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 /* Truflow Table APIs and supporting code */
   7
   8 #include <stdio.h>
   9 #include <string.h>
  10 #include <stdbool.h>
  11 #include <math.h>
  12 #include <sys/param.h>
  13 #include <rte_common.h>
  14 #include <rte_errno.h>
  15 #include "hsi_struct_def_dpdk.h"
  16
  17 #include "tf_core.h"
  18 #include "tf_util.h"
  19 #include "tf_em.h"
  20 #include "tf_msg.h"
  21 #include "tfp.h"
  22 #include "hwrm_tf.h"
  23 #include "bnxt.h"
  24 #include "tf_resources.h"
  25 #include "tf_rm.h"
  26
  27 #define PTU_PTE_VALID          0x1UL
  28 #define PTU_PTE_LAST           0x2UL
  29 #define PTU_PTE_NEXT_TO_LAST   0x4UL
  30
  31 /* Number of pointers per page_size */
  32 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  33
  34 #define TF_EM_PG_SZ_4K        (1 << 12)
  35 #define TF_EM_PG_SZ_8K        (1 << 13)
  36 #define TF_EM_PG_SZ_64K       (1 << 16)
  37 #define TF_EM_PG_SZ_256K      (1 << 18)
  38 #define TF_EM_PG_SZ_1M        (1 << 20)
  39 #define TF_EM_PG_SZ_2M        (1 << 21)
  40 #define TF_EM_PG_SZ_4M        (1 << 22)
  41 #define TF_EM_PG_SZ_1G        (1 << 30)
  42
  43 #define TF_EM_CTX_ID_INVALID   0xFFFF
  44
  45 #define TF_EM_MIN_ENTRIES     (1 << 15) /* 32K */
  46 #define TF_EM_MAX_ENTRIES     (1 << 27) /* 128M */
  47
  48 /**
  49  * Function to free a page table
  50  *
  51  * [in] tp
  52  *   Pointer to the page table to free
  53  */
  54 static void
  55 tf_em_free_pg_tbl(struct tf_em_page_tbl *tp)
  56 {
  57         uint32_t i;
  58
  59         for (i = 0; i < tp->pg_count; i++) {
  60                 if (!tp->pg_va_tbl[i]) {
  61                         PMD_DRV_LOG(WARNING,
  62                                     "No map for page %d table %016" PRIu64 "\n",
  63                                     i,
  64                                     (uint64_t)(uintptr_t)tp);
  65                         continue;
  66                 }
  67
  68                 tfp_free(tp->pg_va_tbl[i]);
  69                 tp->pg_va_tbl[i] = NULL;
  70         }
  71
  72         tp->pg_count = 0;
  73         tfp_free(tp->pg_va_tbl);
  74         tp->pg_va_tbl = NULL;
  75         tfp_free(tp->pg_pa_tbl);
  76         tp->pg_pa_tbl = NULL;
  77 }
  78
  79 /**
  80  * Function to free an EM table
  81  *
  82  * [in] tbl
  83  *   Pointer to the EM table to free
  84  */
  85 static void
  86 tf_em_free_page_table(struct tf_em_table *tbl)
  87 {
  88         struct tf_em_page_tbl *tp;
  89         int i;
  90
  91         for (i = 0; i < tbl->num_lvl; i++) {
  92                 tp = &tbl->pg_tbl[i];
  93
  94                 PMD_DRV_LOG(INFO,
  95                            "EEM: Freeing page table: size %u lvl %d cnt %u\n",
  96                            TF_EM_PAGE_SIZE,
  97                             i,
  98                             tp->pg_count);
  99
 100                 tf_em_free_pg_tbl(tp);
 101         }
 102
 103         tbl->l0_addr = NULL;
 104         tbl->l0_dma_addr = 0;
 105         tbl->num_lvl = 0;
 106         tbl->num_data_pages = 0;
 107 }
 108
 109 /**
 110  * Allocation of page tables
 111  *
 112  * [in] tfp
 113  *   Pointer to a TruFlow handle
 114  *
 115  * [in] pg_count
 116  *   Page count to allocate
 117  *
 118  * [in] pg_size
 119  *   Size of each page
 120  *
 121  * Returns:
 122  *   0       - Success
 123  *   -ENOMEM - Out of memory
 124  */
 125 static int
 126 tf_em_alloc_pg_tbl(struct tf_em_page_tbl *tp,
 127                    uint32_t pg_count,
 128                    uint32_t pg_size)
 129 {
 130         uint32_t i;
 131         struct tfp_calloc_parms parms;
 132
 133         parms.nitems = pg_count;
 134         parms.size = sizeof(void *);
 135         parms.alignment = 0;
 136
 137         if (tfp_calloc(&parms) != 0)
 138                 return -ENOMEM;
 139
 140         tp->pg_va_tbl = parms.mem_va;
 141
 142         if (tfp_calloc(&parms) != 0) {
 143                 tfp_free(tp->pg_va_tbl);
 144                 return -ENOMEM;
 145         }
 146
 147         tp->pg_pa_tbl = parms.mem_va;
 148
 149         tp->pg_count = 0;
 150         tp->pg_size = pg_size;
 151
 152         for (i = 0; i < pg_count; i++) {
 153                 parms.nitems = 1;
 154                 parms.size = pg_size;
 155                 parms.alignment = TF_EM_PAGE_ALIGNMENT;
 156
 157                 if (tfp_calloc(&parms) != 0)
 158                         goto cleanup;
 159
 160                 tp->pg_pa_tbl[i] = (uintptr_t)parms.mem_pa;
 161                 tp->pg_va_tbl[i] = parms.mem_va;
 162
 163                 memset(tp->pg_va_tbl[i], 0, pg_size);
 164                 tp->pg_count++;
 165         }
 166
 167         return 0;
 168
 169 cleanup:
 170         tf_em_free_pg_tbl(tp);
 171         return -ENOMEM;
 172 }
 173
 174 /**
 175  * Allocates EM page tables
 176  *
 177  * [in] tbl
 178  *   Table to allocate pages for
 179  *
 180  * Returns:
 181  *   0       - Success
 182  *   -ENOMEM - Out of memory
 183  */
 184 static int
 185 tf_em_alloc_page_table(struct tf_em_table *tbl)
 186 {
 187         struct tf_em_page_tbl *tp;
 188         int rc = 0;
 189         int i;
 190         uint32_t j;
 191
 192         for (i = 0; i < tbl->num_lvl; i++) {
 193                 tp = &tbl->pg_tbl[i];
 194
 195                 rc = tf_em_alloc_pg_tbl(tp,
 196                                         tbl->page_cnt[i],
 197                                         TF_EM_PAGE_SIZE);
 198                 if (rc) {
 199                         PMD_DRV_LOG(WARNING,
 200                                 "Failed to allocate page table: lvl: %d\n",
 201                                 i);
 202                         goto cleanup;
 203                 }
 204
 205                 for (j = 0; j < tp->pg_count; j++) {
 206                         PMD_DRV_LOG(INFO,
 207                                 "EEM: Allocated page table: size %u lvl %d cnt"
 208                                 " %u VA:%p PA:%p\n",
 209                                 TF_EM_PAGE_SIZE,
 210                                 i,
 211                                 tp->pg_count,
 212                                 (uint32_t *)tp->pg_va_tbl[j],
 213                                 (uint32_t *)(uintptr_t)tp->pg_pa_tbl[j]);
 214                 }
 215         }
 216         return rc;
 217
 218 cleanup:
 219         tf_em_free_page_table(tbl);
 220         return rc;
 221 }
 222
 223 /**
 224  * Links EM page tables
 225  *
 226  * [in] tp
 227  *   Pointer to page table
 228  *
 229  * [in] tp_next
 230  *   Pointer to the next page table
 231  *
 232  * [in] set_pte_last
 233  *   Flag controlling if the page table is last
 234  */
 235 static void
 236 tf_em_link_page_table(struct tf_em_page_tbl *tp,
 237                       struct tf_em_page_tbl *tp_next,
 238                       bool set_pte_last)
 239 {
 240         uint64_t *pg_pa = tp_next->pg_pa_tbl;
 241         uint64_t *pg_va;
 242         uint64_t valid;
 243         uint32_t k = 0;
 244         uint32_t i;
 245         uint32_t j;
 246
 247         for (i = 0; i < tp->pg_count; i++) {
 248                 pg_va = tp->pg_va_tbl[i];
 249
 250                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
 251                         if (k == tp_next->pg_count - 2 && set_pte_last)
 252                                 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
 253                         else if (k == tp_next->pg_count - 1 && set_pte_last)
 254                                 valid = PTU_PTE_LAST | PTU_PTE_VALID;
 255                         else
 256                                 valid = PTU_PTE_VALID;
 257
 258                         pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
 259                         if (++k >= tp_next->pg_count)
 260                                 return;
 261                 }
 262         }
 263 }
 264
 265 /**
 266  * Setup a EM page table
 267  *
 268  * [in] tbl
 269  *   Pointer to EM page table
 270  */
 271 static void
 272 tf_em_setup_page_table(struct tf_em_table *tbl)
 273 {
 274         struct tf_em_page_tbl *tp_next;
 275         struct tf_em_page_tbl *tp;
 276         bool set_pte_last = 0;
 277         int i;
 278
 279         for (i = 0; i < tbl->num_lvl - 1; i++) {
 280                 tp = &tbl->pg_tbl[i];
 281                 tp_next = &tbl->pg_tbl[i + 1];
 282                 if (i == tbl->num_lvl - 2)
 283                         set_pte_last = 1;
 284                 tf_em_link_page_table(tp, tp_next, set_pte_last);
 285         }
 286
 287         tbl->l0_addr = tbl->pg_tbl[PT_LVL_0].pg_va_tbl[0];
 288         tbl->l0_dma_addr = tbl->pg_tbl[PT_LVL_0].pg_pa_tbl[0];
 289 }
 290
 291 /**
 292  * Given the page size, size of each data item (entry size),
 293  * and the total number of entries needed, determine the number
 294  * of page table levels and the number of data pages required.
 295  *
 296  * [in] page_size
 297  *   Page size
 298  *
 299  * [in] entry_size
 300  *   Entry size
 301  *
 302  * [in] num_entries
 303  *   Number of entries needed
 304  *
 305  * [out] num_data_pages
 306  *   Number of pages required
 307  *
 308  * Returns:
 309  *   Success  - Number of EM page levels required
 310  *   -ENOMEM  - Out of memory
 311  */
 312 static int
 313 tf_em_size_page_tbl_lvl(uint32_t page_size,
 314                         uint32_t entry_size,
 315                         uint32_t num_entries,
 316                         uint64_t *num_data_pages)
 317 {
 318         uint64_t lvl_data_size = page_size;
 319         int lvl = PT_LVL_0;
 320         uint64_t data_size;
 321
 322         *num_data_pages = 0;
 323         data_size = (uint64_t)num_entries * entry_size;
 324
 325         while (lvl_data_size < data_size) {
 326                 lvl++;
 327
 328                 if (lvl == PT_LVL_1)
 329                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 330                                 page_size;
 331                 else if (lvl == PT_LVL_2)
 332                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 333                                 MAX_PAGE_PTRS(page_size) * page_size;
 334                 else
 335                         return -ENOMEM;
 336         }
 337
 338         *num_data_pages = roundup(data_size, page_size) / page_size;
 339
 340         return lvl;
 341 }
 342
 343 /**
 344  * Return the number of page table pages needed to
 345  * reference the given number of next level pages.
 346  *
 347  * [in] num_pages
 348  *   Number of EM pages
 349  *
 350  * [in] page_size
 351  *   Size of each EM page
 352  *
 353  * Returns:
 354  *   Number of EM page table pages
 355  */
 356 static uint32_t
 357 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 358                      uint32_t page_size)
 359 {
 360         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 361                        MAX_PAGE_PTRS(page_size);
 362         return 0;
 363 }
 364
 365 /**
 366  * Given the number of data pages, page_size and the maximum
 367  * number of page table levels (already determined), size
 368  * the number of page table pages required at each level.
 369  *
 370  * [in] max_lvl
 371  *   Max number of levels
 372  *
 373  * [in] num_data_pages
 374  *   Number of EM data pages
 375  *
 376  * [in] page_size
 377  *   Size of an EM page
 378  *
 379  * [out] *page_cnt
 380  *   EM page count
 381  */
 382 static void
 383 tf_em_size_page_tbls(int max_lvl,
 384                      uint64_t num_data_pages,
 385                      uint32_t page_size,
 386                      uint32_t *page_cnt)
 387 {
 388         if (max_lvl == PT_LVL_0) {
 389                 page_cnt[PT_LVL_0] = num_data_pages;
 390         } else if (max_lvl == PT_LVL_1) {
 391                 page_cnt[PT_LVL_1] = num_data_pages;
 392                 page_cnt[PT_LVL_0] =
 393                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_1], page_size);
 394         } else if (max_lvl == PT_LVL_2) {
 395                 page_cnt[PT_LVL_2] = num_data_pages;
 396                 page_cnt[PT_LVL_1] =
 397                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_2], page_size);
 398                 page_cnt[PT_LVL_0] =
 399                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_1], page_size);
 400         } else {
 401                 return;
 402         }
 403 }
 404
 405 /**
 406  * Size the EM table based on capabilities
 407  *
 408  * [in] tbl
 409  *   EM table to size
 410  *
 411  * Returns:
 412  *   0        - Success
 413  *   - EINVAL - Parameter error
 414  *   - ENOMEM - Out of memory
 415  */
 416 static int
 417 tf_em_size_table(struct tf_em_table *tbl)
 418 {
 419         uint64_t num_data_pages;
 420         uint32_t *page_cnt;
 421         int max_lvl;
 422         uint32_t num_entries;
 423         uint32_t cnt = TF_EM_MIN_ENTRIES;
 424
 425         /* Ignore entry if both size and number are zero */
 426         if (!tbl->entry_size && !tbl->num_entries)
 427                 return 0;
 428
 429         /* If only one is set then error */
 430         if (!tbl->entry_size || !tbl->num_entries)
 431                 return -EINVAL;
 432
 433         /* Determine number of page table levels and the number
 434          * of data pages needed to process the given eem table.
 435          */
 436         if (tbl->type == RECORD_TABLE) {
 437                 /*
 438                  * For action records just a memory size is provided. Work
 439                  * backwards to resolve to number of entries
 440                  */
 441                 num_entries = tbl->num_entries / tbl->entry_size;
 442                 if (num_entries < TF_EM_MIN_ENTRIES) {
 443                         num_entries = TF_EM_MIN_ENTRIES;
 444                 } else {
 445                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 446                                 cnt *= 2;
 447                         num_entries = cnt;
 448                 }
 449         } else {
 450                 num_entries = tbl->num_entries;
 451         }
 452
 453         max_lvl = tf_em_size_page_tbl_lvl(TF_EM_PAGE_SIZE,
 454                                           tbl->entry_size,
 455                                           tbl->num_entries,
 456                                           &num_data_pages);
 457         if (max_lvl < 0) {
 458                 PMD_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 459                 PMD_DRV_LOG(WARNING,
 460                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 461                             tbl->type,
 462                             (uint64_t)num_entries * tbl->entry_size,
 463                             TF_EM_PAGE_SIZE);
 464                 return -ENOMEM;
 465         }
 466
 467         tbl->num_lvl = max_lvl + 1;
 468         tbl->num_data_pages = num_data_pages;
 469
 470         /* Determine the number of pages needed at each level */
 471         page_cnt = tbl->page_cnt;
 472         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 473         tf_em_size_page_tbls(max_lvl, num_data_pages, TF_EM_PAGE_SIZE,
 474                                 page_cnt);
 475
 476         PMD_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 477         PMD_DRV_LOG(INFO,
 478                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 " l0: %u l1: %u l2: %u\n",
 479                     max_lvl + 1,
 480                     (uint64_t)num_data_pages * TF_EM_PAGE_SIZE,
 481                     num_data_pages,
 482                     page_cnt[PT_LVL_0],
 483                     page_cnt[PT_LVL_1],
 484                     page_cnt[PT_LVL_2]);
 485
 486         return 0;
 487 }
 488
 489 /**
 490  * Unregisters EM Ctx in Firmware
 491  *
 492  * [in] tfp
 493  *   Pointer to a TruFlow handle
 494  *
 495  * [in] tbl_scope_cb
 496  *   Pointer to a table scope control block
 497  *
 498  * [in] dir
 499  *   Receive or transmit direction
 500  */
 501 static void
 502 tf_em_ctx_unreg(struct tf *tfp,
 503                 struct tf_tbl_scope_cb *tbl_scope_cb,
 504                 int dir)
 505 {
 506         struct tf_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 507         struct tf_em_table *tbl;
 508         int i;
 509
 510         for (i = KEY0_TABLE; i < MAX_TABLE; i++) {
 511                 tbl = &ctxp->em_tables[i];
 512
 513                 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
 514                         tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
 515                         tf_em_free_page_table(tbl);
 516                 }
 517         }
 518 }
 519
 520 /**
 521  * Registers EM Ctx in Firmware
 522  *
 523  * [in] tfp
 524  *   Pointer to a TruFlow handle
 525  *
 526  * [in] tbl_scope_cb
 527  *   Pointer to a table scope control block
 528  *
 529  * [in] dir
 530  *   Receive or transmit direction
 531  *
 532  * Returns:
 533  *   0       - Success
 534  *   -ENOMEM - Out of Memory
 535  */
 536 static int
 537 tf_em_ctx_reg(struct tf *tfp,
 538               struct tf_tbl_scope_cb *tbl_scope_cb,
 539               int dir)
 540 {
 541         struct tf_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 542         struct tf_em_table *tbl;
 543         int rc = 0;
 544         int i;
 545
 546         for (i = KEY0_TABLE; i < MAX_TABLE; i++) {
 547                 tbl = &ctxp->em_tables[i];
 548
 549                 if (tbl->num_entries && tbl->entry_size) {
 550                         rc = tf_em_size_table(tbl);
 551
 552                         if (rc)
 553                                 goto cleanup;
 554
 555                         rc = tf_em_alloc_page_table(tbl);
 556                         if (rc)
 557                                 goto cleanup;
 558
 559                         tf_em_setup_page_table(tbl);
 560                         rc = tf_msg_em_mem_rgtr(tfp,
 561                                                 tbl->num_lvl - 1,
 562                                                 TF_EM_PAGE_SIZE_ENUM,
 563                                                 tbl->l0_dma_addr,
 564                                                 &tbl->ctx_id);
 565                         if (rc)
 566                                 goto cleanup;
 567                 }
 568         }
 569         return rc;
 570
 571 cleanup:
 572         tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
 573         return rc;
 574 }
 575
 576 /**
 577  * Validates EM number of entries requested
 578  *
 579  * [in] tbl_scope_cb
 580  *   Pointer to table scope control block to be populated
 581  *
 582  * [in] parms
 583  *   Pointer to input parameters
 584  *
 585  * Returns:
 586  *   0       - Success
 587  *   -EINVAL - Parameter error
 588  */
 589 static int
 590 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 591                            struct tf_alloc_tbl_scope_parms *parms)
 592 {
 593         uint32_t cnt;
 594
 595         if (parms->rx_mem_size_in_mb != 0) {
 596                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 597                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 598                                      + 1);
 599                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 600                                         TF_MEGABYTE) / (key_b + action_b);
 601
 602                 if (num_entries < TF_EM_MIN_ENTRIES) {
 603                         PMD_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 604                                     "%uMB\n",
 605                                     parms->rx_mem_size_in_mb);
 606                         return -EINVAL;
 607                 }
 608
 609                 cnt = TF_EM_MIN_ENTRIES;
 610                 while (num_entries > cnt &&
 611                        cnt <= TF_EM_MAX_ENTRIES)
 612                         cnt *= 2;
 613
 614                 if (cnt > TF_EM_MAX_ENTRIES) {
 615                         PMD_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 616                                     "%u\n",
 617                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 618                         return -EINVAL;
 619                 }
 620
 621                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 622         } else {
 623                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 624                     TF_EM_MIN_ENTRIES ||
 625                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 626                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 627                         PMD_DRV_LOG(ERR,
 628                                     "EEM: Invalid number of Rx flows "
 629                                     "requested:%u max:%u\n",
 630                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 631                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 632                         return -EINVAL;
 633                 }
 634
 635                 /* must be a power-of-2 supported value
 636                  * in the range 32K - 128M
 637                  */
 638                 cnt = TF_EM_MIN_ENTRIES;
 639                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 640                        cnt <= TF_EM_MAX_ENTRIES)
 641                         cnt *= 2;
 642
 643                 if (cnt > TF_EM_MAX_ENTRIES) {
 644                         PMD_DRV_LOG(ERR,
 645                                     "EEM: Invalid number of Rx requested: %u\n",
 646                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 647                         return -EINVAL;
 648                 }
 649         }
 650
 651         if (parms->tx_mem_size_in_mb != 0) {
 652                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 653                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 654                                      + 1);
 655                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 656                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 657                         (key_b + action_b);
 658
 659                 if (num_entries < TF_EM_MIN_ENTRIES) {
 660                         PMD_DRV_LOG(ERR,
 661                                     "EEM: Insufficient memory requested:%uMB\n",
 662                                     parms->rx_mem_size_in_mb);
 663                         return -EINVAL;
 664                 }
 665
 666                 cnt = TF_EM_MIN_ENTRIES;
 667                 while (num_entries > cnt &&
 668                        cnt <= TF_EM_MAX_ENTRIES)
 669                         cnt *= 2;
 670
 671                 if (cnt > TF_EM_MAX_ENTRIES) {
 672                         PMD_DRV_LOG(ERR,
 673                                     "EEM: Invalid number of Tx requested: %u\n",
 674                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 675                         return -EINVAL;
 676                 }
 677
 678                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 679         } else {
 680                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 681                     TF_EM_MIN_ENTRIES ||
 682                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 683                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 684                         PMD_DRV_LOG(ERR,
 685                                     "EEM: Invalid number of Tx flows "
 686                                     "requested:%u max:%u\n",
 687                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 688                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 689                         return -EINVAL;
 690                 }
 691
 692                 cnt = TF_EM_MIN_ENTRIES;
 693                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 694                        cnt <= TF_EM_MAX_ENTRIES)
 695                         cnt *= 2;
 696
 697                 if (cnt > TF_EM_MAX_ENTRIES) {
 698                         PMD_DRV_LOG(ERR,
 699                                     "EEM: Invalid number of Tx requested: %u\n",
 700                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 701                         return -EINVAL;
 702                 }
 703         }
 704
 705         if (parms->rx_num_flows_in_k != 0 &&
 706             (parms->rx_max_key_sz_in_bits / 8 == 0)) {
 707                 PMD_DRV_LOG(ERR,
 708                             "EEM: Rx key size required: %u\n",
 709                             (parms->rx_max_key_sz_in_bits));
 710                 return -EINVAL;
 711         }
 712
 713         if (parms->tx_num_flows_in_k != 0 &&
 714             (parms->tx_max_key_sz_in_bits / 8 == 0)) {
 715                 PMD_DRV_LOG(ERR,
 716                             "EEM: Tx key size required: %u\n",
 717                             (parms->tx_max_key_sz_in_bits));
 718                 return -EINVAL;
 719         }
 720         /* Rx */
 721         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY0_TABLE].num_entries =
 722                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 723         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY0_TABLE].entry_size =
 724                 parms->rx_max_key_sz_in_bits / 8;
 725
 726         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY1_TABLE].num_entries =
 727                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 728         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY1_TABLE].entry_size =
 729                 parms->rx_max_key_sz_in_bits / 8;
 730
 731         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[RECORD_TABLE].num_entries =
 732                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 733         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[RECORD_TABLE].entry_size =
 734                 parms->rx_max_action_entry_sz_in_bits / 8;
 735
 736         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[EFC_TABLE].num_entries =
 737                 0;
 738
 739         /* Tx */
 740         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY0_TABLE].num_entries =
 741                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 742         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY0_TABLE].entry_size =
 743                 parms->tx_max_key_sz_in_bits / 8;
 744
 745         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY1_TABLE].num_entries =
 746                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 747         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY1_TABLE].entry_size =
 748                 parms->tx_max_key_sz_in_bits / 8;
 749
 750         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[RECORD_TABLE].num_entries =
 751                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 752         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[RECORD_TABLE].entry_size =
 753                 parms->tx_max_action_entry_sz_in_bits / 8;
 754
 755         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[EFC_TABLE].num_entries =
 756                 0;
 757
 758         return 0;
 759 }
 760
 761 /**
 762  * Internal function to set a Table Entry. Supports all internal Table Types
 763  *
 764  * [in] tfp
 765  *   Pointer to TruFlow handle
 766  *
 767  * [in] parms
 768  *   Pointer to input parameters
 769  *
 770  * Returns:
 771  *   0       - Success
 772  *   -EINVAL - Parameter error
 773  */
 774 static int
 775 tf_set_tbl_entry_internal(struct tf *tfp,
 776                           struct tf_set_tbl_entry_parms *parms)
 777 {
 778         int rc;
 779         int id;
 780         uint32_t index;
 781         struct bitalloc *session_pool;
 782         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 783
 784         /* Lookup the pool using the table type of the element */
 785         rc = tf_rm_lookup_tbl_type_pool(tfs,
 786                                         parms->dir,
 787                                         parms->type,
 788                                         &session_pool);
 789         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 790         if (rc)
 791                 return rc;
 792
 793         index = parms->idx;
 794
 795         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
 796             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
 797             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
 798                 PMD_DRV_LOG(ERR,
 799                             "dir:%d, Type not supported, type:%d\n",
 800                             parms->dir,
 801                             parms->type);
 802                 return -EOPNOTSUPP;
 803         }
 804
 805         /* Adjust the returned index/offset as there is no guarantee
 806          * that the start is 0 at time of RM allocation
 807          */
 808         tf_rm_convert_index(tfs,
 809                             parms->dir,
 810                             parms->type,
 811                             TF_RM_CONVERT_RM_BASE,
 812                             parms->idx,
 813                             &index);
 814
 815         /* Verify that the entry has been previously allocated */
 816         id = ba_inuse(session_pool, index);
 817         if (id != 1) {
 818                 PMD_DRV_LOG(ERR,
 819                    "dir:%d, Invalid or not allocated index, type:%d, idx:%d\n",
 820                    parms->dir,
 821                    parms->type,
 822                    index);
 823                 return -EINVAL;
 824         }
 825
 826         /* Set the entry */
 827         rc = tf_msg_set_tbl_entry(tfp,
 828                                   parms->dir,
 829                                   parms->type,
 830                                   parms->data_sz_in_bytes,
 831                                   parms->data,
 832                                   parms->idx);
 833         if (rc) {
 834                 PMD_DRV_LOG(ERR,
 835                             "dir:%d, Set failed, type:%d, rc:%d\n",
 836                             parms->dir,
 837                             parms->type,
 838                             rc);
 839         }
 840
 841         return rc;
 842 }
 843
 844 /**
 845  * Internal function to get a Table Entry. Supports all Table Types
 846  * except the TF_TBL_TYPE_EXT as that is handled as a table scope.
 847  *
 848  * [in] tfp
 849  *   Pointer to TruFlow handle
 850  *
 851  * [in] parms
 852  *   Pointer to input parameters
 853  *
 854  * Returns:
 855  *   0       - Success
 856  *   -EINVAL - Parameter error
 857  */
 858 static int
 859 tf_get_tbl_entry_internal(struct tf *tfp,
 860                           struct tf_get_tbl_entry_parms *parms)
 861 {
 862         int rc;
 863         int id;
 864         uint32_t index;
 865         struct bitalloc *session_pool;
 866         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 867
 868         /* Lookup the pool using the table type of the element */
 869         rc = tf_rm_lookup_tbl_type_pool(tfs,
 870                                         parms->dir,
 871                                         parms->type,
 872                                         &session_pool);
 873         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 874         if (rc)
 875                 return rc;
 876
 877         index = parms->idx;
 878
 879         /* Adjust the returned index/offset as there is no guarantee
 880          * that the start is 0 at time of RM allocation
 881          */
 882         tf_rm_convert_index(tfs,
 883                             parms->dir,
 884                             parms->type,
 885                             TF_RM_CONVERT_RM_BASE,
 886                             parms->idx,
 887                             &index);
 888
 889         /* Verify that the entry has been previously allocated */
 890         id = ba_inuse(session_pool, index);
 891         if (id != 1) {
 892                 PMD_DRV_LOG(ERR,
 893                    "dir:%d, Invalid or not allocated index, type:%d, idx:%d\n",
 894                    parms->dir,
 895                    parms->type,
 896                    index);
 897                 return -EINVAL;
 898         }
 899
 900         /* Get the entry */
 901         rc = tf_msg_get_tbl_entry(tfp,
 902                                   parms->dir,
 903                                   parms->type,
 904                                   parms->data_sz_in_bytes,
 905                                   parms->data,
 906                                   parms->idx);
 907         if (rc) {
 908                 PMD_DRV_LOG(ERR,
 909                             "dir:%d, Get failed, type:%d, rc:%d\n",
 910                             parms->dir,
 911                             parms->type,
 912                             rc);
 913         }
 914
 915         return rc;
 916 }
 917
 918 #if (TF_SHADOW == 1)
 919 /**
 920  * Allocate Tbl entry from the Shadow DB. Shadow DB is searched for
 921  * the requested entry. If found the ref count is incremente and
 922  * returned.
 923  *
 924  * [in] tfs
 925  *   Pointer to session
 926  * [in] parms
 927  *   Allocation parameters
 928  *
 929  * Return:
 930  *  0       - Success, entry found and ref count incremented
 931  *  -ENOENT - Failure, entry not found
 932  */
 933 static int
 934 tf_alloc_tbl_entry_shadow(struct tf_session *tfs __rte_unused,
 935                           struct tf_alloc_tbl_entry_parms *parms __rte_unused)
 936 {
 937         PMD_DRV_LOG(ERR,
 938                     "dir:%d, Entry Alloc with search not supported\n",
 939                     parms->dir);
 940
 941
 942         return -EOPNOTSUPP;
 943 }
 944
 945 /**
 946  * Free Tbl entry from the Shadow DB. Shadow DB is searched for
 947  * the requested entry. If found the ref count is decremente and
 948  * new ref_count returned.
 949  *
 950  * [in] tfs
 951  *   Pointer to session
 952  * [in] parms
 953  *   Allocation parameters
 954  *
 955  * Return:
 956  *  0       - Success, entry found and ref count decremented
 957  *  -ENOENT - Failure, entry not found
 958  */
 959 static int
 960 tf_free_tbl_entry_shadow(struct tf_session *tfs,
 961                          struct tf_free_tbl_entry_parms *parms)
 962 {
 963         PMD_DRV_LOG(ERR,
 964                     "dir:%d, Entry Free with search not supported\n",
 965                     parms->dir);
 966
 967         return -EOPNOTSUPP;
 968 }
 969 #endif /* TF_SHADOW */
 970
 971 /**
 972  * Create External Tbl pool of memory indexes.
 973  *
 974  * [in] dir
 975  *   direction
 976  * [in] tbl_scope_cb
 977  *   pointer to the table scope
 978  * [in] num_entries
 979  *   number of entries to write
 980  * [in] entry_sz_bytes
 981  *   size of each entry
 982  *
 983  * Return:
 984  *  0       - Success, entry allocated - no search support
 985  *  -ENOMEM -EINVAL -EOPNOTSUPP
 986  *          - Failure, entry not allocated, out of resources
 987  */
 988 static int
 989 tf_create_tbl_pool_external(enum tf_dir dir,
 990                             struct tf_tbl_scope_cb *tbl_scope_cb,
 991                             uint32_t num_entries,
 992                             uint32_t entry_sz_bytes)
 993 {
 994         struct tfp_calloc_parms parms;
 995         uint32_t i;
 996         int32_t j;
 997         int rc = 0;
 998         struct stack *pool = &tbl_scope_cb->ext_act_pool[dir];
 999
1000         parms.nitems = num_entries;
1001         parms.size = sizeof(uint32_t);
1002         parms.alignment = 0;
1003
1004         if (tfp_calloc(&parms) != 0) {
1005                 PMD_DRV_LOG(ERR, "%d: TBL: external pool failure %s\n",
1006                             dir, strerror(-ENOMEM));
1007                 return -ENOMEM;
1008         }
1009
1010         /* Create empty stack
1011          */
1012         rc = stack_init(num_entries, parms.mem_va, pool);
1013
1014         if (rc != 0) {
1015                 PMD_DRV_LOG(ERR, "%d: TBL: stack init failure %s\n",
1016                             dir, strerror(-rc));
1017                 goto cleanup;
1018         }
1019
1020         /* Save the  malloced memory address so that it can
1021          * be freed when the table scope is freed.
1022          */
1023         tbl_scope_cb->ext_act_pool_mem[dir] = (uint32_t *)parms.mem_va;
1024
1025         /* Fill pool with indexes in reverse
1026          */
1027         j = (num_entries - 1) * entry_sz_bytes;
1028
1029         for (i = 0; i < num_entries; i++) {
1030                 rc = stack_push(pool, j);
1031                 if (rc != 0) {
1032                         PMD_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
1033                                     tf_dir_2_str(dir), strerror(-rc));
1034                         goto cleanup;
1035                 }
1036
1037                 if (j < 0) {
1038                         PMD_DRV_LOG(ERR, "%d TBL: invalid offset (%d)\n",
1039                                     dir, j);
1040                         goto cleanup;
1041                 }
1042                 j -= entry_sz_bytes;
1043         }
1044
1045         if (!stack_is_full(pool)) {
1046                 rc = -EINVAL;
1047                 PMD_DRV_LOG(ERR, "%d TBL: stack failure %s\n",
1048                             dir, strerror(-rc));
1049                 goto cleanup;
1050         }
1051         return 0;
1052 cleanup:
1053         tfp_free((void *)parms.mem_va);
1054         return rc;
1055 }
1056
1057 /**
1058  * Destroy External Tbl pool of memory indexes.
1059  *
1060  * [in] dir
1061  *   direction
1062  * [in] tbl_scope_cb
1063  *   pointer to the table scope
1064  *
1065  */
1066 static void
1067 tf_destroy_tbl_pool_external(enum tf_dir dir,
1068                              struct tf_tbl_scope_cb *tbl_scope_cb)
1069 {
1070         uint32_t *ext_act_pool_mem =
1071                 tbl_scope_cb->ext_act_pool_mem[dir];
1072
1073         tfp_free(ext_act_pool_mem);
1074 }
1075
1076 /**
1077  * Allocate External Tbl entry from the Session Pool.
1078  *
1079  * [in] tfp
1080  *   Pointer to Truflow Handle
1081  * [in] parms
1082  *   Allocation parameters
1083  *
1084  * Return:
1085  *  0       - Success, entry allocated - no search support
1086  *  -ENOMEM -EINVAL -EOPNOTSUPP
1087  *          - Failure, entry not allocated, out of resources
1088  */
1089 static int
1090 tf_alloc_tbl_entry_pool_external(struct tf *tfp,
1091                                  struct tf_alloc_tbl_entry_parms *parms)
1092 {
1093         int rc;
1094         uint32_t index;
1095         struct tf_session *tfs;
1096         struct tf_tbl_scope_cb *tbl_scope_cb;
1097         struct stack *pool;
1098
1099         /* Check parameters */
1100         if (tfp == NULL || parms == NULL) {
1101                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1102                 return -EINVAL;
1103         }
1104
1105         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1106                 PMD_DRV_LOG(ERR,
1107                             "dir:%d, Session info invalid\n",
1108                             parms->dir);
1109                 return -EINVAL;
1110         }
1111
1112         tfs = (struct tf_session *)(tfp->session->core_data);
1113
1114         /* Get the pool info from the table scope
1115          */
1116         tbl_scope_cb = tbl_scope_cb_find(tfs, parms->tbl_scope_id);
1117
1118         if (tbl_scope_cb == NULL) {
1119                 PMD_DRV_LOG(ERR,
1120                                         "%s, table scope not allocated\n",
1121                                         tf_dir_2_str(parms->dir));
1122                 return -EINVAL;
1123         }
1124         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
1125
1126         /* Allocate an element
1127          */
1128         rc = stack_pop(pool, &index);
1129
1130         if (rc != 0) {
1131                 PMD_DRV_LOG(ERR,
1132                    "dir:%d, Allocation failed, type:%d\n",
1133                    parms->dir,
1134                    parms->type);
1135                 return rc;
1136         }
1137         parms->idx = index;
1138         return rc;
1139 }
1140
1141 /**
1142  * Allocate Internal Tbl entry from the Session Pool.
1143  *
1144  * [in] tfp
1145  *   Pointer to Truflow Handle
1146  * [in] parms
1147  *   Allocation parameters
1148  *
1149  * Return:
1150  *  0       - Success, entry found and ref count decremented
1151  *  -ENOMEM - Failure, entry not allocated, out of resources
1152  */
1153 static int
1154 tf_alloc_tbl_entry_pool_internal(struct tf *tfp,
1155                                  struct tf_alloc_tbl_entry_parms *parms)
1156 {
1157         int rc;
1158         int id;
1159         int free_cnt;
1160         uint32_t index;
1161         struct bitalloc *session_pool;
1162         struct tf_session *tfs;
1163
1164         /* Check parameters */
1165         if (tfp == NULL || parms == NULL) {
1166                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1167                 return -EINVAL;
1168         }
1169
1170         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1171                 PMD_DRV_LOG(ERR,
1172                             "dir:%d, Session info invalid\n",
1173                             parms->dir);
1174                 return -EINVAL;
1175         }
1176
1177         tfs = (struct tf_session *)(tfp->session->core_data);
1178
1179         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1180             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1181             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1182             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1183             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1184             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B &&
1185             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
1186                 PMD_DRV_LOG(ERR,
1187                             "dir:%d, Type not supported, type:%d\n",
1188                             parms->dir,
1189                             parms->type);
1190                 return -EOPNOTSUPP;
1191         }
1192
1193         /* Lookup the pool using the table type of the element */
1194         rc = tf_rm_lookup_tbl_type_pool(tfs,
1195                                         parms->dir,
1196                                         parms->type,
1197                                         &session_pool);
1198         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1199         if (rc)
1200                 return rc;
1201
1202         id = ba_alloc(session_pool);
1203         if (id == -1) {
1204                 free_cnt = ba_free_count(session_pool);
1205
1206                 PMD_DRV_LOG(ERR,
1207                    "dir:%d, Allocation failed, type:%d, free:%d\n",
1208                    parms->dir,
1209                    parms->type,
1210                    free_cnt);
1211                 return -ENOMEM;
1212         }
1213
1214         /* Adjust the returned index/offset as there is no guarantee
1215          * that the start is 0 at time of RM allocation
1216          */
1217         tf_rm_convert_index(tfs,
1218                             parms->dir,
1219                             parms->type,
1220                             TF_RM_CONVERT_ADD_BASE,
1221                             id,
1222                             &index);
1223         parms->idx = index;
1224         return rc;
1225 }
1226
1227 /**
1228  * Free External Tbl entry to the session pool.
1229  *
1230  * [in] tfp
1231  *   Pointer to Truflow Handle
1232  * [in] parms
1233  *   Allocation parameters
1234  *
1235  * Return:
1236  *  0       - Success, entry freed
1237  *
1238  * - Failure, entry not successfully freed for these reasons
1239  *  -ENOMEM
1240  *  -EOPNOTSUPP
1241  *  -EINVAL
1242  */
1243 static int
1244 tf_free_tbl_entry_pool_external(struct tf *tfp,
1245                                 struct tf_free_tbl_entry_parms *parms)
1246 {
1247         int rc = 0;
1248         struct tf_session *tfs;
1249         uint32_t index;
1250         struct tf_tbl_scope_cb *tbl_scope_cb;
1251         struct stack *pool;
1252
1253         /* Check parameters */
1254         if (tfp == NULL || parms == NULL) {
1255                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1256                 return -EINVAL;
1257         }
1258
1259         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1260                 PMD_DRV_LOG(ERR,
1261                             "dir:%d, Session info invalid\n",
1262                             parms->dir);
1263                 return -EINVAL;
1264         }
1265
1266         tfs = (struct tf_session *)(tfp->session->core_data);
1267
1268         /* Get the pool info from the table scope
1269          */
1270         tbl_scope_cb = tbl_scope_cb_find(tfs, parms->tbl_scope_id);
1271
1272         if (tbl_scope_cb == NULL) {
1273                 PMD_DRV_LOG(ERR,
1274                             "dir:%d, Session info invalid\n",
1275                             parms->dir);
1276                 return -EINVAL;
1277         }
1278         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
1279
1280         index = parms->idx;
1281
1282         rc = stack_push(pool, index);
1283
1284         if (rc != 0) {
1285                 PMD_DRV_LOG(ERR,
1286                    "dir:%d, consistency error, stack full, type:%d, idx:%d\n",
1287                    parms->dir,
1288                    parms->type,
1289                    index);
1290         }
1291         return rc;
1292 }
1293
1294 /**
1295  * Free Internal Tbl entry from the Session Pool.
1296  *
1297  * [in] tfp
1298  *   Pointer to Truflow Handle
1299  * [in] parms
1300  *   Allocation parameters
1301  *
1302  * Return:
1303  *  0       - Success, entry found and ref count decremented
1304  *  -ENOMEM - Failure, entry not allocated, out of resources
1305  */
1306 static int
1307 tf_free_tbl_entry_pool_internal(struct tf *tfp,
1308                        struct tf_free_tbl_entry_parms *parms)
1309 {
1310         int rc = 0;
1311         int id;
1312         struct bitalloc *session_pool;
1313         struct tf_session *tfs;
1314         uint32_t index;
1315
1316         /* Check parameters */
1317         if (tfp == NULL || parms == NULL) {
1318                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1319                 return -EINVAL;
1320         }
1321
1322         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1323                 PMD_DRV_LOG(ERR,
1324                             "dir:%d, Session info invalid\n",
1325                             parms->dir);
1326                 return -EINVAL;
1327         }
1328
1329         tfs = (struct tf_session *)(tfp->session->core_data);
1330
1331         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1332             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1333             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1334             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1335             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1336             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B &&
1337             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
1338                 PMD_DRV_LOG(ERR,
1339                             "dir:%d, Type not supported, type:%d\n",
1340                             parms->dir,
1341                             parms->type);
1342                 return -EOPNOTSUPP;
1343         }
1344
1345         /* Lookup the pool using the table type of the element */
1346         rc = tf_rm_lookup_tbl_type_pool(tfs,
1347                                         parms->dir,
1348                                         parms->type,
1349                                         &session_pool);
1350         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1351         if (rc)
1352                 return rc;
1353
1354         index = parms->idx;
1355
1356         /* Adjust the returned index/offset as there is no guarantee
1357          * that the start is 0 at time of RM allocation
1358          */
1359         tf_rm_convert_index(tfs,
1360                             parms->dir,
1361                             parms->type,
1362                             TF_RM_CONVERT_RM_BASE,
1363                             parms->idx,
1364                             &index);
1365
1366         /* Check if element was indeed allocated */
1367         id = ba_inuse_free(session_pool, index);
1368         if (id == -1) {
1369                 PMD_DRV_LOG(ERR,
1370                    "dir:%d, Element not previously alloc'ed, type:%d, idx:%d\n",
1371                    parms->dir,
1372                    parms->type,
1373                    index);
1374                 return -ENOMEM;
1375         }
1376
1377         return rc;
1378 }
1379
1380 /* API defined in tf_em.h */
1381 struct tf_tbl_scope_cb *
1382 tbl_scope_cb_find(struct tf_session *session,
1383                   uint32_t tbl_scope_id)
1384 {
1385         int i;
1386
1387         /* Check that id is valid */
1388         i = ba_inuse(session->tbl_scope_pool_rx, tbl_scope_id);
1389         if (i < 0)
1390                 return NULL;
1391
1392         for (i = 0; i < TF_NUM_TBL_SCOPE; i++) {
1393                 if (session->tbl_scopes[i].tbl_scope_id == tbl_scope_id)
1394                         return &session->tbl_scopes[i];
1395         }
1396
1397         return NULL;
1398 }
1399
1400 /* API defined in tf_core.h */
1401 int
1402 tf_free_eem_tbl_scope_cb(struct tf *tfp,
1403                          struct tf_free_tbl_scope_parms *parms)
1404 {
1405         int rc = 0;
1406         enum tf_dir  dir;
1407         struct tf_tbl_scope_cb *tbl_scope_cb;
1408         struct tf_session *session;
1409
1410         session = (struct tf_session *)(tfp->session->core_data);
1411
1412         tbl_scope_cb = tbl_scope_cb_find(session,
1413                                          parms->tbl_scope_id);
1414
1415         if (tbl_scope_cb == NULL)
1416                 return -EINVAL;
1417
1418         /* Free Table control block */
1419         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1420
1421         /* free table scope locks */
1422         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1423                 /* Free associated external pools
1424                  */
1425                 tf_destroy_tbl_pool_external(dir,
1426                                              tbl_scope_cb);
1427                 tf_msg_em_op(tfp,
1428                              dir,
1429                              HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
1430
1431                 /* free table scope and all associated resources */
1432                 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
1433         }
1434
1435         return rc;
1436 }
1437
1438 /* API defined in tf_em.h */
1439 int
1440 tf_alloc_eem_tbl_scope(struct tf *tfp,
1441                        struct tf_alloc_tbl_scope_parms *parms)
1442 {
1443         int rc;
1444         enum tf_dir dir;
1445         struct tf_tbl_scope_cb *tbl_scope_cb;
1446         struct tf_em_table *em_tables;
1447         int index;
1448         struct tf_session *session;
1449         struct tf_free_tbl_scope_parms free_parms;
1450
1451         /* check parameters */
1452         if (parms == NULL || tfp->session == NULL) {
1453                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1454                 return -EINVAL;
1455         }
1456
1457         session = (struct tf_session *)tfp->session->core_data;
1458
1459         /* Get Table Scope control block from the session pool */
1460         index = ba_alloc(session->tbl_scope_pool_rx);
1461         if (index == -1) {
1462                 PMD_DRV_LOG(ERR, "EEM: Unable to allocate table scope "
1463                             "Control Block\n");
1464                 return -ENOMEM;
1465         }
1466
1467         tbl_scope_cb = &session->tbl_scopes[index];
1468         tbl_scope_cb->index = index;
1469         tbl_scope_cb->tbl_scope_id = index;
1470         parms->tbl_scope_id = index;
1471
1472         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1473                 rc = tf_msg_em_qcaps(tfp,
1474                                      dir,
1475                                      &tbl_scope_cb->em_caps[dir]);
1476                 if (rc) {
1477                         PMD_DRV_LOG(ERR,
1478                                 "EEM: Unable to query for EEM capability\n");
1479                         goto cleanup;
1480                 }
1481         }
1482
1483         /*
1484          * Validate and setup table sizes
1485          */
1486         if (tf_em_validate_num_entries(tbl_scope_cb, parms))
1487                 goto cleanup;
1488
1489         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1490                 /*
1491                  * Allocate tables and signal configuration to FW
1492                  */
1493                 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
1494                 if (rc) {
1495                         PMD_DRV_LOG(ERR,
1496                                     "EEM: Unable to register for EEM ctx\n");
1497                         goto cleanup;
1498                 }
1499
1500                 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
1501                 rc = tf_msg_em_cfg(tfp,
1502                                    em_tables[KEY0_TABLE].num_entries,
1503                                    em_tables[KEY0_TABLE].ctx_id,
1504                                    em_tables[KEY1_TABLE].ctx_id,
1505                                    em_tables[RECORD_TABLE].ctx_id,
1506                                    em_tables[EFC_TABLE].ctx_id,
1507                                    parms->hw_flow_cache_flush_timer,
1508                                    dir);
1509                 if (rc) {
1510                         PMD_DRV_LOG(ERR,
1511                                 "TBL: Unable to configure EEM in firmware\n");
1512                         goto cleanup_full;
1513                 }
1514
1515                 rc = tf_msg_em_op(tfp,
1516                                   dir,
1517                                   HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
1518
1519                 if (rc) {
1520                         PMD_DRV_LOG(ERR,
1521                                     "EEM: Unable to enable EEM in firmware\n");
1522                         goto cleanup_full;
1523                 }
1524
1525                 /* Allocate the pool of offsets of the external memory.
1526                  * Initially, this is a single fixed size pool for all external
1527                  * actions related to a single table scope.
1528                  */
1529                 rc = tf_create_tbl_pool_external(dir,
1530                                             tbl_scope_cb,
1531                                             em_tables[RECORD_TABLE].num_entries,
1532                                             em_tables[RECORD_TABLE].entry_size);
1533                 if (rc) {
1534                         PMD_DRV_LOG(ERR,
1535                                     "%d TBL: Unable to allocate idx pools %s\n",
1536                                     dir,
1537                                     strerror(-rc));
1538                         goto cleanup_full;
1539                 }
1540         }
1541
1542         return 0;
1543
1544 cleanup_full:
1545         free_parms.tbl_scope_id = index;
1546         tf_free_eem_tbl_scope_cb(tfp, &free_parms);
1547         return -EINVAL;
1548
1549 cleanup:
1550         /* Free Table control block */
1551         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1552         return -EINVAL;
1553 }
1554
1555 /* API defined in tf_core.h */
1556 int
1557 tf_set_tbl_entry(struct tf *tfp,
1558                  struct tf_set_tbl_entry_parms *parms)
1559 {
1560         int rc = 0;
1561         struct tf_tbl_scope_cb *tbl_scope_cb;
1562         struct tf_session *session;
1563
1564         if (tfp == NULL || parms == NULL || parms->data == NULL)
1565                 return -EINVAL;
1566
1567         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1568                 PMD_DRV_LOG(ERR,
1569                             "dir:%d, Session info invalid\n",
1570                             parms->dir);
1571                 return -EINVAL;
1572         }
1573
1574         if (parms->type == TF_TBL_TYPE_EXT) {
1575                 void *base_addr;
1576                 uint32_t offset = parms->idx;
1577                 uint32_t tbl_scope_id;
1578
1579                 session = (struct tf_session *)(tfp->session->core_data);
1580
1581                 tbl_scope_id = parms->tbl_scope_id;
1582
1583                 if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1584                         PMD_DRV_LOG(ERR,
1585                                     "dir:%d, Table scope not allocated\n",
1586                                     parms->dir);
1587                         return -EINVAL;
1588                 }
1589
1590                 /* Get the table scope control block associated with the
1591                  * external pool
1592                  */
1593                 tbl_scope_cb = tbl_scope_cb_find(session, tbl_scope_id);
1594
1595                 if (tbl_scope_cb == NULL)
1596                         return -EINVAL;
1597
1598                 /* External table, implicitly the Action table */
1599                 base_addr = tf_em_get_table_page(tbl_scope_cb,
1600                                                  parms->dir,
1601                                                  offset,
1602                                                  RECORD_TABLE);
1603                 if (base_addr == NULL) {
1604                         PMD_DRV_LOG(ERR,
1605                                     "dir:%d, Base address lookup failed\n",
1606                                     parms->dir);
1607                         return -EINVAL;
1608                 }
1609
1610                 offset %= TF_EM_PAGE_SIZE;
1611                 rte_memcpy((char *)base_addr + offset,
1612                            parms->data,
1613                            parms->data_sz_in_bytes);
1614         } else {
1615                 /* Internal table type processing */
1616                 rc = tf_set_tbl_entry_internal(tfp, parms);
1617                 if (rc) {
1618                         PMD_DRV_LOG(ERR,
1619                                     "dir:%d, Set failed, type:%d, rc:%d\n",
1620                                     parms->dir,
1621                                     parms->type,
1622                                     rc);
1623                 }
1624         }
1625
1626         return rc;
1627 }
1628
1629 /* API defined in tf_core.h */
1630 int
1631 tf_get_tbl_entry(struct tf *tfp,
1632                  struct tf_get_tbl_entry_parms *parms)
1633 {
1634         int rc = 0;
1635
1636         if (tfp == NULL || parms == NULL)
1637                 return -EINVAL;
1638
1639         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1640                 PMD_DRV_LOG(ERR,
1641                             "dir:%d, Session info invalid\n",
1642                             parms->dir);
1643                 return -EINVAL;
1644         }
1645
1646         if (parms->type == TF_TBL_TYPE_EXT) {
1647                 PMD_DRV_LOG(ERR,
1648                             "dir:%d, External table type not supported\n",
1649                             parms->dir);
1650
1651                 rc = -EOPNOTSUPP;
1652         } else {
1653                 /* Internal table type processing */
1654                 rc = tf_get_tbl_entry_internal(tfp, parms);
1655                 if (rc)
1656                         PMD_DRV_LOG(ERR,
1657                                     "dir:%d, Get failed, type:%d, rc:%d\n",
1658                                     parms->dir,
1659                                     parms->type,
1660                                     rc);
1661         }
1662
1663         return rc;
1664 }
1665
1666 /* API defined in tf_core.h */
1667 int
1668 tf_alloc_tbl_scope(struct tf *tfp,
1669                    struct tf_alloc_tbl_scope_parms *parms)
1670 {
1671         int rc;
1672
1673         /* check parameters */
1674         if (parms == NULL || tfp == NULL) {
1675                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1676                 return -EINVAL;
1677         }
1678
1679         rc = tf_alloc_eem_tbl_scope(tfp, parms);
1680
1681         return rc;
1682 }
1683
1684 /* API defined in tf_core.h */
1685 int
1686 tf_free_tbl_scope(struct tf *tfp,
1687                   struct tf_free_tbl_scope_parms *parms)
1688 {
1689         int rc;
1690
1691         /* check parameters */
1692         if (parms == NULL || tfp == NULL) {
1693                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1694                 return -EINVAL;
1695         }
1696
1697         /* free table scope and all associated resources */
1698         rc = tf_free_eem_tbl_scope_cb(tfp, parms);
1699
1700         return rc;
1701 }
1702
1703 /* API defined in tf_core.h */
1704 int
1705 tf_alloc_tbl_entry(struct tf *tfp,
1706                    struct tf_alloc_tbl_entry_parms *parms)
1707 {
1708         int rc;
1709 #if (TF_SHADOW == 1)
1710         struct tf_session *tfs;
1711 #endif /* TF_SHADOW */
1712
1713         /* Check parameters */
1714         if (parms == NULL || tfp == NULL) {
1715                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1716                 return -EINVAL;
1717         }
1718         /*
1719          * No shadow copy support for external tables, allocate and return
1720          */
1721         if (parms->type == TF_TBL_TYPE_EXT) {
1722                 rc = tf_alloc_tbl_entry_pool_external(tfp, parms);
1723                 return rc;
1724         }
1725
1726 #if (TF_SHADOW == 1)
1727         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1728                 PMD_DRV_LOG(ERR,
1729                             "dir:%d, Session info invalid\n",
1730                             parms->dir);
1731                 return -EINVAL;
1732         }
1733
1734         tfs = (struct tf_session *)(tfp->session->core_data);
1735
1736         /* Search the Shadow DB for requested element. If not found go
1737          * allocate one from the Session Pool
1738          */
1739         if (parms->search_enable && tfs->shadow_copy) {
1740                 rc = tf_alloc_tbl_entry_shadow(tfs, parms);
1741                 /* Entry found and parms populated with return data */
1742                 if (rc == 0)
1743                         return rc;
1744         }
1745 #endif /* TF_SHADOW */
1746
1747         rc = tf_alloc_tbl_entry_pool_internal(tfp, parms);
1748         if (rc)
1749                 PMD_DRV_LOG(ERR, "dir%d, Alloc failed, rc:%d\n",
1750                             parms->dir,
1751                             rc);
1752
1753         return rc;
1754 }
1755
1756 /* API defined in tf_core.h */
1757 int
1758 tf_free_tbl_entry(struct tf *tfp,
1759                   struct tf_free_tbl_entry_parms *parms)
1760 {
1761         int rc;
1762 #if (TF_SHADOW == 1)
1763         struct tf_session *tfs;
1764 #endif /* TF_SHADOW */
1765
1766         /* Check parameters */
1767         if (parms == NULL || tfp == NULL) {
1768                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1769                 return -EINVAL;
1770         }
1771         /*
1772          * No shadow of external tables so just free the entry
1773          */
1774         if (parms->type == TF_TBL_TYPE_EXT) {
1775                 rc = tf_free_tbl_entry_pool_external(tfp, parms);
1776                 return rc;
1777         }
1778
1779 #if (TF_SHADOW == 1)
1780         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1781                 PMD_DRV_LOG(ERR,
1782                             "dir:%d, Session info invalid\n",
1783                             parms->dir);
1784                 return -EINVAL;
1785         }
1786
1787         tfs = (struct tf_session *)(tfp->session->core_data);
1788
1789         /* Search the Shadow DB for requested element. If not found go
1790          * allocate one from the Session Pool
1791          */
1792         if (parms->search_enable && tfs->shadow_copy) {
1793                 rc = tf_free_tbl_entry_shadow(tfs, parms);
1794                 /* Entry free'ed and parms populated with return data */
1795                 if (rc == 0)
1796                         return rc;
1797         }
1798 #endif /* TF_SHADOW */
1799
1800         rc = tf_free_tbl_entry_pool_internal(tfp, parms);
1801
1802         if (rc)
1803                 PMD_DRV_LOG(ERR, "dir:%d, Alloc failed, rc:%d\n",
1804                             parms->dir,
1805                             rc);
1806         return rc;
1807 }
1808
1809
1810 static void
1811 tf_dump_link_page_table(struct tf_em_page_tbl *tp,
1812                         struct tf_em_page_tbl *tp_next)
1813 {
1814         uint64_t *pg_va;
1815         uint32_t i;
1816         uint32_t j;
1817         uint32_t k = 0;
1818
1819         printf("pg_count:%d pg_size:0x%x\n",
1820                tp->pg_count,
1821                tp->pg_size);
1822         for (i = 0; i < tp->pg_count; i++) {
1823                 pg_va = tp->pg_va_tbl[i];
1824                 printf("\t%p\n", (void *)pg_va);
1825                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
1826                         printf("\t\t%p\n", (void *)(uintptr_t)pg_va[j]);
1827                         if (((pg_va[j] & 0x7) ==
1828                              tfp_cpu_to_le_64(PTU_PTE_LAST |
1829                                               PTU_PTE_VALID)))
1830                                 return;
1831
1832                         if (!(pg_va[j] & tfp_cpu_to_le_64(PTU_PTE_VALID))) {
1833                                 printf("** Invalid entry **\n");
1834                                 return;
1835                         }
1836
1837                         if (++k >= tp_next->pg_count) {
1838                                 printf("** Shouldn't get here **\n");
1839                                 return;
1840                         }
1841                 }
1842         }
1843 }
1844
1845 void tf_dump_dma(struct tf *tfp, uint32_t tbl_scope_id);
1846
1847 void tf_dump_dma(struct tf *tfp, uint32_t tbl_scope_id)
1848 {
1849         struct tf_session      *session;
1850         struct tf_tbl_scope_cb *tbl_scope_cb;
1851         struct tf_em_page_tbl *tp;
1852         struct tf_em_page_tbl *tp_next;
1853         struct tf_em_table *tbl;
1854         int i;
1855         int j;
1856         int dir;
1857
1858         printf("called %s\n", __func__);
1859
1860         /* find session struct */
1861         session = (struct tf_session *)tfp->session->core_data;
1862
1863         /* find control block for table scope */
1864         tbl_scope_cb = tbl_scope_cb_find(session,
1865                                          tbl_scope_id);
1866         if (tbl_scope_cb == NULL)
1867                 PMD_DRV_LOG(ERR, "No table scope\n");
1868
1869         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1870                 printf("Direction %s:\n", (dir == TF_DIR_RX ? "Rx" : "Tx"));
1871
1872                 for (j = KEY0_TABLE; j < MAX_TABLE; j++) {
1873                         tbl = &tbl_scope_cb->em_ctx_info[dir].em_tables[j];
1874                         printf
1875         ("Table: j:%d type:%d num_entries:%d entry_size:0x%x num_lvl:%d ",
1876                                j,
1877                                tbl->type,
1878                                tbl->num_entries,
1879                                tbl->entry_size,
1880                                tbl->num_lvl);
1881                         if (tbl->pg_tbl[0].pg_va_tbl &&
1882                             tbl->pg_tbl[0].pg_pa_tbl)
1883                                 printf("%p %p\n",
1884                                tbl->pg_tbl[0].pg_va_tbl[0],
1885                                (void *)(uintptr_t)tbl->pg_tbl[0].pg_pa_tbl[0]);
1886                         for (i = 0; i < tbl->num_lvl - 1; i++) {
1887                                 printf("Level:%d\n", i);
1888                                 tp = &tbl->pg_tbl[i];
1889                                 tp_next = &tbl->pg_tbl[i + 1];
1890                                 tf_dump_link_page_table(tp, tp_next);
1891                         }
1892                         printf("\n");
1893                 }
1894         }
1895 }