drivers/net/bnxt/tf_core/tf_tbl.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 /* Truflow Table APIs and supporting code */
   7
   8 #include <stdio.h>
   9 #include <string.h>
  10 #include <stdbool.h>
  11 #include <math.h>
  12 #include <sys/param.h>
  13 #include <rte_common.h>
  14 #include <rte_errno.h>
  15 #include "hsi_struct_def_dpdk.h"
  16
  17 #include "tf_core.h"
  18 #include "tf_em.h"
  19 #include "tf_msg.h"
  20 #include "tfp.h"
  21 #include "hwrm_tf.h"
  22 #include "bnxt.h"
  23 #include "tf_resources.h"
  24 #include "tf_rm.h"
  25
  26 #define PTU_PTE_VALID          0x1UL
  27 #define PTU_PTE_LAST           0x2UL
  28 #define PTU_PTE_NEXT_TO_LAST   0x4UL
  29
  30 /* Number of pointers per page_size */
  31 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  32
  33 #define TF_EM_PG_SZ_4K        (1 << 12)
  34 #define TF_EM_PG_SZ_8K        (1 << 13)
  35 #define TF_EM_PG_SZ_64K       (1 << 16)
  36 #define TF_EM_PG_SZ_256K      (1 << 18)
  37 #define TF_EM_PG_SZ_1M        (1 << 20)
  38 #define TF_EM_PG_SZ_2M        (1 << 21)
  39 #define TF_EM_PG_SZ_4M        (1 << 22)
  40 #define TF_EM_PG_SZ_1G        (1 << 30)
  41
  42 #define TF_EM_CTX_ID_INVALID   0xFFFF
  43
  44 #define TF_EM_MIN_ENTRIES     (1 << 15) /* 32K */
  45 #define TF_EM_MAX_ENTRIES     (1 << 27) /* 128M */
  46
  47 /**
  48  * Function to free a page table
  49  *
  50  * [in] tp
  51  *   Pointer to the page table to free
  52  */
  53 static void
  54 tf_em_free_pg_tbl(struct tf_em_page_tbl *tp)
  55 {
  56         uint32_t i;
  57
  58         for (i = 0; i < tp->pg_count; i++) {
  59                 if (!tp->pg_va_tbl[i]) {
  60                         PMD_DRV_LOG(WARNING,
  61                                     "No map for page %d table %016" PRIu64 "\n",
  62                                     i,
  63                                     (uint64_t)(uintptr_t)tp);
  64                         continue;
  65                 }
  66
  67                 tfp_free(tp->pg_va_tbl[i]);
  68                 tp->pg_va_tbl[i] = NULL;
  69         }
  70
  71         tp->pg_count = 0;
  72         tfp_free(tp->pg_va_tbl);
  73         tp->pg_va_tbl = NULL;
  74         tfp_free(tp->pg_pa_tbl);
  75         tp->pg_pa_tbl = NULL;
  76 }
  77
  78 /**
  79  * Function to free an EM table
  80  *
  81  * [in] tbl
  82  *   Pointer to the EM table to free
  83  */
  84 static void
  85 tf_em_free_page_table(struct tf_em_table *tbl)
  86 {
  87         struct tf_em_page_tbl *tp;
  88         int i;
  89
  90         for (i = 0; i < tbl->num_lvl; i++) {
  91                 tp = &tbl->pg_tbl[i];
  92
  93                 PMD_DRV_LOG(INFO,
  94                            "EEM: Freeing page table: size %u lvl %d cnt %u\n",
  95                            TF_EM_PAGE_SIZE,
  96                             i,
  97                             tp->pg_count);
  98
  99                 tf_em_free_pg_tbl(tp);
 100         }
 101
 102         tbl->l0_addr = NULL;
 103         tbl->l0_dma_addr = 0;
 104         tbl->num_lvl = 0;
 105         tbl->num_data_pages = 0;
 106 }
 107
 108 /**
 109  * Allocation of page tables
 110  *
 111  * [in] tfp
 112  *   Pointer to a TruFlow handle
 113  *
 114  * [in] pg_count
 115  *   Page count to allocate
 116  *
 117  * [in] pg_size
 118  *   Size of each page
 119  *
 120  * Returns:
 121  *   0       - Success
 122  *   -ENOMEM - Out of memory
 123  */
 124 static int
 125 tf_em_alloc_pg_tbl(struct tf_em_page_tbl *tp,
 126                    uint32_t pg_count,
 127                    uint32_t pg_size)
 128 {
 129         uint32_t i;
 130         struct tfp_calloc_parms parms;
 131
 132         parms.nitems = pg_count;
 133         parms.size = sizeof(void *);
 134         parms.alignment = 0;
 135
 136         if (tfp_calloc(&parms) != 0)
 137                 return -ENOMEM;
 138
 139         tp->pg_va_tbl = parms.mem_va;
 140
 141         if (tfp_calloc(&parms) != 0) {
 142                 tfp_free(tp->pg_va_tbl);
 143                 return -ENOMEM;
 144         }
 145
 146         tp->pg_pa_tbl = parms.mem_va;
 147
 148         tp->pg_count = 0;
 149         tp->pg_size = pg_size;
 150
 151         for (i = 0; i < pg_count; i++) {
 152                 parms.nitems = 1;
 153                 parms.size = pg_size;
 154                 parms.alignment = TF_EM_PAGE_ALIGNMENT;
 155
 156                 if (tfp_calloc(&parms) != 0)
 157                         goto cleanup;
 158
 159                 tp->pg_pa_tbl[i] = (uint64_t)(uintptr_t)parms.mem_pa;
 160                 tp->pg_va_tbl[i] = parms.mem_va;
 161
 162                 memset(tp->pg_va_tbl[i], 0, pg_size);
 163                 tp->pg_count++;
 164         }
 165
 166         return 0;
 167
 168 cleanup:
 169         tf_em_free_pg_tbl(tp);
 170         return -ENOMEM;
 171 }
 172
 173 /**
 174  * Allocates EM page tables
 175  *
 176  * [in] tbl
 177  *   Table to allocate pages for
 178  *
 179  * Returns:
 180  *   0       - Success
 181  *   -ENOMEM - Out of memory
 182  */
 183 static int
 184 tf_em_alloc_page_table(struct tf_em_table *tbl)
 185 {
 186         struct tf_em_page_tbl *tp;
 187         int rc = 0;
 188         int i;
 189         uint32_t j;
 190
 191         for (i = 0; i < tbl->num_lvl; i++) {
 192                 tp = &tbl->pg_tbl[i];
 193
 194                 rc = tf_em_alloc_pg_tbl(tp,
 195                                         tbl->page_cnt[i],
 196                                         TF_EM_PAGE_SIZE);
 197                 if (rc) {
 198                         PMD_DRV_LOG(WARNING,
 199                                 "Failed to allocate page table: lvl: %d\n",
 200                                 i);
 201                         goto cleanup;
 202                 }
 203
 204                 for (j = 0; j < tp->pg_count; j++) {
 205                         PMD_DRV_LOG(INFO,
 206                                 "EEM: Allocated page table: size %u lvl %d cnt"
 207                                 " %u VA:%p PA:%p\n",
 208                                 TF_EM_PAGE_SIZE,
 209                                 i,
 210                                 tp->pg_count,
 211                                 (uint32_t *)tp->pg_va_tbl[j],
 212                                 (uint32_t *)(uintptr_t)tp->pg_pa_tbl[j]);
 213                 }
 214         }
 215         return rc;
 216
 217 cleanup:
 218         tf_em_free_page_table(tbl);
 219         return rc;
 220 }
 221
 222 /**
 223  * Links EM page tables
 224  *
 225  * [in] tp
 226  *   Pointer to page table
 227  *
 228  * [in] tp_next
 229  *   Pointer to the next page table
 230  *
 231  * [in] set_pte_last
 232  *   Flag controlling if the page table is last
 233  */
 234 static void
 235 tf_em_link_page_table(struct tf_em_page_tbl *tp,
 236                       struct tf_em_page_tbl *tp_next,
 237                       bool set_pte_last)
 238 {
 239         uint64_t *pg_pa = tp_next->pg_pa_tbl;
 240         uint64_t *pg_va;
 241         uint64_t valid;
 242         uint32_t k = 0;
 243         uint32_t i;
 244         uint32_t j;
 245
 246         for (i = 0; i < tp->pg_count; i++) {
 247                 pg_va = tp->pg_va_tbl[i];
 248
 249                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
 250                         if (k == tp_next->pg_count - 2 && set_pte_last)
 251                                 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
 252                         else if (k == tp_next->pg_count - 1 && set_pte_last)
 253                                 valid = PTU_PTE_LAST | PTU_PTE_VALID;
 254                         else
 255                                 valid = PTU_PTE_VALID;
 256
 257                         pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
 258                         if (++k >= tp_next->pg_count)
 259                                 return;
 260                 }
 261         }
 262 }
 263
 264 /**
 265  * Setup a EM page table
 266  *
 267  * [in] tbl
 268  *   Pointer to EM page table
 269  */
 270 static void
 271 tf_em_setup_page_table(struct tf_em_table *tbl)
 272 {
 273         struct tf_em_page_tbl *tp_next;
 274         struct tf_em_page_tbl *tp;
 275         bool set_pte_last = 0;
 276         int i;
 277
 278         for (i = 0; i < tbl->num_lvl - 1; i++) {
 279                 tp = &tbl->pg_tbl[i];
 280                 tp_next = &tbl->pg_tbl[i + 1];
 281                 if (i == tbl->num_lvl - 2)
 282                         set_pte_last = 1;
 283                 tf_em_link_page_table(tp, tp_next, set_pte_last);
 284         }
 285
 286         tbl->l0_addr = tbl->pg_tbl[PT_LVL_0].pg_va_tbl[0];
 287         tbl->l0_dma_addr = tbl->pg_tbl[PT_LVL_0].pg_pa_tbl[0];
 288 }
 289
 290 /**
 291  * Given the page size, size of each data item (entry size),
 292  * and the total number of entries needed, determine the number
 293  * of page table levels and the number of data pages required.
 294  *
 295  * [in] page_size
 296  *   Page size
 297  *
 298  * [in] entry_size
 299  *   Entry size
 300  *
 301  * [in] num_entries
 302  *   Number of entries needed
 303  *
 304  * [out] num_data_pages
 305  *   Number of pages required
 306  *
 307  * Returns:
 308  *   Success  - Number of EM page levels required
 309  *   -ENOMEM  - Out of memory
 310  */
 311 static int
 312 tf_em_size_page_tbl_lvl(uint32_t page_size,
 313                         uint32_t entry_size,
 314                         uint32_t num_entries,
 315                         uint64_t *num_data_pages)
 316 {
 317         uint64_t lvl_data_size = page_size;
 318         int lvl = PT_LVL_0;
 319         uint64_t data_size;
 320
 321         *num_data_pages = 0;
 322         data_size = (uint64_t)num_entries * entry_size;
 323
 324         while (lvl_data_size < data_size) {
 325                 lvl++;
 326
 327                 if (lvl == PT_LVL_1)
 328                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 329                                 page_size;
 330                 else if (lvl == PT_LVL_2)
 331                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 332                                 MAX_PAGE_PTRS(page_size) * page_size;
 333                 else
 334                         return -ENOMEM;
 335         }
 336
 337         *num_data_pages = roundup(data_size, page_size) / page_size;
 338
 339         return lvl;
 340 }
 341
 342 /**
 343  * Return the number of page table pages needed to
 344  * reference the given number of next level pages.
 345  *
 346  * [in] num_pages
 347  *   Number of EM pages
 348  *
 349  * [in] page_size
 350  *   Size of each EM page
 351  *
 352  * Returns:
 353  *   Number of EM page table pages
 354  */
 355 static uint32_t
 356 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 357                      uint32_t page_size)
 358 {
 359         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 360                        MAX_PAGE_PTRS(page_size);
 361         return 0;
 362 }
 363
 364 /**
 365  * Given the number of data pages, page_size and the maximum
 366  * number of page table levels (already determined), size
 367  * the number of page table pages required at each level.
 368  *
 369  * [in] max_lvl
 370  *   Max number of levels
 371  *
 372  * [in] num_data_pages
 373  *   Number of EM data pages
 374  *
 375  * [in] page_size
 376  *   Size of an EM page
 377  *
 378  * [out] *page_cnt
 379  *   EM page count
 380  */
 381 static void
 382 tf_em_size_page_tbls(int max_lvl,
 383                      uint64_t num_data_pages,
 384                      uint32_t page_size,
 385                      uint32_t *page_cnt)
 386 {
 387         if (max_lvl == PT_LVL_0) {
 388                 page_cnt[PT_LVL_0] = num_data_pages;
 389         } else if (max_lvl == PT_LVL_1) {
 390                 page_cnt[PT_LVL_1] = num_data_pages;
 391                 page_cnt[PT_LVL_0] =
 392                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_1], page_size);
 393         } else if (max_lvl == PT_LVL_2) {
 394                 page_cnt[PT_LVL_2] = num_data_pages;
 395                 page_cnt[PT_LVL_1] =
 396                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_2], page_size);
 397                 page_cnt[PT_LVL_0] =
 398                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_1], page_size);
 399         } else {
 400                 return;
 401         }
 402 }
 403
 404 /**
 405  * Size the EM table based on capabilities
 406  *
 407  * [in] tbl
 408  *   EM table to size
 409  *
 410  * Returns:
 411  *   0        - Success
 412  *   - EINVAL - Parameter error
 413  *   - ENOMEM - Out of memory
 414  */
 415 static int
 416 tf_em_size_table(struct tf_em_table *tbl)
 417 {
 418         uint64_t num_data_pages;
 419         uint32_t *page_cnt;
 420         int max_lvl;
 421         uint32_t num_entries;
 422         uint32_t cnt = TF_EM_MIN_ENTRIES;
 423
 424         /* Ignore entry if both size and number are zero */
 425         if (!tbl->entry_size && !tbl->num_entries)
 426                 return 0;
 427
 428         /* If only one is set then error */
 429         if (!tbl->entry_size || !tbl->num_entries)
 430                 return -EINVAL;
 431
 432         /* Determine number of page table levels and the number
 433          * of data pages needed to process the given eem table.
 434          */
 435         if (tbl->type == RECORD_TABLE) {
 436                 /*
 437                  * For action records just a memory size is provided. Work
 438                  * backwards to resolve to number of entries
 439                  */
 440                 num_entries = tbl->num_entries / tbl->entry_size;
 441                 if (num_entries < TF_EM_MIN_ENTRIES) {
 442                         num_entries = TF_EM_MIN_ENTRIES;
 443                 } else {
 444                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 445                                 cnt *= 2;
 446                         num_entries = cnt;
 447                 }
 448         } else {
 449                 num_entries = tbl->num_entries;
 450         }
 451
 452         max_lvl = tf_em_size_page_tbl_lvl(TF_EM_PAGE_SIZE,
 453                                           tbl->entry_size,
 454                                           tbl->num_entries,
 455                                           &num_data_pages);
 456         if (max_lvl < 0) {
 457                 PMD_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 458                 PMD_DRV_LOG(WARNING,
 459                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 460                             tbl->type,
 461                             (uint64_t)num_entries * tbl->entry_size,
 462                             TF_EM_PAGE_SIZE);
 463                 return -ENOMEM;
 464         }
 465
 466         tbl->num_lvl = max_lvl + 1;
 467         tbl->num_data_pages = num_data_pages;
 468
 469         /* Determine the number of pages needed at each level */
 470         page_cnt = tbl->page_cnt;
 471         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 472         tf_em_size_page_tbls(max_lvl, num_data_pages, TF_EM_PAGE_SIZE,
 473                                 page_cnt);
 474
 475         PMD_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 476         PMD_DRV_LOG(INFO,
 477                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 " l0: %u l1: %u l2: %u\n",
 478                     max_lvl + 1,
 479                     (uint64_t)num_data_pages * TF_EM_PAGE_SIZE,
 480                     num_data_pages,
 481                     page_cnt[PT_LVL_0],
 482                     page_cnt[PT_LVL_1],
 483                     page_cnt[PT_LVL_2]);
 484
 485         return 0;
 486 }
 487
 488 /**
 489  * Unregisters EM Ctx in Firmware
 490  *
 491  * [in] tfp
 492  *   Pointer to a TruFlow handle
 493  *
 494  * [in] tbl_scope_cb
 495  *   Pointer to a table scope control block
 496  *
 497  * [in] dir
 498  *   Receive or transmit direction
 499  */
 500 static void
 501 tf_em_ctx_unreg(struct tf *tfp,
 502                 struct tf_tbl_scope_cb *tbl_scope_cb,
 503                 int dir)
 504 {
 505         struct tf_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 506         struct tf_em_table *tbl;
 507         int i;
 508
 509         for (i = KEY0_TABLE; i < MAX_TABLE; i++) {
 510                 tbl = &ctxp->em_tables[i];
 511
 512                 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
 513                         tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
 514                         tf_em_free_page_table(tbl);
 515                 }
 516         }
 517 }
 518
 519 /**
 520  * Registers EM Ctx in Firmware
 521  *
 522  * [in] tfp
 523  *   Pointer to a TruFlow handle
 524  *
 525  * [in] tbl_scope_cb
 526  *   Pointer to a table scope control block
 527  *
 528  * [in] dir
 529  *   Receive or transmit direction
 530  *
 531  * Returns:
 532  *   0       - Success
 533  *   -ENOMEM - Out of Memory
 534  */
 535 static int
 536 tf_em_ctx_reg(struct tf *tfp,
 537               struct tf_tbl_scope_cb *tbl_scope_cb,
 538               int dir)
 539 {
 540         struct tf_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 541         struct tf_em_table *tbl;
 542         int rc;
 543         int i;
 544
 545         for (i = KEY0_TABLE; i < MAX_TABLE; i++) {
 546                 tbl = &ctxp->em_tables[i];
 547
 548                 if (tbl->num_entries && tbl->entry_size) {
 549                         rc = tf_em_size_table(tbl);
 550
 551                         if (rc)
 552                                 goto cleanup;
 553
 554                         rc = tf_em_alloc_page_table(tbl);
 555                         if (rc)
 556                                 goto cleanup;
 557
 558                         tf_em_setup_page_table(tbl);
 559                         rc = tf_msg_em_mem_rgtr(tfp,
 560                                                 tbl->num_lvl - 1,
 561                                                 TF_EM_PAGE_SIZE_ENUM,
 562                                                 tbl->l0_dma_addr,
 563                                                 &tbl->ctx_id);
 564                         if (rc)
 565                                 goto cleanup;
 566                 }
 567         }
 568         return rc;
 569
 570 cleanup:
 571         tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
 572         return rc;
 573 }
 574
 575 /**
 576  * Validates EM number of entries requested
 577  *
 578  * [in] tbl_scope_cb
 579  *   Pointer to table scope control block to be populated
 580  *
 581  * [in] parms
 582  *   Pointer to input parameters
 583  *
 584  * Returns:
 585  *   0       - Success
 586  *   -EINVAL - Parameter error
 587  */
 588 static int
 589 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 590                            struct tf_alloc_tbl_scope_parms *parms)
 591 {
 592         uint32_t cnt;
 593
 594         if (parms->rx_mem_size_in_mb != 0) {
 595                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 596                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 597                                      + 1);
 598                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 599                                         TF_MEGABYTE) / (key_b + action_b);
 600
 601                 if (num_entries < TF_EM_MIN_ENTRIES) {
 602                         PMD_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 603                                     "%uMB\n",
 604                                     parms->rx_mem_size_in_mb);
 605                         return -EINVAL;
 606                 }
 607
 608                 cnt = TF_EM_MIN_ENTRIES;
 609                 while (num_entries > cnt &&
 610                        cnt <= TF_EM_MAX_ENTRIES)
 611                         cnt *= 2;
 612
 613                 if (cnt > TF_EM_MAX_ENTRIES) {
 614                         PMD_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 615                                     "%u\n",
 616                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 617                         return -EINVAL;
 618                 }
 619
 620                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 621         } else {
 622                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 623                     TF_EM_MIN_ENTRIES ||
 624                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 625                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 626                         PMD_DRV_LOG(ERR,
 627                                     "EEM: Invalid number of Rx flows "
 628                                     "requested:%u max:%u\n",
 629                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 630                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 631                         return -EINVAL;
 632                 }
 633
 634                 /* must be a power-of-2 supported value
 635                  * in the range 32K - 128M
 636                  */
 637                 cnt = TF_EM_MIN_ENTRIES;
 638                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 639                        cnt <= TF_EM_MAX_ENTRIES)
 640                         cnt *= 2;
 641
 642                 if (cnt > TF_EM_MAX_ENTRIES) {
 643                         PMD_DRV_LOG(ERR,
 644                                     "EEM: Invalid number of Rx requested: %u\n",
 645                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 646                         return -EINVAL;
 647                 }
 648         }
 649
 650         if (parms->tx_mem_size_in_mb != 0) {
 651                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 652                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 653                                      + 1);
 654                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 655                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 656                         (key_b + action_b);
 657
 658                 if (num_entries < TF_EM_MIN_ENTRIES) {
 659                         PMD_DRV_LOG(ERR,
 660                                     "EEM: Insufficient memory requested:%uMB\n",
 661                                     parms->rx_mem_size_in_mb);
 662                         return -EINVAL;
 663                 }
 664
 665                 cnt = TF_EM_MIN_ENTRIES;
 666                 while (num_entries > cnt &&
 667                        cnt <= TF_EM_MAX_ENTRIES)
 668                         cnt *= 2;
 669
 670                 if (cnt > TF_EM_MAX_ENTRIES) {
 671                         PMD_DRV_LOG(ERR,
 672                                     "EEM: Invalid number of Tx requested: %u\n",
 673                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 674                         return -EINVAL;
 675                 }
 676
 677                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 678         } else {
 679                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 680                     TF_EM_MIN_ENTRIES ||
 681                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 682                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 683                         PMD_DRV_LOG(ERR,
 684                                     "EEM: Invalid number of Tx flows "
 685                                     "requested:%u max:%u\n",
 686                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 687                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 688                         return -EINVAL;
 689                 }
 690
 691                 cnt = TF_EM_MIN_ENTRIES;
 692                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 693                        cnt <= TF_EM_MAX_ENTRIES)
 694                         cnt *= 2;
 695
 696                 if (cnt > TF_EM_MAX_ENTRIES) {
 697                         PMD_DRV_LOG(ERR,
 698                                     "EEM: Invalid number of Tx requested: %u\n",
 699                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 700                         return -EINVAL;
 701                 }
 702         }
 703
 704         /* Rx */
 705         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY0_TABLE].num_entries =
 706                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 707         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY0_TABLE].entry_size =
 708                 parms->rx_max_key_sz_in_bits / 8;
 709
 710         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY1_TABLE].num_entries =
 711                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 712         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY1_TABLE].entry_size =
 713                 parms->rx_max_key_sz_in_bits / 8;
 714
 715         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[RECORD_TABLE].num_entries =
 716                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 717         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[RECORD_TABLE].entry_size =
 718                 parms->tx_max_action_entry_sz_in_bits / 8;
 719
 720         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[EFC_TABLE].num_entries =
 721                 0;
 722
 723         /* Tx */
 724         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY0_TABLE].num_entries =
 725                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 726         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY0_TABLE].entry_size =
 727                 parms->tx_max_key_sz_in_bits / 8;
 728
 729         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY1_TABLE].num_entries =
 730                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 731         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY1_TABLE].entry_size =
 732                 parms->tx_max_key_sz_in_bits / 8;
 733
 734         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[RECORD_TABLE].num_entries =
 735                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 736         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[RECORD_TABLE].entry_size =
 737                 parms->tx_max_action_entry_sz_in_bits / 8;
 738
 739         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[EFC_TABLE].num_entries =
 740                 0;
 741
 742         return 0;
 743 }
 744
 745 /**
 746  * Internal function to set a Table Entry. Supports all internal Table Types
 747  *
 748  * [in] tfp
 749  *   Pointer to TruFlow handle
 750  *
 751  * [in] parms
 752  *   Pointer to input parameters
 753  *
 754  * Returns:
 755  *   0       - Success
 756  *   -EINVAL - Parameter error
 757  */
 758 static int
 759 tf_set_tbl_entry_internal(struct tf *tfp,
 760                           struct tf_set_tbl_entry_parms *parms)
 761 {
 762         int rc;
 763         int id;
 764         uint32_t index;
 765         struct bitalloc *session_pool;
 766         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 767
 768         /* Lookup the pool using the table type of the element */
 769         rc = tf_rm_lookup_tbl_type_pool(tfs,
 770                                         parms->dir,
 771                                         parms->type,
 772                                         &session_pool);
 773         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 774         if (rc)
 775                 return rc;
 776
 777         index = parms->idx;
 778
 779         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
 780             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4) {
 781                 PMD_DRV_LOG(ERR,
 782                             "dir:%d, Type not supported, type:%d\n",
 783                             parms->dir,
 784                             parms->type);
 785                 return -EOPNOTSUPP;
 786         }
 787
 788         /* Adjust the returned index/offset as there is no guarantee
 789          * that the start is 0 at time of RM allocation
 790          */
 791         tf_rm_convert_index(tfs,
 792                             parms->dir,
 793                             parms->type,
 794                             TF_RM_CONVERT_RM_BASE,
 795                             parms->idx,
 796                             &index);
 797
 798         /* Verify that the entry has been previously allocated */
 799         id = ba_inuse(session_pool, index);
 800         if (id != 1) {
 801                 PMD_DRV_LOG(ERR,
 802                    "dir:%d, Invalid or not allocated index, type:%d, idx:%d\n",
 803                    parms->dir,
 804                    parms->type,
 805                    index);
 806                 return -EINVAL;
 807         }
 808
 809         /* Set the entry */
 810         rc = tf_msg_set_tbl_entry(tfp,
 811                                   parms->dir,
 812                                   parms->type,
 813                                   parms->data_sz_in_bytes,
 814                                   parms->data,
 815                                   parms->idx);
 816         if (rc) {
 817                 PMD_DRV_LOG(ERR,
 818                             "dir:%d, Set failed, type:%d, rc:%d\n",
 819                             parms->dir,
 820                             parms->type,
 821                             rc);
 822         }
 823
 824         return rc;
 825 }
 826
 827 /**
 828  * Internal function to get a Table Entry. Supports all Table Types
 829  * except the TF_TBL_TYPE_EXT as that is handled as a table scope.
 830  *
 831  * [in] tfp
 832  *   Pointer to TruFlow handle
 833  *
 834  * [in] parms
 835  *   Pointer to input parameters
 836  *
 837  * Returns:
 838  *   0       - Success
 839  *   -EINVAL - Parameter error
 840  */
 841 static int
 842 tf_get_tbl_entry_internal(struct tf *tfp,
 843                           struct tf_get_tbl_entry_parms *parms)
 844 {
 845         int rc;
 846         int id;
 847         uint32_t index;
 848         struct bitalloc *session_pool;
 849         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 850
 851         /* Lookup the pool using the table type of the element */
 852         rc = tf_rm_lookup_tbl_type_pool(tfs,
 853                                         parms->dir,
 854                                         parms->type,
 855                                         &session_pool);
 856         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 857         if (rc)
 858                 return rc;
 859
 860         index = parms->idx;
 861
 862         /* Adjust the returned index/offset as there is no guarantee
 863          * that the start is 0 at time of RM allocation
 864          */
 865         tf_rm_convert_index(tfs,
 866                             parms->dir,
 867                             parms->type,
 868                             TF_RM_CONVERT_RM_BASE,
 869                             parms->idx,
 870                             &index);
 871
 872         /* Verify that the entry has been previously allocated */
 873         id = ba_inuse(session_pool, index);
 874         if (id != 1) {
 875                 PMD_DRV_LOG(ERR,
 876                    "dir:%d, Invalid or not allocated index, type:%d, idx:%d\n",
 877                    parms->dir,
 878                    parms->type,
 879                    index);
 880                 return -EINVAL;
 881         }
 882
 883         /* Get the entry */
 884         rc = tf_msg_get_tbl_entry(tfp,
 885                                   parms->dir,
 886                                   parms->type,
 887                                   parms->data_sz_in_bytes,
 888                                   parms->data,
 889                                   parms->idx);
 890         if (rc) {
 891                 PMD_DRV_LOG(ERR,
 892                             "dir:%d, Get failed, type:%d, rc:%d\n",
 893                             parms->dir,
 894                             parms->type,
 895                             rc);
 896         }
 897
 898         return rc;
 899 }
 900
 901 #if (TF_SHADOW == 1)
 902 /**
 903  * Allocate Tbl entry from the Shadow DB. Shadow DB is searched for
 904  * the requested entry. If found the ref count is incremente and
 905  * returned.
 906  *
 907  * [in] tfs
 908  *   Pointer to session
 909  * [in] parms
 910  *   Allocation parameters
 911  *
 912  * Return:
 913  *  0       - Success, entry found and ref count incremented
 914  *  -ENOENT - Failure, entry not found
 915  */
 916 static int
 917 tf_alloc_tbl_entry_shadow(struct tf_session *tfs __rte_unused,
 918                           struct tf_alloc_tbl_entry_parms *parms __rte_unused)
 919 {
 920         PMD_DRV_LOG(ERR,
 921                     "dir:%d, Entry Alloc with search not supported\n",
 922                     parms->dir);
 923
 924
 925         return -EOPNOTSUPP;
 926 }
 927
 928 /**
 929  * Free Tbl entry from the Shadow DB. Shadow DB is searched for
 930  * the requested entry. If found the ref count is decremente and
 931  * new ref_count returned.
 932  *
 933  * [in] tfs
 934  *   Pointer to session
 935  * [in] parms
 936  *   Allocation parameters
 937  *
 938  * Return:
 939  *  0       - Success, entry found and ref count decremented
 940  *  -ENOENT - Failure, entry not found
 941  */
 942 static int
 943 tf_free_tbl_entry_shadow(struct tf_session *tfs,
 944                          struct tf_free_tbl_entry_parms *parms)
 945 {
 946         PMD_DRV_LOG(ERR,
 947                     "dir:%d, Entry Free with search not supported\n",
 948                     parms->dir);
 949
 950         return -EOPNOTSUPP;
 951 }
 952 #endif /* TF_SHADOW */
 953
 954 /**
 955  * Create External Tbl pool of memory indexes.
 956  *
 957  * [in] session
 958  *   Pointer to session
 959  * [in] dir
 960  *   direction
 961  * [in] tbl_scope_cb
 962  *   pointer to the table scope
 963  * [in] tbl_scope_id
 964  *   id of the table scope
 965  * [in] num_entries
 966  *   number of entries to write
 967  * [in] entry_sz_bytes
 968  *   size of each entry
 969  *
 970  * Return:
 971  *  0       - Success, entry allocated - no search support
 972  *  -ENOMEM -EINVAL -EOPNOTSUPP
 973  *          - Failure, entry not allocated, out of resources
 974  */
 975 static int
 976 tf_create_tbl_pool_external(struct tf_session *session,
 977                             enum tf_dir dir,
 978                             struct tf_tbl_scope_cb *tbl_scope_cb,
 979                             uint32_t table_scope_id,
 980                             uint32_t num_entries,
 981                             uint32_t entry_sz_bytes)
 982
 983 {
 984         struct tfp_calloc_parms parms;
 985         uint32_t i, j;
 986         int rc = 0;
 987         struct stack *pool = &tbl_scope_cb->ext_pool[dir][TF_EXT_POOL_0];
 988
 989         parms.nitems = num_entries;
 990         parms.size = sizeof(uint32_t);
 991         parms.alignment = 0;
 992
 993         if (tfp_calloc(&parms) != 0) {
 994                 PMD_DRV_LOG(ERR, "%d: TBL: external pool failure %s\n",
 995                             dir, strerror(-ENOMEM));
 996                 return -ENOMEM;
 997         }
 998
 999         /* Create empty stack
1000          */
1001         rc = stack_init(num_entries, parms.mem_va, pool);
1002
1003         if (rc != 0) {
1004                 PMD_DRV_LOG(ERR, "%d: TBL: stack init failure %s\n",
1005                             dir, strerror(-rc));
1006                 goto cleanup;
1007         }
1008
1009         /* Save the  malloced memory address so that it can
1010          * be freed when the table scope is freed.
1011          */
1012         tbl_scope_cb->ext_pool_mem[dir][TF_EXT_POOL_0] =
1013                 (uint32_t *)parms.mem_va;
1014
1015         /* Fill pool with indexes
1016          */
1017         j = num_entries * entry_sz_bytes - 1;
1018
1019         for (i = 0; i < num_entries; i++) {
1020                 rc = stack_push(pool, j);
1021                 if (rc != 0) {
1022                         PMD_DRV_LOG(ERR, "%d TBL: stack failure %s\n",
1023                                     dir, strerror(-rc));
1024                         goto cleanup;
1025                 }
1026                 j -= entry_sz_bytes;
1027         }
1028
1029         if (!stack_is_full(pool)) {
1030                 rc = -EINVAL;
1031                 PMD_DRV_LOG(ERR, "%d TBL: stack failure %s\n",
1032                             dir, strerror(-rc));
1033                 goto cleanup;
1034         }
1035         /* Set the table scope associated with the pool
1036          */
1037         session->ext_pool_2_scope[dir][TF_EXT_POOL_0] = table_scope_id;
1038
1039         return 0;
1040 cleanup:
1041         tfp_free((void *)parms.mem_va);
1042         return rc;
1043 }
1044
1045 /**
1046  * Destroy External Tbl pool of memory indexes.
1047  *
1048  * [in] session
1049  *   Pointer to session
1050  * [in] dir
1051  *   direction
1052  * [in] tbl_scope_cb
1053  *   pointer to the table scope
1054  *
1055  */
1056 static void
1057 tf_destroy_tbl_pool_external(struct tf_session *session,
1058                             enum tf_dir dir,
1059                             struct tf_tbl_scope_cb *tbl_scope_cb)
1060 {
1061         uint32_t *ext_pool_mem =
1062                 tbl_scope_cb->ext_pool_mem[dir][TF_EXT_POOL_0];
1063
1064         tfp_free(ext_pool_mem);
1065
1066         /* Set the table scope associated with the pool
1067          */
1068         session->ext_pool_2_scope[dir][TF_EXT_POOL_0] = TF_TBL_SCOPE_INVALID;
1069 }
1070
1071 /**
1072  * Allocate External Tbl entry from the Session Pool.
1073  *
1074  * [in] tfp
1075  *   Pointer to Truflow Handle
1076  * [in] parms
1077  *   Allocation parameters
1078  *
1079  * Return:
1080  *  0       - Success, entry allocated - no search support
1081  *  -ENOMEM -EINVAL -EOPNOTSUPP
1082  *          - Failure, entry not allocated, out of resources
1083  */
1084 static int
1085 tf_alloc_tbl_entry_pool_external(struct tf *tfp,
1086                                  struct tf_alloc_tbl_entry_parms *parms)
1087 {
1088         int rc;
1089         uint32_t index;
1090         struct tf_session *tfs;
1091         uint32_t tbl_scope_id;
1092         struct tf_tbl_scope_cb *tbl_scope_cb;
1093         struct stack *pool;
1094
1095         /* Check parameters */
1096         if (tfp == NULL || parms == NULL) {
1097                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1098                 return -EINVAL;
1099         }
1100
1101         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1102                 PMD_DRV_LOG(ERR,
1103                             "dir:%d, Session info invalid\n",
1104                             parms->dir);
1105                 return -EINVAL;
1106         }
1107
1108         tfs = (struct tf_session *)(tfp->session->core_data);
1109
1110         if (parms->type != TF_TBL_TYPE_EXT) {
1111                 PMD_DRV_LOG(ERR,
1112                             "dir:%d, Type not supported, type:%d\n",
1113                             parms->dir,
1114                             parms->type);
1115                 return -EOPNOTSUPP;
1116         }
1117
1118         /* Get the pool info from the table scope
1119          */
1120         tbl_scope_id = tfs->ext_pool_2_scope[parms->dir][TF_EXT_POOL_0];
1121         tbl_scope_cb = tbl_scope_cb_find(tfs, tbl_scope_id);
1122
1123         if (tbl_scope_cb == NULL) {
1124                 PMD_DRV_LOG(ERR,
1125                             "dir:%d, table scope not allocated\n",
1126                             parms->dir);
1127                 return -EINVAL;
1128         }
1129         pool = &tbl_scope_cb->ext_pool[parms->dir][TF_EXT_POOL_0];
1130
1131         /* Allocate an element
1132          */
1133         rc = stack_pop(pool, &index);
1134
1135         if (rc != 0) {
1136                 PMD_DRV_LOG(ERR,
1137                    "dir:%d, Allocation failed, type:%d\n",
1138                    parms->dir,
1139                    parms->type);
1140                 return rc;
1141         }
1142         parms->idx = index;
1143         return rc;
1144 }
1145
1146 /**
1147  * Allocate Internal Tbl entry from the Session Pool.
1148  *
1149  * [in] tfp
1150  *   Pointer to Truflow Handle
1151  * [in] parms
1152  *   Allocation parameters
1153  *
1154  * Return:
1155  *  0       - Success, entry found and ref count decremented
1156  *  -ENOMEM - Failure, entry not allocated, out of resources
1157  */
1158 static int
1159 tf_alloc_tbl_entry_pool_internal(struct tf *tfp,
1160                                  struct tf_alloc_tbl_entry_parms *parms)
1161 {
1162         int rc;
1163         int id;
1164         int free_cnt;
1165         uint32_t index;
1166         struct bitalloc *session_pool;
1167         struct tf_session *tfs;
1168
1169         /* Check parameters */
1170         if (tfp == NULL || parms == NULL) {
1171                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1172                 return -EINVAL;
1173         }
1174
1175         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1176                 PMD_DRV_LOG(ERR,
1177                             "dir:%d, Session info invalid\n",
1178                             parms->dir);
1179                 return -EINVAL;
1180         }
1181
1182         tfs = (struct tf_session *)(tfp->session->core_data);
1183
1184         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1185             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1186             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1187             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1188             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1189             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B) {
1190                 PMD_DRV_LOG(ERR,
1191                             "dir:%d, Type not supported, type:%d\n",
1192                             parms->dir,
1193                             parms->type);
1194                 return -EOPNOTSUPP;
1195         }
1196
1197         /* Lookup the pool using the table type of the element */
1198         rc = tf_rm_lookup_tbl_type_pool(tfs,
1199                                         parms->dir,
1200                                         parms->type,
1201                                         &session_pool);
1202         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1203         if (rc)
1204                 return rc;
1205
1206         id = ba_alloc(session_pool);
1207         if (id == -1) {
1208                 free_cnt = ba_free_count(session_pool);
1209
1210                 PMD_DRV_LOG(ERR,
1211                    "dir:%d, Allocation failed, type:%d, free:%d\n",
1212                    parms->dir,
1213                    parms->type,
1214                    free_cnt);
1215                 return -ENOMEM;
1216         }
1217
1218         /* Adjust the returned index/offset as there is no guarantee
1219          * that the start is 0 at time of RM allocation
1220          */
1221         tf_rm_convert_index(tfs,
1222                             parms->dir,
1223                             parms->type,
1224                             TF_RM_CONVERT_ADD_BASE,
1225                             id,
1226                             &index);
1227         parms->idx = index;
1228         return rc;
1229 }
1230
1231 /**
1232  * Free External Tbl entry to the session pool.
1233  *
1234  * [in] tfp
1235  *   Pointer to Truflow Handle
1236  * [in] parms
1237  *   Allocation parameters
1238  *
1239  * Return:
1240  *  0       - Success, entry freed
1241  *
1242  * - Failure, entry not successfully freed for these reasons
1243  *  -ENOMEM
1244  *  -EOPNOTSUPP
1245  *  -EINVAL
1246  */
1247 static int
1248 tf_free_tbl_entry_pool_external(struct tf *tfp,
1249                        struct tf_free_tbl_entry_parms *parms)
1250 {
1251         int rc = 0;
1252         struct tf_session *tfs;
1253         uint32_t index;
1254         uint32_t tbl_scope_id;
1255         struct tf_tbl_scope_cb *tbl_scope_cb;
1256         struct stack *pool;
1257
1258         /* Check parameters */
1259         if (tfp == NULL || parms == NULL) {
1260                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1261                 return -EINVAL;
1262         }
1263
1264         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1265                 PMD_DRV_LOG(ERR,
1266                             "dir:%d, Session info invalid\n",
1267                             parms->dir);
1268                 return -EINVAL;
1269         }
1270
1271         tfs = (struct tf_session *)(tfp->session->core_data);
1272
1273         if (parms->type != TF_TBL_TYPE_EXT) {
1274                 PMD_DRV_LOG(ERR,
1275                             "dir:%d, Type not supported, type:%d\n",
1276                             parms->dir,
1277                             parms->type);
1278                 return -EOPNOTSUPP;
1279         }
1280
1281         /* Get the pool info from the table scope
1282          */
1283         tbl_scope_id = tfs->ext_pool_2_scope[parms->dir][TF_EXT_POOL_0];
1284         tbl_scope_cb = tbl_scope_cb_find(tfs, tbl_scope_id);
1285
1286         if (tbl_scope_cb == NULL) {
1287                 PMD_DRV_LOG(ERR,
1288                             "dir:%d, table scope error\n",
1289                             parms->dir);
1290                 return -EINVAL;
1291         }
1292         pool = &tbl_scope_cb->ext_pool[parms->dir][TF_EXT_POOL_0];
1293
1294         index = parms->idx;
1295
1296         rc = stack_push(pool, index);
1297
1298         if (rc != 0) {
1299                 PMD_DRV_LOG(ERR,
1300                    "dir:%d, consistency error, stack full, type:%d, idx:%d\n",
1301                    parms->dir,
1302                    parms->type,
1303                    index);
1304         }
1305         return rc;
1306 }
1307
1308 /**
1309  * Free Internal Tbl entry from the Session Pool.
1310  *
1311  * [in] tfp
1312  *   Pointer to Truflow Handle
1313  * [in] parms
1314  *   Allocation parameters
1315  *
1316  * Return:
1317  *  0       - Success, entry found and ref count decremented
1318  *  -ENOMEM - Failure, entry not allocated, out of resources
1319  */
1320 static int
1321 tf_free_tbl_entry_pool_internal(struct tf *tfp,
1322                        struct tf_free_tbl_entry_parms *parms)
1323 {
1324         int rc = 0;
1325         int id;
1326         struct bitalloc *session_pool;
1327         struct tf_session *tfs;
1328         uint32_t index;
1329
1330         /* Check parameters */
1331         if (tfp == NULL || parms == NULL) {
1332                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1333                 return -EINVAL;
1334         }
1335
1336         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1337                 PMD_DRV_LOG(ERR,
1338                             "dir:%d, Session info invalid\n",
1339                             parms->dir);
1340                 return -EINVAL;
1341         }
1342
1343         tfs = (struct tf_session *)(tfp->session->core_data);
1344
1345         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1346             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1347             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1348             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1349             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1350             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B) {
1351                 PMD_DRV_LOG(ERR,
1352                             "dir:%d, Type not supported, type:%d\n",
1353                             parms->dir,
1354                             parms->type);
1355                 return -EOPNOTSUPP;
1356         }
1357
1358         /* Lookup the pool using the table type of the element */
1359         rc = tf_rm_lookup_tbl_type_pool(tfs,
1360                                         parms->dir,
1361                                         parms->type,
1362                                         &session_pool);
1363         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1364         if (rc)
1365                 return rc;
1366
1367         index = parms->idx;
1368
1369         /* Adjust the returned index/offset as there is no guarantee
1370          * that the start is 0 at time of RM allocation
1371          */
1372         tf_rm_convert_index(tfs,
1373                             parms->dir,
1374                             parms->type,
1375                             TF_RM_CONVERT_RM_BASE,
1376                             parms->idx,
1377                             &index);
1378
1379         /* Check if element was indeed allocated */
1380         id = ba_inuse_free(session_pool, index);
1381         if (id == -1) {
1382                 PMD_DRV_LOG(ERR,
1383                    "dir:%d, Element not previously alloc'ed, type:%d, idx:%d\n",
1384                    parms->dir,
1385                    parms->type,
1386                    index);
1387                 return -ENOMEM;
1388         }
1389
1390         return rc;
1391 }
1392
1393 /* API defined in tf_tbl.h */
1394 void
1395 tf_init_tbl_pool(struct tf_session *session)
1396 {
1397         enum tf_dir dir;
1398
1399         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1400                 session->ext_pool_2_scope[dir][TF_EXT_POOL_0] =
1401                         TF_TBL_SCOPE_INVALID;
1402         }
1403 }
1404
1405 /* API defined in tf_em.h */
1406 struct tf_tbl_scope_cb *
1407 tbl_scope_cb_find(struct tf_session *session,
1408                   uint32_t tbl_scope_id)
1409 {
1410         int i;
1411
1412         /* Check that id is valid */
1413         i = ba_inuse(session->tbl_scope_pool_rx, tbl_scope_id);
1414         if (i < 0)
1415                 return NULL;
1416
1417         for (i = 0; i < TF_NUM_TBL_SCOPE; i++) {
1418                 if (session->tbl_scopes[i].tbl_scope_id == tbl_scope_id)
1419                         return &session->tbl_scopes[i];
1420         }
1421
1422         return NULL;
1423 }
1424
1425 /* API defined in tf_core.h */
1426 int
1427 tf_free_eem_tbl_scope_cb(struct tf *tfp,
1428                          struct tf_free_tbl_scope_parms *parms)
1429 {
1430         int rc = 0;
1431         enum tf_dir  dir;
1432         struct tf_tbl_scope_cb *tbl_scope_cb;
1433         struct tf_session *session;
1434
1435         session = (struct tf_session *)(tfp->session->core_data);
1436
1437         tbl_scope_cb = tbl_scope_cb_find(session,
1438                                          parms->tbl_scope_id);
1439
1440         if (tbl_scope_cb == NULL)
1441                 return -EINVAL;
1442
1443         /* Free Table control block */
1444         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1445
1446         /* free table scope locks */
1447         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1448                 /* Free associated external pools
1449                  */
1450                 tf_destroy_tbl_pool_external(session,
1451                                              dir,
1452                                              tbl_scope_cb);
1453                 tf_msg_em_op(tfp,
1454                              dir,
1455                              HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
1456
1457                 /* free table scope and all associated resources */
1458                 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
1459         }
1460
1461         return rc;
1462 }
1463
1464 /* API defined in tf_em.h */
1465 int
1466 tf_alloc_eem_tbl_scope(struct tf *tfp,
1467                        struct tf_alloc_tbl_scope_parms *parms)
1468 {
1469         int rc;
1470         enum tf_dir dir;
1471         struct tf_tbl_scope_cb *tbl_scope_cb;
1472         struct tf_em_table *em_tables;
1473         int index;
1474         struct tf_session *session;
1475         struct tf_free_tbl_scope_parms free_parms;
1476
1477         /* check parameters */
1478         if (parms == NULL || tfp->session == NULL) {
1479                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1480                 return -EINVAL;
1481         }
1482
1483         session = (struct tf_session *)tfp->session->core_data;
1484
1485         /* Get Table Scope control block from the session pool */
1486         index = ba_alloc(session->tbl_scope_pool_rx);
1487         if (index == -1) {
1488                 PMD_DRV_LOG(ERR, "EEM: Unable to allocate table scope "
1489                             "Control Block\n");
1490                 return -ENOMEM;
1491         }
1492
1493         tbl_scope_cb = &session->tbl_scopes[index];
1494         tbl_scope_cb->index = index;
1495         tbl_scope_cb->tbl_scope_id = index;
1496         parms->tbl_scope_id = index;
1497
1498         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1499                 rc = tf_msg_em_qcaps(tfp,
1500                                      dir,
1501                                      &tbl_scope_cb->em_caps[dir]);
1502                 if (rc) {
1503                         PMD_DRV_LOG(ERR,
1504                                 "EEM: Unable to query for EEM capability\n");
1505                         goto cleanup;
1506                 }
1507         }
1508
1509         /*
1510          * Validate and setup table sizes
1511          */
1512         if (tf_em_validate_num_entries(tbl_scope_cb, parms))
1513                 goto cleanup;
1514
1515         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1516                 /*
1517                  * Allocate tables and signal configuration to FW
1518                  */
1519                 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
1520                 if (rc) {
1521                         PMD_DRV_LOG(ERR,
1522                                     "EEM: Unable to register for EEM ctx\n");
1523                         goto cleanup;
1524                 }
1525
1526                 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
1527                 rc = tf_msg_em_cfg(tfp,
1528                                    em_tables[KEY0_TABLE].num_entries,
1529                                    em_tables[KEY0_TABLE].ctx_id,
1530                                    em_tables[KEY1_TABLE].ctx_id,
1531                                    em_tables[RECORD_TABLE].ctx_id,
1532                                    em_tables[EFC_TABLE].ctx_id,
1533                                    dir);
1534                 if (rc) {
1535                         PMD_DRV_LOG(ERR,
1536                                 "TBL: Unable to configure EEM in firmware\n");
1537                         goto cleanup_full;
1538                 }
1539
1540                 rc = tf_msg_em_op(tfp,
1541                                   dir,
1542                                   HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
1543
1544                 if (rc) {
1545                         PMD_DRV_LOG(ERR,
1546                                     "EEM: Unable to enable EEM in firmware\n");
1547                         goto cleanup_full;
1548                 }
1549
1550                 /* Allocate the pool of offsets of the external memory.
1551                  * Initially, this is a single fixed size pool for all external
1552                  * actions related to a single table scope.
1553                  */
1554                 rc = tf_create_tbl_pool_external(session,
1555                                                  dir,
1556                                                  tbl_scope_cb,
1557                                                  index,
1558                                                  TF_EXT_POOL_ENTRY_CNT,
1559                                                  TF_EXT_POOL_ENTRY_SZ_BYTES);
1560                 if (rc) {
1561                         PMD_DRV_LOG(ERR,
1562                                     "%d TBL: Unable to allocate idx pools %s\n",
1563                                     dir,
1564                                     strerror(-rc));
1565                         goto cleanup_full;
1566                 }
1567         }
1568
1569         return 0;
1570
1571 cleanup_full:
1572         free_parms.tbl_scope_id = index;
1573         tf_free_eem_tbl_scope_cb(tfp, &free_parms);
1574         return -EINVAL;
1575
1576 cleanup:
1577         /* Free Table control block */
1578         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1579         return -EINVAL;
1580 }
1581
1582 /* API defined in tf_core.h */
1583 int
1584 tf_set_tbl_entry(struct tf *tfp,
1585                  struct tf_set_tbl_entry_parms *parms)
1586 {
1587         int rc = 0;
1588         struct tf_tbl_scope_cb *tbl_scope_cb;
1589         struct tf_session *session;
1590
1591         if (tfp == NULL || parms == NULL || parms->data == NULL)
1592                 return -EINVAL;
1593
1594         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1595                 PMD_DRV_LOG(ERR,
1596                             "dir:%d, Session info invalid\n",
1597                             parms->dir);
1598                 return -EINVAL;
1599         }
1600
1601         if (parms->type == TF_TBL_TYPE_EXT) {
1602                 void *base_addr;
1603                 uint32_t offset = TF_ACT_REC_INDEX_2_OFFSET(parms->idx);
1604                 uint32_t tbl_scope_id;
1605
1606                 session = (struct tf_session *)(tfp->session->core_data);
1607
1608                 tbl_scope_id =
1609                         session->ext_pool_2_scope[parms->dir][TF_EXT_POOL_0];
1610
1611                 if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1612                         PMD_DRV_LOG(ERR,
1613                                     "dir:%d, Table scope not allocated\n",
1614                                     parms->dir);
1615                         return -EINVAL;
1616                 }
1617
1618                 /* Get the table scope control block associated with the
1619                  * external pool
1620                  */
1621
1622                 tbl_scope_cb = tbl_scope_cb_find(session, tbl_scope_id);
1623
1624                 if (tbl_scope_cb == NULL)
1625                         return -EINVAL;
1626
1627                 /* External table, implicitly the Action table */
1628                 base_addr = tf_em_get_table_page(tbl_scope_cb,
1629                                                  parms->dir,
1630                                                  offset,
1631                                                  RECORD_TABLE);
1632                 if (base_addr == NULL) {
1633                         PMD_DRV_LOG(ERR,
1634                                     "dir:%d, Base address lookup failed\n",
1635                                     parms->dir);
1636                         return -EINVAL;
1637                 }
1638
1639                 offset %= TF_EM_PAGE_SIZE;
1640                 rte_memcpy((char *)base_addr + offset,
1641                            parms->data,
1642                            parms->data_sz_in_bytes);
1643         } else {
1644                 /* Internal table type processing */
1645                 rc = tf_set_tbl_entry_internal(tfp, parms);
1646                 if (rc) {
1647                         PMD_DRV_LOG(ERR,
1648                                     "dir:%d, Set failed, type:%d, rc:%d\n",
1649                                     parms->dir,
1650                                     parms->type,
1651                                     rc);
1652                 }
1653         }
1654
1655         return rc;
1656 }
1657
1658 /* API defined in tf_core.h */
1659 int
1660 tf_get_tbl_entry(struct tf *tfp,
1661                  struct tf_get_tbl_entry_parms *parms)
1662 {
1663         int rc = 0;
1664
1665         if (tfp == NULL || parms == NULL)
1666                 return -EINVAL;
1667
1668         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1669                 PMD_DRV_LOG(ERR,
1670                             "dir:%d, Session info invalid\n",
1671                             parms->dir);
1672                 return -EINVAL;
1673         }
1674
1675         if (parms->type == TF_TBL_TYPE_EXT) {
1676                 PMD_DRV_LOG(ERR,
1677                             "dir:%d, External table type not supported\n",
1678                             parms->dir);
1679
1680                 rc = -EOPNOTSUPP;
1681         } else {
1682                 /* Internal table type processing */
1683                 rc = tf_get_tbl_entry_internal(tfp, parms);
1684                 if (rc)
1685                         PMD_DRV_LOG(ERR,
1686                                     "dir:%d, Get failed, type:%d, rc:%d\n",
1687                                     parms->dir,
1688                                     parms->type,
1689                                     rc);
1690         }
1691
1692         return rc;
1693 }
1694
1695 /* API defined in tf_core.h */
1696 int
1697 tf_alloc_tbl_scope(struct tf *tfp,
1698                    struct tf_alloc_tbl_scope_parms *parms)
1699 {
1700         int rc;
1701
1702         /* check parameters */
1703         if (parms == NULL || tfp == NULL) {
1704                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1705                 return -EINVAL;
1706         }
1707
1708         rc = tf_alloc_eem_tbl_scope(tfp, parms);
1709
1710         return rc;
1711 }
1712
1713 /* API defined in tf_core.h */
1714 int
1715 tf_free_tbl_scope(struct tf *tfp,
1716                   struct tf_free_tbl_scope_parms *parms)
1717 {
1718         int rc;
1719
1720         /* check parameters */
1721         if (parms == NULL || tfp == NULL) {
1722                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1723                 return -EINVAL;
1724         }
1725
1726         /* free table scope and all associated resources */
1727         rc = tf_free_eem_tbl_scope_cb(tfp, parms);
1728
1729         return rc;
1730 }
1731
1732 /* API defined in tf_core.h */
1733 int
1734 tf_alloc_tbl_entry(struct tf *tfp,
1735                    struct tf_alloc_tbl_entry_parms *parms)
1736 {
1737         int rc;
1738 #if (TF_SHADOW == 1)
1739         struct tf_session *tfs;
1740 #endif /* TF_SHADOW */
1741
1742         /* Check parameters */
1743         if (parms == NULL || tfp == NULL) {
1744                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1745                 return -EINVAL;
1746         }
1747         /*
1748          * No shadow copy support for external tables, allocate and return
1749          */
1750         if (parms->type == TF_TBL_TYPE_EXT) {
1751                 rc = tf_alloc_tbl_entry_pool_external(tfp, parms);
1752                 return rc;
1753         }
1754
1755 #if (TF_SHADOW == 1)
1756         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1757                 PMD_DRV_LOG(ERR,
1758                             "dir:%d, Session info invalid\n",
1759                             parms->dir);
1760                 return -EINVAL;
1761         }
1762
1763         tfs = (struct tf_session *)(tfp->session->core_data);
1764
1765         /* Search the Shadow DB for requested element. If not found go
1766          * allocate one from the Session Pool
1767          */
1768         if (parms->search_enable && tfs->shadow_copy) {
1769                 rc = tf_alloc_tbl_entry_shadow(tfs, parms);
1770                 /* Entry found and parms populated with return data */
1771                 if (rc == 0)
1772                         return rc;
1773         }
1774 #endif /* TF_SHADOW */
1775
1776         rc = tf_alloc_tbl_entry_pool_internal(tfp, parms);
1777         if (rc)
1778                 PMD_DRV_LOG(ERR, "dir%d, Alloc failed, rc:%d\n",
1779                             parms->dir,
1780                             rc);
1781
1782         return rc;
1783 }
1784
1785 /* API defined in tf_core.h */
1786 int
1787 tf_free_tbl_entry(struct tf *tfp,
1788                   struct tf_free_tbl_entry_parms *parms)
1789 {
1790         int rc;
1791 #if (TF_SHADOW == 1)
1792         struct tf_session *tfs;
1793 #endif /* TF_SHADOW */
1794
1795         /* Check parameters */
1796         if (parms == NULL || tfp == NULL) {
1797                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1798                 return -EINVAL;
1799         }
1800         /*
1801          * No shadow of external tables so just free the entry
1802          */
1803         if (parms->type == TF_TBL_TYPE_EXT) {
1804                 rc = tf_free_tbl_entry_pool_external(tfp, parms);
1805                 return rc;
1806         }
1807
1808 #if (TF_SHADOW == 1)
1809         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1810                 PMD_DRV_LOG(ERR,
1811                             "dir:%d, Session info invalid\n",
1812                             parms->dir);
1813                 return -EINVAL;
1814         }
1815
1816         tfs = (struct tf_session *)(tfp->session->core_data);
1817
1818         /* Search the Shadow DB for requested element. If not found go
1819          * allocate one from the Session Pool
1820          */
1821         if (parms->search_enable && tfs->shadow_copy) {
1822                 rc = tf_free_tbl_entry_shadow(tfs, parms);
1823                 /* Entry free'ed and parms populated with return data */
1824                 if (rc == 0)
1825                         return rc;
1826         }
1827 #endif /* TF_SHADOW */
1828
1829         rc = tf_free_tbl_entry_pool_internal(tfp, parms);
1830
1831         if (rc)
1832                 PMD_DRV_LOG(ERR, "dir:%d, Alloc failed, rc:%d\n",
1833                             parms->dir,
1834                             rc);
1835         return rc;
1836 }