drivers/net/bnxt/tf_core/tf_tbl.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 /* Truflow Table APIs and supporting code */
   7
   8 #include <stdio.h>
   9 #include <string.h>
  10 #include <stdbool.h>
  11 #include <math.h>
  12 #include <sys/param.h>
  13 #include <rte_common.h>
  14 #include <rte_errno.h>
  15 #include "hsi_struct_def_dpdk.h"
  16
  17 #include "tf_core.h"
  18 #include "tf_em.h"
  19 #include "tf_msg.h"
  20 #include "tfp.h"
  21 #include "hwrm_tf.h"
  22 #include "bnxt.h"
  23 #include "tf_resources.h"
  24 #include "tf_rm.h"
  25
  26 #define PTU_PTE_VALID          0x1UL
  27 #define PTU_PTE_LAST           0x2UL
  28 #define PTU_PTE_NEXT_TO_LAST   0x4UL
  29
  30 /* Number of pointers per page_size */
  31 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  32
  33 #define TF_EM_PG_SZ_4K        (1 << 12)
  34 #define TF_EM_PG_SZ_8K        (1 << 13)
  35 #define TF_EM_PG_SZ_64K       (1 << 16)
  36 #define TF_EM_PG_SZ_256K      (1 << 18)
  37 #define TF_EM_PG_SZ_1M        (1 << 20)
  38 #define TF_EM_PG_SZ_2M        (1 << 21)
  39 #define TF_EM_PG_SZ_4M        (1 << 22)
  40 #define TF_EM_PG_SZ_1G        (1 << 30)
  41
  42 #define TF_EM_CTX_ID_INVALID   0xFFFF
  43
  44 #define TF_EM_MIN_ENTRIES     (1 << 15) /* 32K */
  45 #define TF_EM_MAX_ENTRIES     (1 << 27) /* 128M */
  46
  47 /**
  48  * Function to free a page table
  49  *
  50  * [in] tp
  51  *   Pointer to the page table to free
  52  */
  53 static void
  54 tf_em_free_pg_tbl(struct tf_em_page_tbl *tp)
  55 {
  56         uint32_t i;
  57
  58         for (i = 0; i < tp->pg_count; i++) {
  59                 if (!tp->pg_va_tbl[i]) {
  60                         PMD_DRV_LOG(WARNING,
  61                                     "No map for page %d table %016" PRIu64 "\n",
  62                                     i,
  63                                     (uint64_t)(uintptr_t)tp);
  64                         continue;
  65                 }
  66
  67                 tfp_free(tp->pg_va_tbl[i]);
  68                 tp->pg_va_tbl[i] = NULL;
  69         }
  70
  71         tp->pg_count = 0;
  72         tfp_free(tp->pg_va_tbl);
  73         tp->pg_va_tbl = NULL;
  74         tfp_free(tp->pg_pa_tbl);
  75         tp->pg_pa_tbl = NULL;
  76 }
  77
  78 /**
  79  * Function to free an EM table
  80  *
  81  * [in] tbl
  82  *   Pointer to the EM table to free
  83  */
  84 static void
  85 tf_em_free_page_table(struct tf_em_table *tbl)
  86 {
  87         struct tf_em_page_tbl *tp;
  88         int i;
  89
  90         for (i = 0; i < tbl->num_lvl; i++) {
  91                 tp = &tbl->pg_tbl[i];
  92
  93                 PMD_DRV_LOG(INFO,
  94                            "EEM: Freeing page table: size %u lvl %d cnt %u\n",
  95                            TF_EM_PAGE_SIZE,
  96                             i,
  97                             tp->pg_count);
  98
  99                 tf_em_free_pg_tbl(tp);
 100         }
 101
 102         tbl->l0_addr = NULL;
 103         tbl->l0_dma_addr = 0;
 104         tbl->num_lvl = 0;
 105         tbl->num_data_pages = 0;
 106 }
 107
 108 /**
 109  * Allocation of page tables
 110  *
 111  * [in] tfp
 112  *   Pointer to a TruFlow handle
 113  *
 114  * [in] pg_count
 115  *   Page count to allocate
 116  *
 117  * [in] pg_size
 118  *   Size of each page
 119  *
 120  * Returns:
 121  *   0       - Success
 122  *   -ENOMEM - Out of memory
 123  */
 124 static int
 125 tf_em_alloc_pg_tbl(struct tf_em_page_tbl *tp,
 126                    uint32_t pg_count,
 127                    uint32_t pg_size)
 128 {
 129         uint32_t i;
 130         struct tfp_calloc_parms parms;
 131
 132         parms.nitems = pg_count;
 133         parms.size = sizeof(void *);
 134         parms.alignment = 0;
 135
 136         if (tfp_calloc(&parms) != 0)
 137                 return -ENOMEM;
 138
 139         tp->pg_va_tbl = parms.mem_va;
 140
 141         if (tfp_calloc(&parms) != 0) {
 142                 tfp_free(tp->pg_va_tbl);
 143                 return -ENOMEM;
 144         }
 145
 146         tp->pg_pa_tbl = parms.mem_va;
 147
 148         tp->pg_count = 0;
 149         tp->pg_size = pg_size;
 150
 151         for (i = 0; i < pg_count; i++) {
 152                 parms.nitems = 1;
 153                 parms.size = pg_size;
 154                 parms.alignment = TF_EM_PAGE_ALIGNMENT;
 155
 156                 if (tfp_calloc(&parms) != 0)
 157                         goto cleanup;
 158
 159                 tp->pg_pa_tbl[i] = (uintptr_t)parms.mem_pa;
 160                 tp->pg_va_tbl[i] = parms.mem_va;
 161
 162                 memset(tp->pg_va_tbl[i], 0, pg_size);
 163                 tp->pg_count++;
 164         }
 165
 166         return 0;
 167
 168 cleanup:
 169         tf_em_free_pg_tbl(tp);
 170         return -ENOMEM;
 171 }
 172
 173 /**
 174  * Allocates EM page tables
 175  *
 176  * [in] tbl
 177  *   Table to allocate pages for
 178  *
 179  * Returns:
 180  *   0       - Success
 181  *   -ENOMEM - Out of memory
 182  */
 183 static int
 184 tf_em_alloc_page_table(struct tf_em_table *tbl)
 185 {
 186         struct tf_em_page_tbl *tp;
 187         int rc = 0;
 188         int i;
 189         uint32_t j;
 190
 191         for (i = 0; i < tbl->num_lvl; i++) {
 192                 tp = &tbl->pg_tbl[i];
 193
 194                 rc = tf_em_alloc_pg_tbl(tp,
 195                                         tbl->page_cnt[i],
 196                                         TF_EM_PAGE_SIZE);
 197                 if (rc) {
 198                         PMD_DRV_LOG(WARNING,
 199                                 "Failed to allocate page table: lvl: %d\n",
 200                                 i);
 201                         goto cleanup;
 202                 }
 203
 204                 for (j = 0; j < tp->pg_count; j++) {
 205                         PMD_DRV_LOG(INFO,
 206                                 "EEM: Allocated page table: size %u lvl %d cnt"
 207                                 " %u VA:%p PA:%p\n",
 208                                 TF_EM_PAGE_SIZE,
 209                                 i,
 210                                 tp->pg_count,
 211                                 (uint32_t *)tp->pg_va_tbl[j],
 212                                 (uint32_t *)(uintptr_t)tp->pg_pa_tbl[j]);
 213                 }
 214         }
 215         return rc;
 216
 217 cleanup:
 218         tf_em_free_page_table(tbl);
 219         return rc;
 220 }
 221
 222 /**
 223  * Links EM page tables
 224  *
 225  * [in] tp
 226  *   Pointer to page table
 227  *
 228  * [in] tp_next
 229  *   Pointer to the next page table
 230  *
 231  * [in] set_pte_last
 232  *   Flag controlling if the page table is last
 233  */
 234 static void
 235 tf_em_link_page_table(struct tf_em_page_tbl *tp,
 236                       struct tf_em_page_tbl *tp_next,
 237                       bool set_pte_last)
 238 {
 239         uint64_t *pg_pa = tp_next->pg_pa_tbl;
 240         uint64_t *pg_va;
 241         uint64_t valid;
 242         uint32_t k = 0;
 243         uint32_t i;
 244         uint32_t j;
 245
 246         for (i = 0; i < tp->pg_count; i++) {
 247                 pg_va = tp->pg_va_tbl[i];
 248
 249                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
 250                         if (k == tp_next->pg_count - 2 && set_pte_last)
 251                                 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
 252                         else if (k == tp_next->pg_count - 1 && set_pte_last)
 253                                 valid = PTU_PTE_LAST | PTU_PTE_VALID;
 254                         else
 255                                 valid = PTU_PTE_VALID;
 256
 257                         pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
 258                         if (++k >= tp_next->pg_count)
 259                                 return;
 260                 }
 261         }
 262 }
 263
 264 /**
 265  * Setup a EM page table
 266  *
 267  * [in] tbl
 268  *   Pointer to EM page table
 269  */
 270 static void
 271 tf_em_setup_page_table(struct tf_em_table *tbl)
 272 {
 273         struct tf_em_page_tbl *tp_next;
 274         struct tf_em_page_tbl *tp;
 275         bool set_pte_last = 0;
 276         int i;
 277
 278         for (i = 0; i < tbl->num_lvl - 1; i++) {
 279                 tp = &tbl->pg_tbl[i];
 280                 tp_next = &tbl->pg_tbl[i + 1];
 281                 if (i == tbl->num_lvl - 2)
 282                         set_pte_last = 1;
 283                 tf_em_link_page_table(tp, tp_next, set_pte_last);
 284         }
 285
 286         tbl->l0_addr = tbl->pg_tbl[PT_LVL_0].pg_va_tbl[0];
 287         tbl->l0_dma_addr = tbl->pg_tbl[PT_LVL_0].pg_pa_tbl[0];
 288 }
 289
 290 /**
 291  * Given the page size, size of each data item (entry size),
 292  * and the total number of entries needed, determine the number
 293  * of page table levels and the number of data pages required.
 294  *
 295  * [in] page_size
 296  *   Page size
 297  *
 298  * [in] entry_size
 299  *   Entry size
 300  *
 301  * [in] num_entries
 302  *   Number of entries needed
 303  *
 304  * [out] num_data_pages
 305  *   Number of pages required
 306  *
 307  * Returns:
 308  *   Success  - Number of EM page levels required
 309  *   -ENOMEM  - Out of memory
 310  */
 311 static int
 312 tf_em_size_page_tbl_lvl(uint32_t page_size,
 313                         uint32_t entry_size,
 314                         uint32_t num_entries,
 315                         uint64_t *num_data_pages)
 316 {
 317         uint64_t lvl_data_size = page_size;
 318         int lvl = PT_LVL_0;
 319         uint64_t data_size;
 320
 321         *num_data_pages = 0;
 322         data_size = (uint64_t)num_entries * entry_size;
 323
 324         while (lvl_data_size < data_size) {
 325                 lvl++;
 326
 327                 if (lvl == PT_LVL_1)
 328                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 329                                 page_size;
 330                 else if (lvl == PT_LVL_2)
 331                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 332                                 MAX_PAGE_PTRS(page_size) * page_size;
 333                 else
 334                         return -ENOMEM;
 335         }
 336
 337         *num_data_pages = roundup(data_size, page_size) / page_size;
 338
 339         return lvl;
 340 }
 341
 342 /**
 343  * Return the number of page table pages needed to
 344  * reference the given number of next level pages.
 345  *
 346  * [in] num_pages
 347  *   Number of EM pages
 348  *
 349  * [in] page_size
 350  *   Size of each EM page
 351  *
 352  * Returns:
 353  *   Number of EM page table pages
 354  */
 355 static uint32_t
 356 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 357                      uint32_t page_size)
 358 {
 359         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 360                        MAX_PAGE_PTRS(page_size);
 361         return 0;
 362 }
 363
 364 /**
 365  * Given the number of data pages, page_size and the maximum
 366  * number of page table levels (already determined), size
 367  * the number of page table pages required at each level.
 368  *
 369  * [in] max_lvl
 370  *   Max number of levels
 371  *
 372  * [in] num_data_pages
 373  *   Number of EM data pages
 374  *
 375  * [in] page_size
 376  *   Size of an EM page
 377  *
 378  * [out] *page_cnt
 379  *   EM page count
 380  */
 381 static void
 382 tf_em_size_page_tbls(int max_lvl,
 383                      uint64_t num_data_pages,
 384                      uint32_t page_size,
 385                      uint32_t *page_cnt)
 386 {
 387         if (max_lvl == PT_LVL_0) {
 388                 page_cnt[PT_LVL_0] = num_data_pages;
 389         } else if (max_lvl == PT_LVL_1) {
 390                 page_cnt[PT_LVL_1] = num_data_pages;
 391                 page_cnt[PT_LVL_0] =
 392                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_1], page_size);
 393         } else if (max_lvl == PT_LVL_2) {
 394                 page_cnt[PT_LVL_2] = num_data_pages;
 395                 page_cnt[PT_LVL_1] =
 396                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_2], page_size);
 397                 page_cnt[PT_LVL_0] =
 398                 tf_em_page_tbl_pgcnt(page_cnt[PT_LVL_1], page_size);
 399         } else {
 400                 return;
 401         }
 402 }
 403
 404 /**
 405  * Size the EM table based on capabilities
 406  *
 407  * [in] tbl
 408  *   EM table to size
 409  *
 410  * Returns:
 411  *   0        - Success
 412  *   - EINVAL - Parameter error
 413  *   - ENOMEM - Out of memory
 414  */
 415 static int
 416 tf_em_size_table(struct tf_em_table *tbl)
 417 {
 418         uint64_t num_data_pages;
 419         uint32_t *page_cnt;
 420         int max_lvl;
 421         uint32_t num_entries;
 422         uint32_t cnt = TF_EM_MIN_ENTRIES;
 423
 424         /* Ignore entry if both size and number are zero */
 425         if (!tbl->entry_size && !tbl->num_entries)
 426                 return 0;
 427
 428         /* If only one is set then error */
 429         if (!tbl->entry_size || !tbl->num_entries)
 430                 return -EINVAL;
 431
 432         /* Determine number of page table levels and the number
 433          * of data pages needed to process the given eem table.
 434          */
 435         if (tbl->type == RECORD_TABLE) {
 436                 /*
 437                  * For action records just a memory size is provided. Work
 438                  * backwards to resolve to number of entries
 439                  */
 440                 num_entries = tbl->num_entries / tbl->entry_size;
 441                 if (num_entries < TF_EM_MIN_ENTRIES) {
 442                         num_entries = TF_EM_MIN_ENTRIES;
 443                 } else {
 444                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 445                                 cnt *= 2;
 446                         num_entries = cnt;
 447                 }
 448         } else {
 449                 num_entries = tbl->num_entries;
 450         }
 451
 452         max_lvl = tf_em_size_page_tbl_lvl(TF_EM_PAGE_SIZE,
 453                                           tbl->entry_size,
 454                                           tbl->num_entries,
 455                                           &num_data_pages);
 456         if (max_lvl < 0) {
 457                 PMD_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 458                 PMD_DRV_LOG(WARNING,
 459                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 460                             tbl->type,
 461                             (uint64_t)num_entries * tbl->entry_size,
 462                             TF_EM_PAGE_SIZE);
 463                 return -ENOMEM;
 464         }
 465
 466         tbl->num_lvl = max_lvl + 1;
 467         tbl->num_data_pages = num_data_pages;
 468
 469         /* Determine the number of pages needed at each level */
 470         page_cnt = tbl->page_cnt;
 471         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 472         tf_em_size_page_tbls(max_lvl, num_data_pages, TF_EM_PAGE_SIZE,
 473                                 page_cnt);
 474
 475         PMD_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 476         PMD_DRV_LOG(INFO,
 477                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 " l0: %u l1: %u l2: %u\n",
 478                     max_lvl + 1,
 479                     (uint64_t)num_data_pages * TF_EM_PAGE_SIZE,
 480                     num_data_pages,
 481                     page_cnt[PT_LVL_0],
 482                     page_cnt[PT_LVL_1],
 483                     page_cnt[PT_LVL_2]);
 484
 485         return 0;
 486 }
 487
 488 /**
 489  * Unregisters EM Ctx in Firmware
 490  *
 491  * [in] tfp
 492  *   Pointer to a TruFlow handle
 493  *
 494  * [in] tbl_scope_cb
 495  *   Pointer to a table scope control block
 496  *
 497  * [in] dir
 498  *   Receive or transmit direction
 499  */
 500 static void
 501 tf_em_ctx_unreg(struct tf *tfp,
 502                 struct tf_tbl_scope_cb *tbl_scope_cb,
 503                 int dir)
 504 {
 505         struct tf_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 506         struct tf_em_table *tbl;
 507         int i;
 508
 509         for (i = KEY0_TABLE; i < MAX_TABLE; i++) {
 510                 tbl = &ctxp->em_tables[i];
 511
 512                 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
 513                         tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
 514                         tf_em_free_page_table(tbl);
 515                 }
 516         }
 517 }
 518
 519 /**
 520  * Registers EM Ctx in Firmware
 521  *
 522  * [in] tfp
 523  *   Pointer to a TruFlow handle
 524  *
 525  * [in] tbl_scope_cb
 526  *   Pointer to a table scope control block
 527  *
 528  * [in] dir
 529  *   Receive or transmit direction
 530  *
 531  * Returns:
 532  *   0       - Success
 533  *   -ENOMEM - Out of Memory
 534  */
 535 static int
 536 tf_em_ctx_reg(struct tf *tfp,
 537               struct tf_tbl_scope_cb *tbl_scope_cb,
 538               int dir)
 539 {
 540         struct tf_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 541         struct tf_em_table *tbl;
 542         int rc = 0;
 543         int i;
 544
 545         for (i = KEY0_TABLE; i < MAX_TABLE; i++) {
 546                 tbl = &ctxp->em_tables[i];
 547
 548                 if (tbl->num_entries && tbl->entry_size) {
 549                         rc = tf_em_size_table(tbl);
 550
 551                         if (rc)
 552                                 goto cleanup;
 553
 554                         rc = tf_em_alloc_page_table(tbl);
 555                         if (rc)
 556                                 goto cleanup;
 557
 558                         tf_em_setup_page_table(tbl);
 559                         rc = tf_msg_em_mem_rgtr(tfp,
 560                                                 tbl->num_lvl - 1,
 561                                                 TF_EM_PAGE_SIZE_ENUM,
 562                                                 tbl->l0_dma_addr,
 563                                                 &tbl->ctx_id);
 564                         if (rc)
 565                                 goto cleanup;
 566                 }
 567         }
 568         return rc;
 569
 570 cleanup:
 571         tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
 572         return rc;
 573 }
 574
 575 /**
 576  * Validates EM number of entries requested
 577  *
 578  * [in] tbl_scope_cb
 579  *   Pointer to table scope control block to be populated
 580  *
 581  * [in] parms
 582  *   Pointer to input parameters
 583  *
 584  * Returns:
 585  *   0       - Success
 586  *   -EINVAL - Parameter error
 587  */
 588 static int
 589 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 590                            struct tf_alloc_tbl_scope_parms *parms)
 591 {
 592         uint32_t cnt;
 593
 594         if (parms->rx_mem_size_in_mb != 0) {
 595                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 596                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 597                                      + 1);
 598                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 599                                         TF_MEGABYTE) / (key_b + action_b);
 600
 601                 if (num_entries < TF_EM_MIN_ENTRIES) {
 602                         PMD_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 603                                     "%uMB\n",
 604                                     parms->rx_mem_size_in_mb);
 605                         return -EINVAL;
 606                 }
 607
 608                 cnt = TF_EM_MIN_ENTRIES;
 609                 while (num_entries > cnt &&
 610                        cnt <= TF_EM_MAX_ENTRIES)
 611                         cnt *= 2;
 612
 613                 if (cnt > TF_EM_MAX_ENTRIES) {
 614                         PMD_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 615                                     "%u\n",
 616                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 617                         return -EINVAL;
 618                 }
 619
 620                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 621         } else {
 622                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 623                     TF_EM_MIN_ENTRIES ||
 624                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 625                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 626                         PMD_DRV_LOG(ERR,
 627                                     "EEM: Invalid number of Rx flows "
 628                                     "requested:%u max:%u\n",
 629                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 630                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 631                         return -EINVAL;
 632                 }
 633
 634                 /* must be a power-of-2 supported value
 635                  * in the range 32K - 128M
 636                  */
 637                 cnt = TF_EM_MIN_ENTRIES;
 638                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 639                        cnt <= TF_EM_MAX_ENTRIES)
 640                         cnt *= 2;
 641
 642                 if (cnt > TF_EM_MAX_ENTRIES) {
 643                         PMD_DRV_LOG(ERR,
 644                                     "EEM: Invalid number of Rx requested: %u\n",
 645                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 646                         return -EINVAL;
 647                 }
 648         }
 649
 650         if (parms->tx_mem_size_in_mb != 0) {
 651                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 652                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 653                                      + 1);
 654                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 655                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 656                         (key_b + action_b);
 657
 658                 if (num_entries < TF_EM_MIN_ENTRIES) {
 659                         PMD_DRV_LOG(ERR,
 660                                     "EEM: Insufficient memory requested:%uMB\n",
 661                                     parms->rx_mem_size_in_mb);
 662                         return -EINVAL;
 663                 }
 664
 665                 cnt = TF_EM_MIN_ENTRIES;
 666                 while (num_entries > cnt &&
 667                        cnt <= TF_EM_MAX_ENTRIES)
 668                         cnt *= 2;
 669
 670                 if (cnt > TF_EM_MAX_ENTRIES) {
 671                         PMD_DRV_LOG(ERR,
 672                                     "EEM: Invalid number of Tx requested: %u\n",
 673                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 674                         return -EINVAL;
 675                 }
 676
 677                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 678         } else {
 679                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 680                     TF_EM_MIN_ENTRIES ||
 681                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 682                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 683                         PMD_DRV_LOG(ERR,
 684                                     "EEM: Invalid number of Tx flows "
 685                                     "requested:%u max:%u\n",
 686                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 687                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 688                         return -EINVAL;
 689                 }
 690
 691                 cnt = TF_EM_MIN_ENTRIES;
 692                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 693                        cnt <= TF_EM_MAX_ENTRIES)
 694                         cnt *= 2;
 695
 696                 if (cnt > TF_EM_MAX_ENTRIES) {
 697                         PMD_DRV_LOG(ERR,
 698                                     "EEM: Invalid number of Tx requested: %u\n",
 699                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 700                         return -EINVAL;
 701                 }
 702         }
 703
 704         if (parms->rx_num_flows_in_k != 0 &&
 705             (parms->rx_max_key_sz_in_bits / 8 == 0)) {
 706                 PMD_DRV_LOG(ERR,
 707                             "EEM: Rx key size required: %u\n",
 708                             (parms->rx_max_key_sz_in_bits));
 709                 return -EINVAL;
 710         }
 711
 712         if (parms->tx_num_flows_in_k != 0 &&
 713             (parms->tx_max_key_sz_in_bits / 8 == 0)) {
 714                 PMD_DRV_LOG(ERR,
 715                             "EEM: Tx key size required: %u\n",
 716                             (parms->tx_max_key_sz_in_bits));
 717                 return -EINVAL;
 718         }
 719         /* Rx */
 720         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY0_TABLE].num_entries =
 721                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 722         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY0_TABLE].entry_size =
 723                 parms->rx_max_key_sz_in_bits / 8;
 724
 725         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY1_TABLE].num_entries =
 726                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 727         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[KEY1_TABLE].entry_size =
 728                 parms->rx_max_key_sz_in_bits / 8;
 729
 730         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[RECORD_TABLE].num_entries =
 731                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 732         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[RECORD_TABLE].entry_size =
 733                 parms->rx_max_action_entry_sz_in_bits / 8;
 734
 735         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[EFC_TABLE].num_entries =
 736                 0;
 737
 738         /* Tx */
 739         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY0_TABLE].num_entries =
 740                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 741         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY0_TABLE].entry_size =
 742                 parms->tx_max_key_sz_in_bits / 8;
 743
 744         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY1_TABLE].num_entries =
 745                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 746         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[KEY1_TABLE].entry_size =
 747                 parms->tx_max_key_sz_in_bits / 8;
 748
 749         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[RECORD_TABLE].num_entries =
 750                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 751         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[RECORD_TABLE].entry_size =
 752                 parms->tx_max_action_entry_sz_in_bits / 8;
 753
 754         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[EFC_TABLE].num_entries =
 755                 0;
 756
 757         return 0;
 758 }
 759
 760 /**
 761  * Internal function to set a Table Entry. Supports all internal Table Types
 762  *
 763  * [in] tfp
 764  *   Pointer to TruFlow handle
 765  *
 766  * [in] parms
 767  *   Pointer to input parameters
 768  *
 769  * Returns:
 770  *   0       - Success
 771  *   -EINVAL - Parameter error
 772  */
 773 static int
 774 tf_set_tbl_entry_internal(struct tf *tfp,
 775                           struct tf_set_tbl_entry_parms *parms)
 776 {
 777         int rc;
 778         int id;
 779         uint32_t index;
 780         struct bitalloc *session_pool;
 781         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 782
 783         /* Lookup the pool using the table type of the element */
 784         rc = tf_rm_lookup_tbl_type_pool(tfs,
 785                                         parms->dir,
 786                                         parms->type,
 787                                         &session_pool);
 788         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 789         if (rc)
 790                 return rc;
 791
 792         index = parms->idx;
 793
 794         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
 795             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
 796             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
 797                 PMD_DRV_LOG(ERR,
 798                             "dir:%d, Type not supported, type:%d\n",
 799                             parms->dir,
 800                             parms->type);
 801                 return -EOPNOTSUPP;
 802         }
 803
 804         /* Adjust the returned index/offset as there is no guarantee
 805          * that the start is 0 at time of RM allocation
 806          */
 807         tf_rm_convert_index(tfs,
 808                             parms->dir,
 809                             parms->type,
 810                             TF_RM_CONVERT_RM_BASE,
 811                             parms->idx,
 812                             &index);
 813
 814         /* Verify that the entry has been previously allocated */
 815         id = ba_inuse(session_pool, index);
 816         if (id != 1) {
 817                 PMD_DRV_LOG(ERR,
 818                    "dir:%d, Invalid or not allocated index, type:%d, idx:%d\n",
 819                    parms->dir,
 820                    parms->type,
 821                    index);
 822                 return -EINVAL;
 823         }
 824
 825         /* Set the entry */
 826         rc = tf_msg_set_tbl_entry(tfp,
 827                                   parms->dir,
 828                                   parms->type,
 829                                   parms->data_sz_in_bytes,
 830                                   parms->data,
 831                                   parms->idx);
 832         if (rc) {
 833                 PMD_DRV_LOG(ERR,
 834                             "dir:%d, Set failed, type:%d, rc:%d\n",
 835                             parms->dir,
 836                             parms->type,
 837                             rc);
 838         }
 839
 840         return rc;
 841 }
 842
 843 /**
 844  * Internal function to get a Table Entry. Supports all Table Types
 845  * except the TF_TBL_TYPE_EXT as that is handled as a table scope.
 846  *
 847  * [in] tfp
 848  *   Pointer to TruFlow handle
 849  *
 850  * [in] parms
 851  *   Pointer to input parameters
 852  *
 853  * Returns:
 854  *   0       - Success
 855  *   -EINVAL - Parameter error
 856  */
 857 static int
 858 tf_get_tbl_entry_internal(struct tf *tfp,
 859                           struct tf_get_tbl_entry_parms *parms)
 860 {
 861         int rc;
 862         int id;
 863         uint32_t index;
 864         struct bitalloc *session_pool;
 865         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 866
 867         /* Lookup the pool using the table type of the element */
 868         rc = tf_rm_lookup_tbl_type_pool(tfs,
 869                                         parms->dir,
 870                                         parms->type,
 871                                         &session_pool);
 872         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 873         if (rc)
 874                 return rc;
 875
 876         index = parms->idx;
 877
 878         /* Adjust the returned index/offset as there is no guarantee
 879          * that the start is 0 at time of RM allocation
 880          */
 881         tf_rm_convert_index(tfs,
 882                             parms->dir,
 883                             parms->type,
 884                             TF_RM_CONVERT_RM_BASE,
 885                             parms->idx,
 886                             &index);
 887
 888         /* Verify that the entry has been previously allocated */
 889         id = ba_inuse(session_pool, index);
 890         if (id != 1) {
 891                 PMD_DRV_LOG(ERR,
 892                    "dir:%d, Invalid or not allocated index, type:%d, idx:%d\n",
 893                    parms->dir,
 894                    parms->type,
 895                    index);
 896                 return -EINVAL;
 897         }
 898
 899         /* Get the entry */
 900         rc = tf_msg_get_tbl_entry(tfp,
 901                                   parms->dir,
 902                                   parms->type,
 903                                   parms->data_sz_in_bytes,
 904                                   parms->data,
 905                                   parms->idx);
 906         if (rc) {
 907                 PMD_DRV_LOG(ERR,
 908                             "dir:%d, Get failed, type:%d, rc:%d\n",
 909                             parms->dir,
 910                             parms->type,
 911                             rc);
 912         }
 913
 914         return rc;
 915 }
 916
 917 #if (TF_SHADOW == 1)
 918 /**
 919  * Allocate Tbl entry from the Shadow DB. Shadow DB is searched for
 920  * the requested entry. If found the ref count is incremente and
 921  * returned.
 922  *
 923  * [in] tfs
 924  *   Pointer to session
 925  * [in] parms
 926  *   Allocation parameters
 927  *
 928  * Return:
 929  *  0       - Success, entry found and ref count incremented
 930  *  -ENOENT - Failure, entry not found
 931  */
 932 static int
 933 tf_alloc_tbl_entry_shadow(struct tf_session *tfs __rte_unused,
 934                           struct tf_alloc_tbl_entry_parms *parms __rte_unused)
 935 {
 936         PMD_DRV_LOG(ERR,
 937                     "dir:%d, Entry Alloc with search not supported\n",
 938                     parms->dir);
 939
 940
 941         return -EOPNOTSUPP;
 942 }
 943
 944 /**
 945  * Free Tbl entry from the Shadow DB. Shadow DB is searched for
 946  * the requested entry. If found the ref count is decremente and
 947  * new ref_count returned.
 948  *
 949  * [in] tfs
 950  *   Pointer to session
 951  * [in] parms
 952  *   Allocation parameters
 953  *
 954  * Return:
 955  *  0       - Success, entry found and ref count decremented
 956  *  -ENOENT - Failure, entry not found
 957  */
 958 static int
 959 tf_free_tbl_entry_shadow(struct tf_session *tfs,
 960                          struct tf_free_tbl_entry_parms *parms)
 961 {
 962         PMD_DRV_LOG(ERR,
 963                     "dir:%d, Entry Free with search not supported\n",
 964                     parms->dir);
 965
 966         return -EOPNOTSUPP;
 967 }
 968 #endif /* TF_SHADOW */
 969
 970 /**
 971  * Create External Tbl pool of memory indexes.
 972  *
 973  * [in] dir
 974  *   direction
 975  * [in] tbl_scope_cb
 976  *   pointer to the table scope
 977  * [in] num_entries
 978  *   number of entries to write
 979  * [in] entry_sz_bytes
 980  *   size of each entry
 981  *
 982  * Return:
 983  *  0       - Success, entry allocated - no search support
 984  *  -ENOMEM -EINVAL -EOPNOTSUPP
 985  *          - Failure, entry not allocated, out of resources
 986  */
 987 static int
 988 tf_create_tbl_pool_external(enum tf_dir dir,
 989                             struct tf_tbl_scope_cb *tbl_scope_cb,
 990                             uint32_t num_entries,
 991                             uint32_t entry_sz_bytes)
 992 {
 993         struct tfp_calloc_parms parms;
 994         uint32_t i;
 995         int32_t j;
 996         int rc = 0;
 997         struct stack *pool = &tbl_scope_cb->ext_act_pool[dir];
 998
 999         parms.nitems = num_entries;
1000         parms.size = sizeof(uint32_t);
1001         parms.alignment = 0;
1002
1003         if (tfp_calloc(&parms) != 0) {
1004                 PMD_DRV_LOG(ERR, "%d: TBL: external pool failure %s\n",
1005                             dir, strerror(-ENOMEM));
1006                 return -ENOMEM;
1007         }
1008
1009         /* Create empty stack
1010          */
1011         rc = stack_init(num_entries, parms.mem_va, pool);
1012
1013         if (rc != 0) {
1014                 PMD_DRV_LOG(ERR, "%d: TBL: stack init failure %s\n",
1015                             dir, strerror(-rc));
1016                 goto cleanup;
1017         }
1018
1019         /* Save the  malloced memory address so that it can
1020          * be freed when the table scope is freed.
1021          */
1022         tbl_scope_cb->ext_act_pool_mem[dir] = (uint32_t *)parms.mem_va;
1023
1024         /* Fill pool with indexes in reverse
1025          */
1026         j = (num_entries - 1) * entry_sz_bytes;
1027
1028         for (i = 0; i < num_entries; i++) {
1029                 rc = stack_push(pool, j);
1030                 if (rc != 0) {
1031                         PMD_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
1032                                     tf_dir_2_str(dir), strerror(-rc));
1033                         goto cleanup;
1034                 }
1035
1036                 if (j < 0) {
1037                         PMD_DRV_LOG(ERR, "%d TBL: invalid offset (%d)\n",
1038                                     dir, j);
1039                         goto cleanup;
1040                 }
1041                 j -= entry_sz_bytes;
1042         }
1043
1044         if (!stack_is_full(pool)) {
1045                 rc = -EINVAL;
1046                 PMD_DRV_LOG(ERR, "%d TBL: stack failure %s\n",
1047                             dir, strerror(-rc));
1048                 goto cleanup;
1049         }
1050         return 0;
1051 cleanup:
1052         tfp_free((void *)parms.mem_va);
1053         return rc;
1054 }
1055
1056 /**
1057  * Destroy External Tbl pool of memory indexes.
1058  *
1059  * [in] dir
1060  *   direction
1061  * [in] tbl_scope_cb
1062  *   pointer to the table scope
1063  *
1064  */
1065 static void
1066 tf_destroy_tbl_pool_external(enum tf_dir dir,
1067                              struct tf_tbl_scope_cb *tbl_scope_cb)
1068 {
1069         uint32_t *ext_act_pool_mem =
1070                 tbl_scope_cb->ext_act_pool_mem[dir];
1071
1072         tfp_free(ext_act_pool_mem);
1073 }
1074
1075 /**
1076  * Allocate External Tbl entry from the Session Pool.
1077  *
1078  * [in] tfp
1079  *   Pointer to Truflow Handle
1080  * [in] parms
1081  *   Allocation parameters
1082  *
1083  * Return:
1084  *  0       - Success, entry allocated - no search support
1085  *  -ENOMEM -EINVAL -EOPNOTSUPP
1086  *          - Failure, entry not allocated, out of resources
1087  */
1088 static int
1089 tf_alloc_tbl_entry_pool_external(struct tf *tfp,
1090                                  struct tf_alloc_tbl_entry_parms *parms)
1091 {
1092         int rc;
1093         uint32_t index;
1094         struct tf_session *tfs;
1095         struct tf_tbl_scope_cb *tbl_scope_cb;
1096         struct stack *pool;
1097
1098         /* Check parameters */
1099         if (tfp == NULL || parms == NULL) {
1100                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1101                 return -EINVAL;
1102         }
1103
1104         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1105                 PMD_DRV_LOG(ERR,
1106                             "dir:%d, Session info invalid\n",
1107                             parms->dir);
1108                 return -EINVAL;
1109         }
1110
1111         tfs = (struct tf_session *)(tfp->session->core_data);
1112
1113         /* Get the pool info from the table scope
1114          */
1115         tbl_scope_cb = tbl_scope_cb_find(tfs, parms->tbl_scope_id);
1116
1117         if (tbl_scope_cb == NULL) {
1118                 PMD_DRV_LOG(ERR,
1119                                         "%s, table scope not allocated\n",
1120                                         tf_dir_2_str(parms->dir));
1121                 return -EINVAL;
1122         }
1123         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
1124
1125         /* Allocate an element
1126          */
1127         rc = stack_pop(pool, &index);
1128
1129         if (rc != 0) {
1130                 PMD_DRV_LOG(ERR,
1131                    "dir:%d, Allocation failed, type:%d\n",
1132                    parms->dir,
1133                    parms->type);
1134                 return rc;
1135         }
1136         parms->idx = index;
1137         return rc;
1138 }
1139
1140 /**
1141  * Allocate Internal Tbl entry from the Session Pool.
1142  *
1143  * [in] tfp
1144  *   Pointer to Truflow Handle
1145  * [in] parms
1146  *   Allocation parameters
1147  *
1148  * Return:
1149  *  0       - Success, entry found and ref count decremented
1150  *  -ENOMEM - Failure, entry not allocated, out of resources
1151  */
1152 static int
1153 tf_alloc_tbl_entry_pool_internal(struct tf *tfp,
1154                                  struct tf_alloc_tbl_entry_parms *parms)
1155 {
1156         int rc;
1157         int id;
1158         int free_cnt;
1159         uint32_t index;
1160         struct bitalloc *session_pool;
1161         struct tf_session *tfs;
1162
1163         /* Check parameters */
1164         if (tfp == NULL || parms == NULL) {
1165                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1166                 return -EINVAL;
1167         }
1168
1169         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1170                 PMD_DRV_LOG(ERR,
1171                             "dir:%d, Session info invalid\n",
1172                             parms->dir);
1173                 return -EINVAL;
1174         }
1175
1176         tfs = (struct tf_session *)(tfp->session->core_data);
1177
1178         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1179             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1180             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1181             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1182             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1183             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B &&
1184             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
1185                 PMD_DRV_LOG(ERR,
1186                             "dir:%d, Type not supported, type:%d\n",
1187                             parms->dir,
1188                             parms->type);
1189                 return -EOPNOTSUPP;
1190         }
1191
1192         /* Lookup the pool using the table type of the element */
1193         rc = tf_rm_lookup_tbl_type_pool(tfs,
1194                                         parms->dir,
1195                                         parms->type,
1196                                         &session_pool);
1197         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1198         if (rc)
1199                 return rc;
1200
1201         id = ba_alloc(session_pool);
1202         if (id == -1) {
1203                 free_cnt = ba_free_count(session_pool);
1204
1205                 PMD_DRV_LOG(ERR,
1206                    "dir:%d, Allocation failed, type:%d, free:%d\n",
1207                    parms->dir,
1208                    parms->type,
1209                    free_cnt);
1210                 return -ENOMEM;
1211         }
1212
1213         /* Adjust the returned index/offset as there is no guarantee
1214          * that the start is 0 at time of RM allocation
1215          */
1216         tf_rm_convert_index(tfs,
1217                             parms->dir,
1218                             parms->type,
1219                             TF_RM_CONVERT_ADD_BASE,
1220                             id,
1221                             &index);
1222         parms->idx = index;
1223         return rc;
1224 }
1225
1226 /**
1227  * Free External Tbl entry to the session pool.
1228  *
1229  * [in] tfp
1230  *   Pointer to Truflow Handle
1231  * [in] parms
1232  *   Allocation parameters
1233  *
1234  * Return:
1235  *  0       - Success, entry freed
1236  *
1237  * - Failure, entry not successfully freed for these reasons
1238  *  -ENOMEM
1239  *  -EOPNOTSUPP
1240  *  -EINVAL
1241  */
1242 static int
1243 tf_free_tbl_entry_pool_external(struct tf *tfp,
1244                                 struct tf_free_tbl_entry_parms *parms)
1245 {
1246         int rc = 0;
1247         struct tf_session *tfs;
1248         uint32_t index;
1249         struct tf_tbl_scope_cb *tbl_scope_cb;
1250         struct stack *pool;
1251
1252         /* Check parameters */
1253         if (tfp == NULL || parms == NULL) {
1254                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1255                 return -EINVAL;
1256         }
1257
1258         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1259                 PMD_DRV_LOG(ERR,
1260                             "dir:%d, Session info invalid\n",
1261                             parms->dir);
1262                 return -EINVAL;
1263         }
1264
1265         tfs = (struct tf_session *)(tfp->session->core_data);
1266
1267         /* Get the pool info from the table scope
1268          */
1269         tbl_scope_cb = tbl_scope_cb_find(tfs, parms->tbl_scope_id);
1270
1271         if (tbl_scope_cb == NULL) {
1272                 PMD_DRV_LOG(ERR,
1273                             "dir:%d, Session info invalid\n",
1274                             parms->dir);
1275                 return -EINVAL;
1276         }
1277         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
1278
1279         index = parms->idx;
1280
1281         rc = stack_push(pool, index);
1282
1283         if (rc != 0) {
1284                 PMD_DRV_LOG(ERR,
1285                    "dir:%d, consistency error, stack full, type:%d, idx:%d\n",
1286                    parms->dir,
1287                    parms->type,
1288                    index);
1289         }
1290         return rc;
1291 }
1292
1293 /**
1294  * Free Internal Tbl entry from the Session Pool.
1295  *
1296  * [in] tfp
1297  *   Pointer to Truflow Handle
1298  * [in] parms
1299  *   Allocation parameters
1300  *
1301  * Return:
1302  *  0       - Success, entry found and ref count decremented
1303  *  -ENOMEM - Failure, entry not allocated, out of resources
1304  */
1305 static int
1306 tf_free_tbl_entry_pool_internal(struct tf *tfp,
1307                        struct tf_free_tbl_entry_parms *parms)
1308 {
1309         int rc = 0;
1310         int id;
1311         struct bitalloc *session_pool;
1312         struct tf_session *tfs;
1313         uint32_t index;
1314
1315         /* Check parameters */
1316         if (tfp == NULL || parms == NULL) {
1317                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1318                 return -EINVAL;
1319         }
1320
1321         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1322                 PMD_DRV_LOG(ERR,
1323                             "dir:%d, Session info invalid\n",
1324                             parms->dir);
1325                 return -EINVAL;
1326         }
1327
1328         tfs = (struct tf_session *)(tfp->session->core_data);
1329
1330         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1331             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1332             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1333             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1334             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1335             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B &&
1336             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
1337                 PMD_DRV_LOG(ERR,
1338                             "dir:%d, Type not supported, type:%d\n",
1339                             parms->dir,
1340                             parms->type);
1341                 return -EOPNOTSUPP;
1342         }
1343
1344         /* Lookup the pool using the table type of the element */
1345         rc = tf_rm_lookup_tbl_type_pool(tfs,
1346                                         parms->dir,
1347                                         parms->type,
1348                                         &session_pool);
1349         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1350         if (rc)
1351                 return rc;
1352
1353         index = parms->idx;
1354
1355         /* Adjust the returned index/offset as there is no guarantee
1356          * that the start is 0 at time of RM allocation
1357          */
1358         tf_rm_convert_index(tfs,
1359                             parms->dir,
1360                             parms->type,
1361                             TF_RM_CONVERT_RM_BASE,
1362                             parms->idx,
1363                             &index);
1364
1365         /* Check if element was indeed allocated */
1366         id = ba_inuse_free(session_pool, index);
1367         if (id == -1) {
1368                 PMD_DRV_LOG(ERR,
1369                    "dir:%d, Element not previously alloc'ed, type:%d, idx:%d\n",
1370                    parms->dir,
1371                    parms->type,
1372                    index);
1373                 return -ENOMEM;
1374         }
1375
1376         return rc;
1377 }
1378
1379 /* API defined in tf_em.h */
1380 struct tf_tbl_scope_cb *
1381 tbl_scope_cb_find(struct tf_session *session,
1382                   uint32_t tbl_scope_id)
1383 {
1384         int i;
1385
1386         /* Check that id is valid */
1387         i = ba_inuse(session->tbl_scope_pool_rx, tbl_scope_id);
1388         if (i < 0)
1389                 return NULL;
1390
1391         for (i = 0; i < TF_NUM_TBL_SCOPE; i++) {
1392                 if (session->tbl_scopes[i].tbl_scope_id == tbl_scope_id)
1393                         return &session->tbl_scopes[i];
1394         }
1395
1396         return NULL;
1397 }
1398
1399 /* API defined in tf_core.h */
1400 int
1401 tf_free_eem_tbl_scope_cb(struct tf *tfp,
1402                          struct tf_free_tbl_scope_parms *parms)
1403 {
1404         int rc = 0;
1405         enum tf_dir  dir;
1406         struct tf_tbl_scope_cb *tbl_scope_cb;
1407         struct tf_session *session;
1408
1409         session = (struct tf_session *)(tfp->session->core_data);
1410
1411         tbl_scope_cb = tbl_scope_cb_find(session,
1412                                          parms->tbl_scope_id);
1413
1414         if (tbl_scope_cb == NULL)
1415                 return -EINVAL;
1416
1417         /* Free Table control block */
1418         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1419
1420         /* free table scope locks */
1421         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1422                 /* Free associated external pools
1423                  */
1424                 tf_destroy_tbl_pool_external(dir,
1425                                              tbl_scope_cb);
1426                 tf_msg_em_op(tfp,
1427                              dir,
1428                              HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
1429
1430                 /* free table scope and all associated resources */
1431                 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
1432         }
1433
1434         return rc;
1435 }
1436
1437 /* API defined in tf_em.h */
1438 int
1439 tf_alloc_eem_tbl_scope(struct tf *tfp,
1440                        struct tf_alloc_tbl_scope_parms *parms)
1441 {
1442         int rc;
1443         enum tf_dir dir;
1444         struct tf_tbl_scope_cb *tbl_scope_cb;
1445         struct tf_em_table *em_tables;
1446         int index;
1447         struct tf_session *session;
1448         struct tf_free_tbl_scope_parms free_parms;
1449
1450         /* check parameters */
1451         if (parms == NULL || tfp->session == NULL) {
1452                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1453                 return -EINVAL;
1454         }
1455
1456         session = (struct tf_session *)tfp->session->core_data;
1457
1458         /* Get Table Scope control block from the session pool */
1459         index = ba_alloc(session->tbl_scope_pool_rx);
1460         if (index == -1) {
1461                 PMD_DRV_LOG(ERR, "EEM: Unable to allocate table scope "
1462                             "Control Block\n");
1463                 return -ENOMEM;
1464         }
1465
1466         tbl_scope_cb = &session->tbl_scopes[index];
1467         tbl_scope_cb->index = index;
1468         tbl_scope_cb->tbl_scope_id = index;
1469         parms->tbl_scope_id = index;
1470
1471         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1472                 rc = tf_msg_em_qcaps(tfp,
1473                                      dir,
1474                                      &tbl_scope_cb->em_caps[dir]);
1475                 if (rc) {
1476                         PMD_DRV_LOG(ERR,
1477                                 "EEM: Unable to query for EEM capability\n");
1478                         goto cleanup;
1479                 }
1480         }
1481
1482         /*
1483          * Validate and setup table sizes
1484          */
1485         if (tf_em_validate_num_entries(tbl_scope_cb, parms))
1486                 goto cleanup;
1487
1488         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1489                 /*
1490                  * Allocate tables and signal configuration to FW
1491                  */
1492                 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
1493                 if (rc) {
1494                         PMD_DRV_LOG(ERR,
1495                                     "EEM: Unable to register for EEM ctx\n");
1496                         goto cleanup;
1497                 }
1498
1499                 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
1500                 rc = tf_msg_em_cfg(tfp,
1501                                    em_tables[KEY0_TABLE].num_entries,
1502                                    em_tables[KEY0_TABLE].ctx_id,
1503                                    em_tables[KEY1_TABLE].ctx_id,
1504                                    em_tables[RECORD_TABLE].ctx_id,
1505                                    em_tables[EFC_TABLE].ctx_id,
1506                                    parms->hw_flow_cache_flush_timer,
1507                                    dir);
1508                 if (rc) {
1509                         PMD_DRV_LOG(ERR,
1510                                 "TBL: Unable to configure EEM in firmware\n");
1511                         goto cleanup_full;
1512                 }
1513
1514                 rc = tf_msg_em_op(tfp,
1515                                   dir,
1516                                   HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
1517
1518                 if (rc) {
1519                         PMD_DRV_LOG(ERR,
1520                                     "EEM: Unable to enable EEM in firmware\n");
1521                         goto cleanup_full;
1522                 }
1523
1524                 /* Allocate the pool of offsets of the external memory.
1525                  * Initially, this is a single fixed size pool for all external
1526                  * actions related to a single table scope.
1527                  */
1528                 rc = tf_create_tbl_pool_external(dir,
1529                                             tbl_scope_cb,
1530                                             em_tables[RECORD_TABLE].num_entries,
1531                                             em_tables[RECORD_TABLE].entry_size);
1532                 if (rc) {
1533                         PMD_DRV_LOG(ERR,
1534                                     "%d TBL: Unable to allocate idx pools %s\n",
1535                                     dir,
1536                                     strerror(-rc));
1537                         goto cleanup_full;
1538                 }
1539         }
1540
1541         return 0;
1542
1543 cleanup_full:
1544         free_parms.tbl_scope_id = index;
1545         tf_free_eem_tbl_scope_cb(tfp, &free_parms);
1546         return -EINVAL;
1547
1548 cleanup:
1549         /* Free Table control block */
1550         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1551         return -EINVAL;
1552 }
1553
1554 /* API defined in tf_core.h */
1555 int
1556 tf_set_tbl_entry(struct tf *tfp,
1557                  struct tf_set_tbl_entry_parms *parms)
1558 {
1559         int rc = 0;
1560         struct tf_tbl_scope_cb *tbl_scope_cb;
1561         struct tf_session *session;
1562
1563         if (tfp == NULL || parms == NULL || parms->data == NULL)
1564                 return -EINVAL;
1565
1566         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1567                 PMD_DRV_LOG(ERR,
1568                             "dir:%d, Session info invalid\n",
1569                             parms->dir);
1570                 return -EINVAL;
1571         }
1572
1573         if (parms->type == TF_TBL_TYPE_EXT) {
1574                 void *base_addr;
1575                 uint32_t offset = parms->idx;
1576                 uint32_t tbl_scope_id;
1577
1578                 session = (struct tf_session *)(tfp->session->core_data);
1579
1580                 tbl_scope_id = parms->tbl_scope_id;
1581
1582                 if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1583                         PMD_DRV_LOG(ERR,
1584                                     "dir:%d, Table scope not allocated\n",
1585                                     parms->dir);
1586                         return -EINVAL;
1587                 }
1588
1589                 /* Get the table scope control block associated with the
1590                  * external pool
1591                  */
1592                 tbl_scope_cb = tbl_scope_cb_find(session, tbl_scope_id);
1593
1594                 if (tbl_scope_cb == NULL)
1595                         return -EINVAL;
1596
1597                 /* External table, implicitly the Action table */
1598                 base_addr = tf_em_get_table_page(tbl_scope_cb,
1599                                                  parms->dir,
1600                                                  offset,
1601                                                  RECORD_TABLE);
1602                 if (base_addr == NULL) {
1603                         PMD_DRV_LOG(ERR,
1604                                     "dir:%d, Base address lookup failed\n",
1605                                     parms->dir);
1606                         return -EINVAL;
1607                 }
1608
1609                 offset %= TF_EM_PAGE_SIZE;
1610                 rte_memcpy((char *)base_addr + offset,
1611                            parms->data,
1612                            parms->data_sz_in_bytes);
1613         } else {
1614                 /* Internal table type processing */
1615                 rc = tf_set_tbl_entry_internal(tfp, parms);
1616                 if (rc) {
1617                         PMD_DRV_LOG(ERR,
1618                                     "dir:%d, Set failed, type:%d, rc:%d\n",
1619                                     parms->dir,
1620                                     parms->type,
1621                                     rc);
1622                 }
1623         }
1624
1625         return rc;
1626 }
1627
1628 /* API defined in tf_core.h */
1629 int
1630 tf_get_tbl_entry(struct tf *tfp,
1631                  struct tf_get_tbl_entry_parms *parms)
1632 {
1633         int rc = 0;
1634
1635         if (tfp == NULL || parms == NULL)
1636                 return -EINVAL;
1637
1638         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1639                 PMD_DRV_LOG(ERR,
1640                             "dir:%d, Session info invalid\n",
1641                             parms->dir);
1642                 return -EINVAL;
1643         }
1644
1645         if (parms->type == TF_TBL_TYPE_EXT) {
1646                 PMD_DRV_LOG(ERR,
1647                             "dir:%d, External table type not supported\n",
1648                             parms->dir);
1649
1650                 rc = -EOPNOTSUPP;
1651         } else {
1652                 /* Internal table type processing */
1653                 rc = tf_get_tbl_entry_internal(tfp, parms);
1654                 if (rc)
1655                         PMD_DRV_LOG(ERR,
1656                                     "dir:%d, Get failed, type:%d, rc:%d\n",
1657                                     parms->dir,
1658                                     parms->type,
1659                                     rc);
1660         }
1661
1662         return rc;
1663 }
1664
1665 /* API defined in tf_core.h */
1666 int
1667 tf_alloc_tbl_scope(struct tf *tfp,
1668                    struct tf_alloc_tbl_scope_parms *parms)
1669 {
1670         int rc;
1671
1672         /* check parameters */
1673         if (parms == NULL || tfp == NULL) {
1674                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1675                 return -EINVAL;
1676         }
1677
1678         rc = tf_alloc_eem_tbl_scope(tfp, parms);
1679
1680         return rc;
1681 }
1682
1683 /* API defined in tf_core.h */
1684 int
1685 tf_free_tbl_scope(struct tf *tfp,
1686                   struct tf_free_tbl_scope_parms *parms)
1687 {
1688         int rc;
1689
1690         /* check parameters */
1691         if (parms == NULL || tfp == NULL) {
1692                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1693                 return -EINVAL;
1694         }
1695
1696         /* free table scope and all associated resources */
1697         rc = tf_free_eem_tbl_scope_cb(tfp, parms);
1698
1699         return rc;
1700 }
1701
1702 /* API defined in tf_core.h */
1703 int
1704 tf_alloc_tbl_entry(struct tf *tfp,
1705                    struct tf_alloc_tbl_entry_parms *parms)
1706 {
1707         int rc;
1708 #if (TF_SHADOW == 1)
1709         struct tf_session *tfs;
1710 #endif /* TF_SHADOW */
1711
1712         /* Check parameters */
1713         if (parms == NULL || tfp == NULL) {
1714                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1715                 return -EINVAL;
1716         }
1717         /*
1718          * No shadow copy support for external tables, allocate and return
1719          */
1720         if (parms->type == TF_TBL_TYPE_EXT) {
1721                 rc = tf_alloc_tbl_entry_pool_external(tfp, parms);
1722                 return rc;
1723         }
1724
1725 #if (TF_SHADOW == 1)
1726         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1727                 PMD_DRV_LOG(ERR,
1728                             "dir:%d, Session info invalid\n",
1729                             parms->dir);
1730                 return -EINVAL;
1731         }
1732
1733         tfs = (struct tf_session *)(tfp->session->core_data);
1734
1735         /* Search the Shadow DB for requested element. If not found go
1736          * allocate one from the Session Pool
1737          */
1738         if (parms->search_enable && tfs->shadow_copy) {
1739                 rc = tf_alloc_tbl_entry_shadow(tfs, parms);
1740                 /* Entry found and parms populated with return data */
1741                 if (rc == 0)
1742                         return rc;
1743         }
1744 #endif /* TF_SHADOW */
1745
1746         rc = tf_alloc_tbl_entry_pool_internal(tfp, parms);
1747         if (rc)
1748                 PMD_DRV_LOG(ERR, "dir%d, Alloc failed, rc:%d\n",
1749                             parms->dir,
1750                             rc);
1751
1752         return rc;
1753 }
1754
1755 /* API defined in tf_core.h */
1756 int
1757 tf_free_tbl_entry(struct tf *tfp,
1758                   struct tf_free_tbl_entry_parms *parms)
1759 {
1760         int rc;
1761 #if (TF_SHADOW == 1)
1762         struct tf_session *tfs;
1763 #endif /* TF_SHADOW */
1764
1765         /* Check parameters */
1766         if (parms == NULL || tfp == NULL) {
1767                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1768                 return -EINVAL;
1769         }
1770         /*
1771          * No shadow of external tables so just free the entry
1772          */
1773         if (parms->type == TF_TBL_TYPE_EXT) {
1774                 rc = tf_free_tbl_entry_pool_external(tfp, parms);
1775                 return rc;
1776         }
1777
1778 #if (TF_SHADOW == 1)
1779         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1780                 PMD_DRV_LOG(ERR,
1781                             "dir:%d, Session info invalid\n",
1782                             parms->dir);
1783                 return -EINVAL;
1784         }
1785
1786         tfs = (struct tf_session *)(tfp->session->core_data);
1787
1788         /* Search the Shadow DB for requested element. If not found go
1789          * allocate one from the Session Pool
1790          */
1791         if (parms->search_enable && tfs->shadow_copy) {
1792                 rc = tf_free_tbl_entry_shadow(tfs, parms);
1793                 /* Entry free'ed and parms populated with return data */
1794                 if (rc == 0)
1795                         return rc;
1796         }
1797 #endif /* TF_SHADOW */
1798
1799         rc = tf_free_tbl_entry_pool_internal(tfp, parms);
1800
1801         if (rc)
1802                 PMD_DRV_LOG(ERR, "dir:%d, Alloc failed, rc:%d\n",
1803                             parms->dir,
1804                             rc);
1805         return rc;
1806 }
1807
1808
1809 static void
1810 tf_dump_link_page_table(struct tf_em_page_tbl *tp,
1811                         struct tf_em_page_tbl *tp_next)
1812 {
1813         uint64_t *pg_va;
1814         uint32_t i;
1815         uint32_t j;
1816         uint32_t k = 0;
1817
1818         printf("pg_count:%d pg_size:0x%x\n",
1819                tp->pg_count,
1820                tp->pg_size);
1821         for (i = 0; i < tp->pg_count; i++) {
1822                 pg_va = tp->pg_va_tbl[i];
1823                 printf("\t%p\n", (void *)pg_va);
1824                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
1825                         printf("\t\t%p\n", (void *)(uintptr_t)pg_va[j]);
1826                         if (((pg_va[j] & 0x7) ==
1827                              tfp_cpu_to_le_64(PTU_PTE_LAST |
1828                                               PTU_PTE_VALID)))
1829                                 return;
1830
1831                         if (!(pg_va[j] & tfp_cpu_to_le_64(PTU_PTE_VALID))) {
1832                                 printf("** Invalid entry **\n");
1833                                 return;
1834                         }
1835
1836                         if (++k >= tp_next->pg_count) {
1837                                 printf("** Shouldn't get here **\n");
1838                                 return;
1839                         }
1840                 }
1841         }
1842 }
1843
1844 void tf_dump_dma(struct tf *tfp, uint32_t tbl_scope_id);
1845
1846 void tf_dump_dma(struct tf *tfp, uint32_t tbl_scope_id)
1847 {
1848         struct tf_session      *session;
1849         struct tf_tbl_scope_cb *tbl_scope_cb;
1850         struct tf_em_page_tbl *tp;
1851         struct tf_em_page_tbl *tp_next;
1852         struct tf_em_table *tbl;
1853         int i;
1854         int j;
1855         int dir;
1856
1857         printf("called %s\n", __func__);
1858
1859         /* find session struct */
1860         session = (struct tf_session *)tfp->session->core_data;
1861
1862         /* find control block for table scope */
1863         tbl_scope_cb = tbl_scope_cb_find(session,
1864                                          tbl_scope_id);
1865         if (tbl_scope_cb == NULL)
1866                 PMD_DRV_LOG(ERR, "No table scope\n");
1867
1868         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1869                 printf("Direction %s:\n", (dir == TF_DIR_RX ? "Rx" : "Tx"));
1870
1871                 for (j = KEY0_TABLE; j < MAX_TABLE; j++) {
1872                         tbl = &tbl_scope_cb->em_ctx_info[dir].em_tables[j];
1873                         printf
1874         ("Table: j:%d type:%d num_entries:%d entry_size:0x%x num_lvl:%d ",
1875                                j,
1876                                tbl->type,
1877                                tbl->num_entries,
1878                                tbl->entry_size,
1879                                tbl->num_lvl);
1880                         if (tbl->pg_tbl[0].pg_va_tbl &&
1881                             tbl->pg_tbl[0].pg_pa_tbl)
1882                                 printf("%p %p\n",
1883                                tbl->pg_tbl[0].pg_va_tbl[0],
1884                                (void *)(uintptr_t)tbl->pg_tbl[0].pg_pa_tbl[0]);
1885                         for (i = 0; i < tbl->num_lvl - 1; i++) {
1886                                 printf("Level:%d\n", i);
1887                                 tp = &tbl->pg_tbl[i];
1888                                 tp_next = &tbl->pg_tbl[i + 1];
1889                                 tf_dump_link_page_table(tp, tp_next);
1890                         }
1891                         printf("\n");
1892                 }
1893         }
1894 }