drivers/net/bnxt/tf_core/tf_em_host.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <math.h>
   8 #include <sys/param.h>
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12
  13 #include "tf_core.h"
  14 #include "tf_util.h"
  15 #include "tf_common.h"
  16 #include "tf_em.h"
  17 #include "tf_em_common.h"
  18 #include "tf_msg.h"
  19 #include "tfp.h"
  20 #include "lookup3.h"
  21 #include "tf_ext_flow_handle.h"
  22
  23 #include "bnxt.h"
  24
  25
  26 #define PTU_PTE_VALID          0x1UL
  27 #define PTU_PTE_LAST           0x2UL
  28 #define PTU_PTE_NEXT_TO_LAST   0x4UL
  29
  30 /* Number of pointers per page_size */
  31 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  32
  33 #define TF_EM_PG_SZ_4K        (1 << 12)
  34 #define TF_EM_PG_SZ_8K        (1 << 13)
  35 #define TF_EM_PG_SZ_64K       (1 << 16)
  36 #define TF_EM_PG_SZ_256K      (1 << 18)
  37 #define TF_EM_PG_SZ_1M        (1 << 20)
  38 #define TF_EM_PG_SZ_2M        (1 << 21)
  39 #define TF_EM_PG_SZ_4M        (1 << 22)
  40 #define TF_EM_PG_SZ_1G        (1 << 30)
  41
  42 #define TF_EM_CTX_ID_INVALID   0xFFFF
  43
  44 #define TF_EM_MIN_ENTRIES     (1 << 15) /* 32K */
  45 #define TF_EM_MAX_ENTRIES     (1 << 27) /* 128M */
  46
  47 /**
  48  * EM DBs.
  49  */
  50 extern void *eem_db[TF_DIR_MAX];
  51
  52 /**
  53  * Function to free a page table
  54  *
  55  * [in] tp
  56  *   Pointer to the page table to free
  57  */
  58 static void
  59 tf_em_free_pg_tbl(struct hcapi_cfa_em_page_tbl *tp)
  60 {
  61         uint32_t i;
  62
  63         for (i = 0; i < tp->pg_count; i++) {
  64                 if (!tp->pg_va_tbl[i]) {
  65                         TFP_DRV_LOG(WARNING,
  66                                     "No mapping for page: %d table: %016" PRIu64 "\n",
  67                                     i,
  68                                     (uint64_t)(uintptr_t)tp);
  69                         continue;
  70                 }
  71
  72                 tfp_free(tp->pg_va_tbl[i]);
  73                 tp->pg_va_tbl[i] = NULL;
  74         }
  75
  76         tp->pg_count = 0;
  77         tfp_free(tp->pg_va_tbl);
  78         tp->pg_va_tbl = NULL;
  79         tfp_free(tp->pg_pa_tbl);
  80         tp->pg_pa_tbl = NULL;
  81 }
  82
  83 /**
  84  * Function to free an EM table
  85  *
  86  * [in] tbl
  87  *   Pointer to the EM table to free
  88  */
  89 static void
  90 tf_em_free_page_table(struct hcapi_cfa_em_table *tbl)
  91 {
  92         struct hcapi_cfa_em_page_tbl *tp;
  93         int i;
  94
  95         for (i = 0; i < tbl->num_lvl; i++) {
  96                 tp = &tbl->pg_tbl[i];
  97                 TFP_DRV_LOG(INFO,
  98                            "EEM: Freeing page table: size %u lvl %d cnt %u\n",
  99                            TF_EM_PAGE_SIZE,
 100                             i,
 101                             tp->pg_count);
 102
 103                 tf_em_free_pg_tbl(tp);
 104         }
 105
 106         tbl->l0_addr = NULL;
 107         tbl->l0_dma_addr = 0;
 108         tbl->num_lvl = 0;
 109         tbl->num_data_pages = 0;
 110 }
 111
 112 /**
 113  * Allocation of page tables
 114  *
 115  * [in] tfp
 116  *   Pointer to a TruFlow handle
 117  *
 118  * [in] pg_count
 119  *   Page count to allocate
 120  *
 121  * [in] pg_size
 122  *   Size of each page
 123  *
 124  * Returns:
 125  *   0       - Success
 126  *   -ENOMEM - Out of memory
 127  */
 128 static int
 129 tf_em_alloc_pg_tbl(struct hcapi_cfa_em_page_tbl *tp,
 130                    uint32_t pg_count,
 131                    uint32_t pg_size)
 132 {
 133         uint32_t i;
 134         struct tfp_calloc_parms parms;
 135
 136         parms.nitems = pg_count;
 137         parms.size = sizeof(void *);
 138         parms.alignment = 0;
 139
 140         if (tfp_calloc(&parms) != 0)
 141                 return -ENOMEM;
 142
 143         tp->pg_va_tbl = parms.mem_va;
 144
 145         if (tfp_calloc(&parms) != 0) {
 146                 tfp_free(tp->pg_va_tbl);
 147                 return -ENOMEM;
 148         }
 149
 150         tp->pg_pa_tbl = parms.mem_va;
 151
 152         tp->pg_count = 0;
 153         tp->pg_size = pg_size;
 154
 155         for (i = 0; i < pg_count; i++) {
 156                 parms.nitems = 1;
 157                 parms.size = pg_size;
 158                 parms.alignment = TF_EM_PAGE_ALIGNMENT;
 159
 160                 if (tfp_calloc(&parms) != 0)
 161                         goto cleanup;
 162
 163                 tp->pg_pa_tbl[i] = (uintptr_t)parms.mem_pa;
 164                 tp->pg_va_tbl[i] = parms.mem_va;
 165
 166                 memset(tp->pg_va_tbl[i], 0, pg_size);
 167                 tp->pg_count++;
 168         }
 169
 170         return 0;
 171
 172 cleanup:
 173         tf_em_free_pg_tbl(tp);
 174         return -ENOMEM;
 175 }
 176
 177 /**
 178  * Allocates EM page tables
 179  *
 180  * [in] tbl
 181  *   Table to allocate pages for
 182  *
 183  * Returns:
 184  *   0       - Success
 185  *   -ENOMEM - Out of memory
 186  */
 187 static int
 188 tf_em_alloc_page_table(struct hcapi_cfa_em_table *tbl)
 189 {
 190         struct hcapi_cfa_em_page_tbl *tp;
 191         int rc = 0;
 192         int i;
 193         uint32_t j;
 194
 195         for (i = 0; i < tbl->num_lvl; i++) {
 196                 tp = &tbl->pg_tbl[i];
 197
 198                 rc = tf_em_alloc_pg_tbl(tp,
 199                                         tbl->page_cnt[i],
 200                                         TF_EM_PAGE_SIZE);
 201                 if (rc) {
 202                         TFP_DRV_LOG(WARNING,
 203                                 "Failed to allocate page table: lvl: %d, rc:%s\n",
 204                                 i,
 205                                 strerror(-rc));
 206                         goto cleanup;
 207                 }
 208
 209                 for (j = 0; j < tp->pg_count; j++) {
 210                         TFP_DRV_LOG(INFO,
 211                                 "EEM: Allocated page table: size %u lvl %d cnt"
 212                                 " %u VA:%p PA:%p\n",
 213                                 TF_EM_PAGE_SIZE,
 214                                 i,
 215                                 tp->pg_count,
 216                                 (void *)(uintptr_t)tp->pg_va_tbl[j],
 217                                 (void *)(uintptr_t)tp->pg_pa_tbl[j]);
 218                 }
 219         }
 220         return rc;
 221
 222 cleanup:
 223         tf_em_free_page_table(tbl);
 224         return rc;
 225 }
 226
 227 /**
 228  * Links EM page tables
 229  *
 230  * [in] tp
 231  *   Pointer to page table
 232  *
 233  * [in] tp_next
 234  *   Pointer to the next page table
 235  *
 236  * [in] set_pte_last
 237  *   Flag controlling if the page table is last
 238  */
 239 static void
 240 tf_em_link_page_table(struct hcapi_cfa_em_page_tbl *tp,
 241                       struct hcapi_cfa_em_page_tbl *tp_next,
 242                       bool set_pte_last)
 243 {
 244         uint64_t *pg_pa = tp_next->pg_pa_tbl;
 245         uint64_t *pg_va;
 246         uint64_t valid;
 247         uint32_t k = 0;
 248         uint32_t i;
 249         uint32_t j;
 250
 251         for (i = 0; i < tp->pg_count; i++) {
 252                 pg_va = tp->pg_va_tbl[i];
 253
 254                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
 255                         if (k == tp_next->pg_count - 2 && set_pte_last)
 256                                 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
 257                         else if (k == tp_next->pg_count - 1 && set_pte_last)
 258                                 valid = PTU_PTE_LAST | PTU_PTE_VALID;
 259                         else
 260                                 valid = PTU_PTE_VALID;
 261
 262                         pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
 263                         if (++k >= tp_next->pg_count)
 264                                 return;
 265                 }
 266         }
 267 }
 268
 269 /**
 270  * Setup a EM page table
 271  *
 272  * [in] tbl
 273  *   Pointer to EM page table
 274  */
 275 static void
 276 tf_em_setup_page_table(struct hcapi_cfa_em_table *tbl)
 277 {
 278         struct hcapi_cfa_em_page_tbl *tp_next;
 279         struct hcapi_cfa_em_page_tbl *tp;
 280         bool set_pte_last = 0;
 281         int i;
 282
 283         for (i = 0; i < tbl->num_lvl - 1; i++) {
 284                 tp = &tbl->pg_tbl[i];
 285                 tp_next = &tbl->pg_tbl[i + 1];
 286                 if (i == tbl->num_lvl - 2)
 287                         set_pte_last = 1;
 288                 tf_em_link_page_table(tp, tp_next, set_pte_last);
 289         }
 290
 291         tbl->l0_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_va_tbl[0];
 292         tbl->l0_dma_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_pa_tbl[0];
 293 }
 294
 295 /**
 296  * Given the page size, size of each data item (entry size),
 297  * and the total number of entries needed, determine the number
 298  * of page table levels and the number of data pages required.
 299  *
 300  * [in] page_size
 301  *   Page size
 302  *
 303  * [in] entry_size
 304  *   Entry size
 305  *
 306  * [in] num_entries
 307  *   Number of entries needed
 308  *
 309  * [out] num_data_pages
 310  *   Number of pages required
 311  *
 312  * Returns:
 313  *   Success  - Number of EM page levels required
 314  *   -ENOMEM  - Out of memory
 315  */
 316 static int
 317 tf_em_size_page_tbl_lvl(uint32_t page_size,
 318                         uint32_t entry_size,
 319                         uint32_t num_entries,
 320                         uint64_t *num_data_pages)
 321 {
 322         uint64_t lvl_data_size = page_size;
 323         int lvl = TF_PT_LVL_0;
 324         uint64_t data_size;
 325
 326         *num_data_pages = 0;
 327         data_size = (uint64_t)num_entries * entry_size;
 328
 329         while (lvl_data_size < data_size) {
 330                 lvl++;
 331
 332                 if (lvl == TF_PT_LVL_1)
 333                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 334                                 page_size;
 335                 else if (lvl == TF_PT_LVL_2)
 336                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 337                                 MAX_PAGE_PTRS(page_size) * page_size;
 338                 else
 339                         return -ENOMEM;
 340         }
 341
 342         *num_data_pages = roundup(data_size, page_size) / page_size;
 343
 344         return lvl;
 345 }
 346
 347 /**
 348  * Return the number of page table pages needed to
 349  * reference the given number of next level pages.
 350  *
 351  * [in] num_pages
 352  *   Number of EM pages
 353  *
 354  * [in] page_size
 355  *   Size of each EM page
 356  *
 357  * Returns:
 358  *   Number of EM page table pages
 359  */
 360 static uint32_t
 361 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 362                      uint32_t page_size)
 363 {
 364         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 365                        MAX_PAGE_PTRS(page_size);
 366         return 0;
 367 }
 368
 369 /**
 370  * Given the number of data pages, page_size and the maximum
 371  * number of page table levels (already determined), size
 372  * the number of page table pages required at each level.
 373  *
 374  * [in] max_lvl
 375  *   Max number of levels
 376  *
 377  * [in] num_data_pages
 378  *   Number of EM data pages
 379  *
 380  * [in] page_size
 381  *   Size of an EM page
 382  *
 383  * [out] *page_cnt
 384  *   EM page count
 385  */
 386 static void
 387 tf_em_size_page_tbls(int max_lvl,
 388                      uint64_t num_data_pages,
 389                      uint32_t page_size,
 390                      uint32_t *page_cnt)
 391 {
 392         if (max_lvl == TF_PT_LVL_0) {
 393                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 394         } else if (max_lvl == TF_PT_LVL_1) {
 395                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 396                 page_cnt[TF_PT_LVL_0] =
 397                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 398         } else if (max_lvl == TF_PT_LVL_2) {
 399                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 400                 page_cnt[TF_PT_LVL_1] =
 401                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 402                 page_cnt[TF_PT_LVL_0] =
 403                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 404         } else {
 405                 return;
 406         }
 407 }
 408
 409 /**
 410  * Size the EM table based on capabilities
 411  *
 412  * [in] tbl
 413  *   EM table to size
 414  *
 415  * Returns:
 416  *   0        - Success
 417  *   - EINVAL - Parameter error
 418  *   - ENOMEM - Out of memory
 419  */
 420 static int
 421 tf_em_size_table(struct hcapi_cfa_em_table *tbl)
 422 {
 423         uint64_t num_data_pages;
 424         uint32_t *page_cnt;
 425         int max_lvl;
 426         uint32_t num_entries;
 427         uint32_t cnt = TF_EM_MIN_ENTRIES;
 428
 429         /* Ignore entry if both size and number are zero */
 430         if (!tbl->entry_size && !tbl->num_entries)
 431                 return 0;
 432
 433         /* If only one is set then error */
 434         if (!tbl->entry_size || !tbl->num_entries)
 435                 return -EINVAL;
 436
 437         /* Determine number of page table levels and the number
 438          * of data pages needed to process the given eem table.
 439          */
 440         if (tbl->type == TF_RECORD_TABLE) {
 441                 /*
 442                  * For action records just a memory size is provided. Work
 443                  * backwards to resolve to number of entries
 444                  */
 445                 num_entries = tbl->num_entries / tbl->entry_size;
 446                 if (num_entries < TF_EM_MIN_ENTRIES) {
 447                         num_entries = TF_EM_MIN_ENTRIES;
 448                 } else {
 449                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 450                                 cnt *= 2;
 451                         num_entries = cnt;
 452                 }
 453         } else {
 454                 num_entries = tbl->num_entries;
 455         }
 456
 457         max_lvl = tf_em_size_page_tbl_lvl(TF_EM_PAGE_SIZE,
 458                                           tbl->entry_size,
 459                                           tbl->num_entries,
 460                                           &num_data_pages);
 461         if (max_lvl < 0) {
 462                 TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 463                 TFP_DRV_LOG(WARNING,
 464                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 465                             tbl->type, (uint64_t)num_entries * tbl->entry_size,
 466                             TF_EM_PAGE_SIZE);
 467                 return -ENOMEM;
 468         }
 469
 470         tbl->num_lvl = max_lvl + 1;
 471         tbl->num_data_pages = num_data_pages;
 472
 473         /* Determine the number of pages needed at each level */
 474         page_cnt = tbl->page_cnt;
 475         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 476         tf_em_size_page_tbls(max_lvl, num_data_pages, TF_EM_PAGE_SIZE,
 477                                 page_cnt);
 478
 479         TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 480         TFP_DRV_LOG(INFO,
 481                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 " l0: %u l1: %u l2: %u\n",
 482                     max_lvl + 1,
 483                     (uint64_t)num_data_pages * TF_EM_PAGE_SIZE,
 484                     num_data_pages,
 485                     page_cnt[TF_PT_LVL_0],
 486                     page_cnt[TF_PT_LVL_1],
 487                     page_cnt[TF_PT_LVL_2]);
 488
 489         return 0;
 490 }
 491
 492 /**
 493  * Unregisters EM Ctx in Firmware
 494  *
 495  * [in] tfp
 496  *   Pointer to a TruFlow handle
 497  *
 498  * [in] tbl_scope_cb
 499  *   Pointer to a table scope control block
 500  *
 501  * [in] dir
 502  *   Receive or transmit direction
 503  */
 504 static void
 505 tf_em_ctx_unreg(struct tf *tfp,
 506                 struct tf_tbl_scope_cb *tbl_scope_cb,
 507                 int dir)
 508 {
 509         struct hcapi_cfa_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 510         struct hcapi_cfa_em_table *tbl;
 511         int i;
 512
 513         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 514                 tbl = &ctxp->em_tables[i];
 515
 516                 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
 517                         tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
 518                         tf_em_free_page_table(tbl);
 519                 }
 520         }
 521 }
 522
 523 /**
 524  * Registers EM Ctx in Firmware
 525  *
 526  * [in] tfp
 527  *   Pointer to a TruFlow handle
 528  *
 529  * [in] tbl_scope_cb
 530  *   Pointer to a table scope control block
 531  *
 532  * [in] dir
 533  *   Receive or transmit direction
 534  *
 535  * Returns:
 536  *   0       - Success
 537  *   -ENOMEM - Out of Memory
 538  */
 539 static int
 540 tf_em_ctx_reg(struct tf *tfp,
 541               struct tf_tbl_scope_cb *tbl_scope_cb,
 542               int dir)
 543 {
 544         struct hcapi_cfa_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 545         struct hcapi_cfa_em_table *tbl;
 546         int rc;
 547         int i;
 548
 549         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 550                 tbl = &ctxp->em_tables[i];
 551
 552                 if (tbl->num_entries && tbl->entry_size) {
 553                         rc = tf_em_size_table(tbl);
 554
 555                         if (rc)
 556                                 goto cleanup;
 557
 558                         rc = tf_em_alloc_page_table(tbl);
 559                         if (rc)
 560                                 goto cleanup;
 561
 562                         tf_em_setup_page_table(tbl);
 563                         rc = tf_msg_em_mem_rgtr(tfp,
 564                                                 tbl->num_lvl - 1,
 565                                                 TF_EM_PAGE_SIZE_ENUM,
 566                                                 tbl->l0_dma_addr,
 567                                                 &tbl->ctx_id);
 568                         if (rc)
 569                                 goto cleanup;
 570                 }
 571         }
 572         return rc;
 573
 574 cleanup:
 575         tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
 576         return rc;
 577 }
 578
 579
 580 /**
 581  * Validates EM number of entries requested
 582  *
 583  * [in] tbl_scope_cb
 584  *   Pointer to table scope control block to be populated
 585  *
 586  * [in] parms
 587  *   Pointer to input parameters
 588  *
 589  * Returns:
 590  *   0       - Success
 591  *   -EINVAL - Parameter error
 592  */
 593 static int
 594 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 595                            struct tf_alloc_tbl_scope_parms *parms)
 596 {
 597         uint32_t cnt;
 598
 599         if (parms->rx_mem_size_in_mb != 0) {
 600                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 601                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 602                                      + 1);
 603                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 604                                         TF_MEGABYTE) / (key_b + action_b);
 605
 606                 if (num_entries < TF_EM_MIN_ENTRIES) {
 607                         TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 608                                     "%uMB\n",
 609                                     parms->rx_mem_size_in_mb);
 610                         return -EINVAL;
 611                 }
 612
 613                 cnt = TF_EM_MIN_ENTRIES;
 614                 while (num_entries > cnt &&
 615                        cnt <= TF_EM_MAX_ENTRIES)
 616                         cnt *= 2;
 617
 618                 if (cnt > TF_EM_MAX_ENTRIES) {
 619                         TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 620                                     "%u\n",
 621                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 622                         return -EINVAL;
 623                 }
 624
 625                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 626         } else {
 627                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 628                     TF_EM_MIN_ENTRIES ||
 629                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 630                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 631                         TFP_DRV_LOG(ERR,
 632                                     "EEM: Invalid number of Rx flows "
 633                                     "requested:%u max:%u\n",
 634                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 635                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 636                         return -EINVAL;
 637                 }
 638
 639                 /* must be a power-of-2 supported value
 640                  * in the range 32K - 128M
 641                  */
 642                 cnt = TF_EM_MIN_ENTRIES;
 643                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 644                        cnt <= TF_EM_MAX_ENTRIES)
 645                         cnt *= 2;
 646
 647                 if (cnt > TF_EM_MAX_ENTRIES) {
 648                         TFP_DRV_LOG(ERR,
 649                                     "EEM: Invalid number of Rx requested: %u\n",
 650                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 651                         return -EINVAL;
 652                 }
 653         }
 654
 655         if (parms->tx_mem_size_in_mb != 0) {
 656                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 657                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 658                                      + 1);
 659                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 660                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 661                         (key_b + action_b);
 662
 663                 if (num_entries < TF_EM_MIN_ENTRIES) {
 664                         TFP_DRV_LOG(ERR,
 665                                     "EEM: Insufficient memory requested:%uMB\n",
 666                                     parms->rx_mem_size_in_mb);
 667                         return -EINVAL;
 668                 }
 669
 670                 cnt = TF_EM_MIN_ENTRIES;
 671                 while (num_entries > cnt &&
 672                        cnt <= TF_EM_MAX_ENTRIES)
 673                         cnt *= 2;
 674
 675                 if (cnt > TF_EM_MAX_ENTRIES) {
 676                         TFP_DRV_LOG(ERR,
 677                                     "EEM: Invalid number of Tx requested: %u\n",
 678                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 679                         return -EINVAL;
 680                 }
 681
 682                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 683         } else {
 684                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 685                     TF_EM_MIN_ENTRIES ||
 686                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 687                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 688                         TFP_DRV_LOG(ERR,
 689                                     "EEM: Invalid number of Tx flows "
 690                                     "requested:%u max:%u\n",
 691                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 692                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 693                         return -EINVAL;
 694                 }
 695
 696                 cnt = TF_EM_MIN_ENTRIES;
 697                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 698                        cnt <= TF_EM_MAX_ENTRIES)
 699                         cnt *= 2;
 700
 701                 if (cnt > TF_EM_MAX_ENTRIES) {
 702                         TFP_DRV_LOG(ERR,
 703                                     "EEM: Invalid number of Tx requested: %u\n",
 704                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 705                         return -EINVAL;
 706                 }
 707         }
 708
 709         if (parms->rx_num_flows_in_k != 0 &&
 710             (parms->rx_max_key_sz_in_bits / 8 == 0)) {
 711                 TFP_DRV_LOG(ERR,
 712                             "EEM: Rx key size required: %u\n",
 713                             (parms->rx_max_key_sz_in_bits));
 714                 return -EINVAL;
 715         }
 716
 717         if (parms->tx_num_flows_in_k != 0 &&
 718             (parms->tx_max_key_sz_in_bits / 8 == 0)) {
 719                 TFP_DRV_LOG(ERR,
 720                             "EEM: Tx key size required: %u\n",
 721                             (parms->tx_max_key_sz_in_bits));
 722                 return -EINVAL;
 723         }
 724         /* Rx */
 725         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 726                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 727         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 728                 parms->rx_max_key_sz_in_bits / 8;
 729
 730         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 731                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 732         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 733                 parms->rx_max_key_sz_in_bits / 8;
 734
 735         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 736                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 737         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 738                 parms->rx_max_action_entry_sz_in_bits / 8;
 739
 740         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries = 0;
 741
 742         /* Tx */
 743         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 744                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 745         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 746                 parms->tx_max_key_sz_in_bits / 8;
 747
 748         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 749                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 750         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 751                 parms->tx_max_key_sz_in_bits / 8;
 752
 753         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 754                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 755         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 756                 parms->tx_max_action_entry_sz_in_bits / 8;
 757
 758         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries = 0;
 759
 760         return 0;
 761 }
 762
 763 /** insert EEM entry API
 764  *
 765  * returns:
 766  *  0
 767  *  TF_ERR          - unable to get lock
 768  *
 769  * insert callback returns:
 770  *   0
 771  *   TF_ERR_EM_DUP  - key is already in table
 772  */
 773 static int
 774 tf_insert_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 775                     struct tf_insert_em_entry_parms *parms)
 776 {
 777         uint32_t mask;
 778         uint32_t key0_hash;
 779         uint32_t key1_hash;
 780         uint32_t key0_index;
 781         uint32_t key1_index;
 782         struct cfa_p4_eem_64b_entry key_entry;
 783         uint32_t index;
 784         enum hcapi_cfa_em_table_type table_type;
 785         uint32_t gfid;
 786         struct hcapi_cfa_hwop op;
 787         struct hcapi_cfa_key_tbl key_tbl;
 788         struct hcapi_cfa_key_data key_obj;
 789         struct hcapi_cfa_key_loc key_loc;
 790         uint64_t big_hash;
 791         int rc;
 792
 793         /* Get mask to use on hash */
 794         mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
 795
 796         if (!mask)
 797                 return -EINVAL;
 798
 799 #ifdef TF_EEM_DEBUG
 800         dump_raw((uint8_t *)parms->key, TF_HW_EM_KEY_MAX_SIZE + 4, "In Key");
 801 #endif
 802
 803         big_hash = hcapi_cfa_key_hash((uint64_t *)parms->key,
 804                                       (TF_HW_EM_KEY_MAX_SIZE + 4) * 8);
 805         key0_hash = (uint32_t)(big_hash >> 32);
 806         key1_hash = (uint32_t)(big_hash & 0xFFFFFFFF);
 807
 808         key0_index = key0_hash & mask;
 809         key1_index = key1_hash & mask;
 810
 811 #ifdef TF_EEM_DEBUG
 812         TFP_DRV_LOG(DEBUG, "Key0 hash:0x%08x\n", key0_hash);
 813         TFP_DRV_LOG(DEBUG, "Key1 hash:0x%08x\n", key1_hash);
 814 #endif
 815         /*
 816          * Use the "result" arg to populate all of the key entry then
 817          * store the byte swapped "raw" entry in a local copy ready
 818          * for insertion in to the table.
 819          */
 820         tf_em_create_key_entry((struct cfa_p4_eem_entry_hdr *)parms->em_record,
 821                                 ((uint8_t *)parms->key),
 822                                 &key_entry);
 823
 824         /*
 825          * Try to add to Key0 table, if that does not work then
 826          * try the key1 table.
 827          */
 828         index = key0_index;
 829         op.opcode = HCAPI_CFA_HWOPS_ADD;
 830         key_tbl.base0 = (uint8_t *)
 831                 &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE];
 832         key_obj.offset = (index * TF_EM_KEY_RECORD_SIZE) % TF_EM_PAGE_SIZE;
 833         key_obj.data = (uint8_t *)&key_entry;
 834         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 835
 836         rc = hcapi_cfa_key_hw_op(&op,
 837                                  &key_tbl,
 838                                  &key_obj,
 839                                  &key_loc);
 840
 841         if (rc == 0) {
 842                 table_type = TF_KEY0_TABLE;
 843         } else {
 844                 index = key1_index;
 845
 846                 key_tbl.base0 = (uint8_t *)
 847                 &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY1_TABLE];
 848                 key_obj.offset =
 849                         (index * TF_EM_KEY_RECORD_SIZE) % TF_EM_PAGE_SIZE;
 850
 851                 rc = hcapi_cfa_key_hw_op(&op,
 852                                          &key_tbl,
 853                                          &key_obj,
 854                                          &key_loc);
 855                 if (rc != 0)
 856                         return rc;
 857
 858                 table_type = TF_KEY1_TABLE;
 859         }
 860
 861         TF_SET_GFID(gfid,
 862                     index,
 863                     table_type);
 864         TF_SET_FLOW_ID(parms->flow_id,
 865                        gfid,
 866                        TF_GFID_TABLE_EXTERNAL,
 867                        parms->dir);
 868         TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
 869                                      0,
 870                                      0,
 871                                      0,
 872                                      index,
 873                                      0,
 874                                      table_type);
 875
 876         return 0;
 877 }
 878
 879 /** delete EEM hash entry API
 880  *
 881  * returns:
 882  *   0
 883  *   -EINVAL      - parameter error
 884  *   TF_NO_SESSION    - bad session ID
 885  *   TF_ERR_TBL_SCOPE - invalid table scope
 886  *   TF_ERR_TBL_IF    - invalid table interface
 887  *
 888  * insert callback returns
 889  *   0
 890  *   TF_NO_EM_MATCH - entry not found
 891  */
 892 static int
 893 tf_delete_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 894                     struct tf_delete_em_entry_parms *parms)
 895 {
 896         enum hcapi_cfa_em_table_type hash_type;
 897         uint32_t index;
 898         struct hcapi_cfa_hwop op;
 899         struct hcapi_cfa_key_tbl key_tbl;
 900         struct hcapi_cfa_key_data key_obj;
 901         struct hcapi_cfa_key_loc key_loc;
 902         int rc;
 903
 904         if (parms->flow_handle == 0)
 905                 return -EINVAL;
 906
 907         TF_GET_HASH_TYPE_FROM_FLOW_HANDLE(parms->flow_handle, hash_type);
 908         TF_GET_INDEX_FROM_FLOW_HANDLE(parms->flow_handle, index);
 909
 910         op.opcode = HCAPI_CFA_HWOPS_DEL;
 911         key_tbl.base0 = (uint8_t *)
 912         &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[(hash_type == 0 ?
 913                                                           TF_KEY0_TABLE :
 914                                                           TF_KEY1_TABLE)];
 915         key_obj.offset = (index * TF_EM_KEY_RECORD_SIZE) % TF_EM_PAGE_SIZE;
 916         key_obj.data = NULL;
 917         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 918
 919         rc = hcapi_cfa_key_hw_op(&op,
 920                                  &key_tbl,
 921                                  &key_obj,
 922                                  &key_loc);
 923
 924         if (!rc)
 925                 return rc;
 926
 927         return 0;
 928 }
 929
 930 /** insert EM hash entry API
 931  *
 932  *    returns:
 933  *    0       - Success
 934  *    -EINVAL - Error
 935  */
 936 int
 937 tf_em_insert_ext_entry(struct tf *tfp,
 938                        struct tf_insert_em_entry_parms *parms)
 939 {
 940         struct tf_tbl_scope_cb *tbl_scope_cb;
 941
 942         tbl_scope_cb =
 943         tbl_scope_cb_find((struct tf_session *)(tfp->session->core_data),
 944                           parms->tbl_scope_id);
 945         if (tbl_scope_cb == NULL) {
 946                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 947                 return -EINVAL;
 948         }
 949
 950         return tf_insert_eem_entry(tbl_scope_cb, parms);
 951 }
 952
 953 /** Delete EM hash entry API
 954  *
 955  *    returns:
 956  *    0       - Success
 957  *    -EINVAL - Error
 958  */
 959 int
 960 tf_em_delete_ext_entry(struct tf *tfp,
 961                        struct tf_delete_em_entry_parms *parms)
 962 {
 963         struct tf_tbl_scope_cb *tbl_scope_cb;
 964
 965         tbl_scope_cb =
 966         tbl_scope_cb_find((struct tf_session *)(tfp->session->core_data),
 967                           parms->tbl_scope_id);
 968         if (tbl_scope_cb == NULL) {
 969                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 970                 return -EINVAL;
 971         }
 972
 973         return tf_delete_eem_entry(tbl_scope_cb, parms);
 974 }
 975
 976 int
 977 tf_em_ext_host_alloc(struct tf *tfp,
 978                      struct tf_alloc_tbl_scope_parms *parms)
 979 {
 980         int rc;
 981         enum tf_dir dir;
 982         struct tf_tbl_scope_cb *tbl_scope_cb;
 983         struct hcapi_cfa_em_table *em_tables;
 984         struct tf_session *session;
 985         struct tf_free_tbl_scope_parms free_parms;
 986         struct tf_rm_allocate_parms aparms = { 0 };
 987         struct tf_rm_free_parms fparms = { 0 };
 988
 989         session = (struct tf_session *)tfp->session->core_data;
 990
 991         /* Get Table Scope control block from the session pool */
 992         aparms.rm_db = eem_db[TF_DIR_RX];
 993         aparms.db_index = 1/**** TYPE TABLE-SCOPE??? ****/;
 994         aparms.index = (uint32_t *)&parms->tbl_scope_id;
 995         rc = tf_rm_allocate(&aparms);
 996         if (rc) {
 997                 TFP_DRV_LOG(ERR,
 998                             "Failed to allocate table scope\n");
 999                 return rc;
1000         }
1001
1002         parms->tbl_scope_id -= TF_HACK_TBL_SCOPE_BASE;
1003         tbl_scope_cb = &session->tbl_scopes[parms->tbl_scope_id];
1004         tbl_scope_cb->index = parms->tbl_scope_id;
1005         tbl_scope_cb->tbl_scope_id = parms->tbl_scope_id;
1006
1007         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1008                 rc = tf_msg_em_qcaps(tfp,
1009                                      dir,
1010                                      &tbl_scope_cb->em_caps[dir]);
1011                 if (rc) {
1012                         TFP_DRV_LOG(ERR,
1013                                     "EEM: Unable to query for EEM capability,"
1014                                     " rc:%s\n",
1015                                     strerror(-rc));
1016                         goto cleanup;
1017                 }
1018         }
1019
1020         /*
1021          * Validate and setup table sizes
1022          */
1023         if (tf_em_validate_num_entries(tbl_scope_cb, parms))
1024                 goto cleanup;
1025
1026         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1027                 /*
1028                  * Allocate tables and signal configuration to FW
1029                  */
1030                 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
1031                 if (rc) {
1032                         TFP_DRV_LOG(ERR,
1033                                     "EEM: Unable to register for EEM ctx,"
1034                                     " rc:%s\n",
1035                                     strerror(-rc));
1036                         goto cleanup;
1037                 }
1038
1039                 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
1040                 rc = tf_msg_em_cfg(tfp,
1041                                    em_tables[TF_KEY0_TABLE].num_entries,
1042                                    em_tables[TF_KEY0_TABLE].ctx_id,
1043                                    em_tables[TF_KEY1_TABLE].ctx_id,
1044                                    em_tables[TF_RECORD_TABLE].ctx_id,
1045                                    em_tables[TF_EFC_TABLE].ctx_id,
1046                                    parms->hw_flow_cache_flush_timer,
1047                                    dir);
1048                 if (rc) {
1049                         TFP_DRV_LOG(ERR,
1050                                     "TBL: Unable to configure EEM in firmware"
1051                                     " rc:%s\n",
1052                                     strerror(-rc));
1053                         goto cleanup_full;
1054                 }
1055
1056                 rc = tf_msg_em_op(tfp,
1057                                   dir,
1058                                   HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
1059
1060                 if (rc) {
1061                         TFP_DRV_LOG(ERR,
1062                                     "EEM: Unable to enable EEM in firmware"
1063                                     " rc:%s\n",
1064                                     strerror(-rc));
1065                         goto cleanup_full;
1066                 }
1067
1068                 /* Allocate the pool of offsets of the external memory.
1069                  * Initially, this is a single fixed size pool for all external
1070                  * actions related to a single table scope.
1071                  */
1072                 rc = tf_create_tbl_pool_external(dir,
1073                                             tbl_scope_cb,
1074                                             em_tables[TF_RECORD_TABLE].num_entries,
1075                                             em_tables[TF_RECORD_TABLE].entry_size);
1076                 if (rc) {
1077                         TFP_DRV_LOG(ERR,
1078                                     "%s TBL: Unable to allocate idx pools %s\n",
1079                                     tf_dir_2_str(dir),
1080                                     strerror(-rc));
1081                         goto cleanup_full;
1082                 }
1083         }
1084
1085         return 0;
1086
1087 cleanup_full:
1088         free_parms.tbl_scope_id = parms->tbl_scope_id;
1089         tf_em_ext_host_free(tfp, &free_parms);
1090         return -EINVAL;
1091
1092 cleanup:
1093         /* Free Table control block */
1094         fparms.rm_db = eem_db[TF_DIR_RX];
1095         fparms.db_index = 1/**** TYPE TABLE-SCOPE??? ****/;
1096         fparms.index = parms->tbl_scope_id + TF_HACK_TBL_SCOPE_BASE;
1097         tf_rm_free(&fparms);
1098         return -EINVAL;
1099 }
1100
1101 int
1102 tf_em_ext_host_free(struct tf *tfp,
1103                     struct tf_free_tbl_scope_parms *parms)
1104 {
1105         int rc = 0;
1106         enum tf_dir  dir;
1107         struct tf_tbl_scope_cb *tbl_scope_cb;
1108         struct tf_session *session;
1109         struct tf_rm_free_parms aparms = { 0 };
1110
1111         session = (struct tf_session *)(tfp->session->core_data);
1112
1113         tbl_scope_cb = tbl_scope_cb_find(session,
1114                                          parms->tbl_scope_id);
1115
1116         if (tbl_scope_cb == NULL) {
1117                 TFP_DRV_LOG(ERR, "Table scope error\n");
1118                 return -EINVAL;
1119         }
1120
1121         /* Free Table control block */
1122         aparms.rm_db = eem_db[TF_DIR_RX];
1123         aparms.db_index = 1/**** TYPE TABLE-SCOPE??? ****/;
1124         aparms.index = parms->tbl_scope_id + TF_HACK_TBL_SCOPE_BASE;
1125         rc = tf_rm_free(&aparms);
1126         if (rc) {
1127                 TFP_DRV_LOG(ERR,
1128                             "Failed to free table scope\n");
1129         }
1130
1131         /* free table scope locks */
1132         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1133                 /* Free associated external pools
1134                  */
1135                 tf_destroy_tbl_pool_external(dir,
1136                                              tbl_scope_cb);
1137                 tf_msg_em_op(tfp,
1138                              dir,
1139                              HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
1140
1141                 /* free table scope and all associated resources */
1142                 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
1143         }
1144
1145         return rc;
1146 }