drivers/net/bnxt/tf_core/tf_em_host.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <math.h>
   8 #include <sys/param.h>
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12
  13 #include "tf_core.h"
  14 #include "tf_util.h"
  15 #include "tf_common.h"
  16 #include "tf_em.h"
  17 #include "tf_em_common.h"
  18 #include "tf_msg.h"
  19 #include "tfp.h"
  20 #include "lookup3.h"
  21 #include "tf_ext_flow_handle.h"
  22
  23 #include "bnxt.h"
  24
  25
  26 #define PTU_PTE_VALID          0x1UL
  27 #define PTU_PTE_LAST           0x2UL
  28 #define PTU_PTE_NEXT_TO_LAST   0x4UL
  29
  30 /* Number of pointers per page_size */
  31 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  32
  33 #define TF_EM_PG_SZ_4K        (1 << 12)
  34 #define TF_EM_PG_SZ_8K        (1 << 13)
  35 #define TF_EM_PG_SZ_64K       (1 << 16)
  36 #define TF_EM_PG_SZ_256K      (1 << 18)
  37 #define TF_EM_PG_SZ_1M        (1 << 20)
  38 #define TF_EM_PG_SZ_2M        (1 << 21)
  39 #define TF_EM_PG_SZ_4M        (1 << 22)
  40 #define TF_EM_PG_SZ_1G        (1 << 30)
  41
  42 #define TF_EM_CTX_ID_INVALID   0xFFFF
  43
  44 #define TF_EM_MIN_ENTRIES     (1 << 15) /* 32K */
  45 #define TF_EM_MAX_ENTRIES     (1 << 27) /* 128M */
  46
  47 /**
  48  * EM DBs.
  49  */
  50 extern void *eem_db[TF_DIR_MAX];
  51
  52 extern struct tf_tbl_scope_cb tbl_scopes[TF_NUM_TBL_SCOPE];
  53
  54 /**
  55  * Function to free a page table
  56  *
  57  * [in] tp
  58  *   Pointer to the page table to free
  59  */
  60 static void
  61 tf_em_free_pg_tbl(struct hcapi_cfa_em_page_tbl *tp)
  62 {
  63         uint32_t i;
  64
  65         for (i = 0; i < tp->pg_count; i++) {
  66                 if (!tp->pg_va_tbl[i]) {
  67                         TFP_DRV_LOG(WARNING,
  68                                     "No mapping for page: %d table: %016" PRIu64 "\n",
  69                                     i,
  70                                     (uint64_t)(uintptr_t)tp);
  71                         continue;
  72                 }
  73
  74                 tfp_free(tp->pg_va_tbl[i]);
  75                 tp->pg_va_tbl[i] = NULL;
  76         }
  77
  78         tp->pg_count = 0;
  79         tfp_free(tp->pg_va_tbl);
  80         tp->pg_va_tbl = NULL;
  81         tfp_free(tp->pg_pa_tbl);
  82         tp->pg_pa_tbl = NULL;
  83 }
  84
  85 /**
  86  * Function to free an EM table
  87  *
  88  * [in] tbl
  89  *   Pointer to the EM table to free
  90  */
  91 static void
  92 tf_em_free_page_table(struct hcapi_cfa_em_table *tbl)
  93 {
  94         struct hcapi_cfa_em_page_tbl *tp;
  95         int i;
  96
  97         for (i = 0; i < tbl->num_lvl; i++) {
  98                 tp = &tbl->pg_tbl[i];
  99                 TFP_DRV_LOG(INFO,
 100                            "EEM: Freeing page table: size %u lvl %d cnt %u\n",
 101                            TF_EM_PAGE_SIZE,
 102                             i,
 103                             tp->pg_count);
 104
 105                 tf_em_free_pg_tbl(tp);
 106         }
 107
 108         tbl->l0_addr = NULL;
 109         tbl->l0_dma_addr = 0;
 110         tbl->num_lvl = 0;
 111         tbl->num_data_pages = 0;
 112 }
 113
 114 /**
 115  * Allocation of page tables
 116  *
 117  * [in] tfp
 118  *   Pointer to a TruFlow handle
 119  *
 120  * [in] pg_count
 121  *   Page count to allocate
 122  *
 123  * [in] pg_size
 124  *   Size of each page
 125  *
 126  * Returns:
 127  *   0       - Success
 128  *   -ENOMEM - Out of memory
 129  */
 130 static int
 131 tf_em_alloc_pg_tbl(struct hcapi_cfa_em_page_tbl *tp,
 132                    uint32_t pg_count,
 133                    uint32_t pg_size)
 134 {
 135         uint32_t i;
 136         struct tfp_calloc_parms parms;
 137
 138         parms.nitems = pg_count;
 139         parms.size = sizeof(void *);
 140         parms.alignment = 0;
 141
 142         if (tfp_calloc(&parms) != 0)
 143                 return -ENOMEM;
 144
 145         tp->pg_va_tbl = parms.mem_va;
 146
 147         if (tfp_calloc(&parms) != 0) {
 148                 tfp_free(tp->pg_va_tbl);
 149                 return -ENOMEM;
 150         }
 151
 152         tp->pg_pa_tbl = parms.mem_va;
 153
 154         tp->pg_count = 0;
 155         tp->pg_size = pg_size;
 156
 157         for (i = 0; i < pg_count; i++) {
 158                 parms.nitems = 1;
 159                 parms.size = pg_size;
 160                 parms.alignment = TF_EM_PAGE_ALIGNMENT;
 161
 162                 if (tfp_calloc(&parms) != 0)
 163                         goto cleanup;
 164
 165                 tp->pg_pa_tbl[i] = (uintptr_t)parms.mem_pa;
 166                 tp->pg_va_tbl[i] = parms.mem_va;
 167
 168                 memset(tp->pg_va_tbl[i], 0, pg_size);
 169                 tp->pg_count++;
 170         }
 171
 172         return 0;
 173
 174 cleanup:
 175         tf_em_free_pg_tbl(tp);
 176         return -ENOMEM;
 177 }
 178
 179 /**
 180  * Allocates EM page tables
 181  *
 182  * [in] tbl
 183  *   Table to allocate pages for
 184  *
 185  * Returns:
 186  *   0       - Success
 187  *   -ENOMEM - Out of memory
 188  */
 189 static int
 190 tf_em_alloc_page_table(struct hcapi_cfa_em_table *tbl)
 191 {
 192         struct hcapi_cfa_em_page_tbl *tp;
 193         int rc = 0;
 194         int i;
 195         uint32_t j;
 196
 197         for (i = 0; i < tbl->num_lvl; i++) {
 198                 tp = &tbl->pg_tbl[i];
 199
 200                 rc = tf_em_alloc_pg_tbl(tp,
 201                                         tbl->page_cnt[i],
 202                                         TF_EM_PAGE_SIZE);
 203                 if (rc) {
 204                         TFP_DRV_LOG(WARNING,
 205                                 "Failed to allocate page table: lvl: %d, rc:%s\n",
 206                                 i,
 207                                 strerror(-rc));
 208                         goto cleanup;
 209                 }
 210
 211                 for (j = 0; j < tp->pg_count; j++) {
 212                         TFP_DRV_LOG(INFO,
 213                                 "EEM: Allocated page table: size %u lvl %d cnt"
 214                                 " %u VA:%p PA:%p\n",
 215                                 TF_EM_PAGE_SIZE,
 216                                 i,
 217                                 tp->pg_count,
 218                                 (void *)(uintptr_t)tp->pg_va_tbl[j],
 219                                 (void *)(uintptr_t)tp->pg_pa_tbl[j]);
 220                 }
 221         }
 222         return rc;
 223
 224 cleanup:
 225         tf_em_free_page_table(tbl);
 226         return rc;
 227 }
 228
 229 /**
 230  * Links EM page tables
 231  *
 232  * [in] tp
 233  *   Pointer to page table
 234  *
 235  * [in] tp_next
 236  *   Pointer to the next page table
 237  *
 238  * [in] set_pte_last
 239  *   Flag controlling if the page table is last
 240  */
 241 static void
 242 tf_em_link_page_table(struct hcapi_cfa_em_page_tbl *tp,
 243                       struct hcapi_cfa_em_page_tbl *tp_next,
 244                       bool set_pte_last)
 245 {
 246         uint64_t *pg_pa = tp_next->pg_pa_tbl;
 247         uint64_t *pg_va;
 248         uint64_t valid;
 249         uint32_t k = 0;
 250         uint32_t i;
 251         uint32_t j;
 252
 253         for (i = 0; i < tp->pg_count; i++) {
 254                 pg_va = tp->pg_va_tbl[i];
 255
 256                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
 257                         if (k == tp_next->pg_count - 2 && set_pte_last)
 258                                 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
 259                         else if (k == tp_next->pg_count - 1 && set_pte_last)
 260                                 valid = PTU_PTE_LAST | PTU_PTE_VALID;
 261                         else
 262                                 valid = PTU_PTE_VALID;
 263
 264                         pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
 265                         if (++k >= tp_next->pg_count)
 266                                 return;
 267                 }
 268         }
 269 }
 270
 271 /**
 272  * Setup a EM page table
 273  *
 274  * [in] tbl
 275  *   Pointer to EM page table
 276  */
 277 static void
 278 tf_em_setup_page_table(struct hcapi_cfa_em_table *tbl)
 279 {
 280         struct hcapi_cfa_em_page_tbl *tp_next;
 281         struct hcapi_cfa_em_page_tbl *tp;
 282         bool set_pte_last = 0;
 283         int i;
 284
 285         for (i = 0; i < tbl->num_lvl - 1; i++) {
 286                 tp = &tbl->pg_tbl[i];
 287                 tp_next = &tbl->pg_tbl[i + 1];
 288                 if (i == tbl->num_lvl - 2)
 289                         set_pte_last = 1;
 290                 tf_em_link_page_table(tp, tp_next, set_pte_last);
 291         }
 292
 293         tbl->l0_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_va_tbl[0];
 294         tbl->l0_dma_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_pa_tbl[0];
 295 }
 296
 297 /**
 298  * Given the page size, size of each data item (entry size),
 299  * and the total number of entries needed, determine the number
 300  * of page table levels and the number of data pages required.
 301  *
 302  * [in] page_size
 303  *   Page size
 304  *
 305  * [in] entry_size
 306  *   Entry size
 307  *
 308  * [in] num_entries
 309  *   Number of entries needed
 310  *
 311  * [out] num_data_pages
 312  *   Number of pages required
 313  *
 314  * Returns:
 315  *   Success  - Number of EM page levels required
 316  *   -ENOMEM  - Out of memory
 317  */
 318 static int
 319 tf_em_size_page_tbl_lvl(uint32_t page_size,
 320                         uint32_t entry_size,
 321                         uint32_t num_entries,
 322                         uint64_t *num_data_pages)
 323 {
 324         uint64_t lvl_data_size = page_size;
 325         int lvl = TF_PT_LVL_0;
 326         uint64_t data_size;
 327
 328         *num_data_pages = 0;
 329         data_size = (uint64_t)num_entries * entry_size;
 330
 331         while (lvl_data_size < data_size) {
 332                 lvl++;
 333
 334                 if (lvl == TF_PT_LVL_1)
 335                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 336                                 page_size;
 337                 else if (lvl == TF_PT_LVL_2)
 338                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 339                                 MAX_PAGE_PTRS(page_size) * page_size;
 340                 else
 341                         return -ENOMEM;
 342         }
 343
 344         *num_data_pages = roundup(data_size, page_size) / page_size;
 345
 346         return lvl;
 347 }
 348
 349 /**
 350  * Return the number of page table pages needed to
 351  * reference the given number of next level pages.
 352  *
 353  * [in] num_pages
 354  *   Number of EM pages
 355  *
 356  * [in] page_size
 357  *   Size of each EM page
 358  *
 359  * Returns:
 360  *   Number of EM page table pages
 361  */
 362 static uint32_t
 363 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 364                      uint32_t page_size)
 365 {
 366         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 367                        MAX_PAGE_PTRS(page_size);
 368         return 0;
 369 }
 370
 371 /**
 372  * Given the number of data pages, page_size and the maximum
 373  * number of page table levels (already determined), size
 374  * the number of page table pages required at each level.
 375  *
 376  * [in] max_lvl
 377  *   Max number of levels
 378  *
 379  * [in] num_data_pages
 380  *   Number of EM data pages
 381  *
 382  * [in] page_size
 383  *   Size of an EM page
 384  *
 385  * [out] *page_cnt
 386  *   EM page count
 387  */
 388 static void
 389 tf_em_size_page_tbls(int max_lvl,
 390                      uint64_t num_data_pages,
 391                      uint32_t page_size,
 392                      uint32_t *page_cnt)
 393 {
 394         if (max_lvl == TF_PT_LVL_0) {
 395                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 396         } else if (max_lvl == TF_PT_LVL_1) {
 397                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 398                 page_cnt[TF_PT_LVL_0] =
 399                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 400         } else if (max_lvl == TF_PT_LVL_2) {
 401                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 402                 page_cnt[TF_PT_LVL_1] =
 403                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 404                 page_cnt[TF_PT_LVL_0] =
 405                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 406         } else {
 407                 return;
 408         }
 409 }
 410
 411 /**
 412  * Size the EM table based on capabilities
 413  *
 414  * [in] tbl
 415  *   EM table to size
 416  *
 417  * Returns:
 418  *   0        - Success
 419  *   - EINVAL - Parameter error
 420  *   - ENOMEM - Out of memory
 421  */
 422 static int
 423 tf_em_size_table(struct hcapi_cfa_em_table *tbl)
 424 {
 425         uint64_t num_data_pages;
 426         uint32_t *page_cnt;
 427         int max_lvl;
 428         uint32_t num_entries;
 429         uint32_t cnt = TF_EM_MIN_ENTRIES;
 430
 431         /* Ignore entry if both size and number are zero */
 432         if (!tbl->entry_size && !tbl->num_entries)
 433                 return 0;
 434
 435         /* If only one is set then error */
 436         if (!tbl->entry_size || !tbl->num_entries)
 437                 return -EINVAL;
 438
 439         /* Determine number of page table levels and the number
 440          * of data pages needed to process the given eem table.
 441          */
 442         if (tbl->type == TF_RECORD_TABLE) {
 443                 /*
 444                  * For action records just a memory size is provided. Work
 445                  * backwards to resolve to number of entries
 446                  */
 447                 num_entries = tbl->num_entries / tbl->entry_size;
 448                 if (num_entries < TF_EM_MIN_ENTRIES) {
 449                         num_entries = TF_EM_MIN_ENTRIES;
 450                 } else {
 451                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 452                                 cnt *= 2;
 453                         num_entries = cnt;
 454                 }
 455         } else {
 456                 num_entries = tbl->num_entries;
 457         }
 458
 459         max_lvl = tf_em_size_page_tbl_lvl(TF_EM_PAGE_SIZE,
 460                                           tbl->entry_size,
 461                                           tbl->num_entries,
 462                                           &num_data_pages);
 463         if (max_lvl < 0) {
 464                 TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 465                 TFP_DRV_LOG(WARNING,
 466                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 467                             tbl->type, (uint64_t)num_entries * tbl->entry_size,
 468                             TF_EM_PAGE_SIZE);
 469                 return -ENOMEM;
 470         }
 471
 472         tbl->num_lvl = max_lvl + 1;
 473         tbl->num_data_pages = num_data_pages;
 474
 475         /* Determine the number of pages needed at each level */
 476         page_cnt = tbl->page_cnt;
 477         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 478         tf_em_size_page_tbls(max_lvl, num_data_pages, TF_EM_PAGE_SIZE,
 479                                 page_cnt);
 480
 481         TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 482         TFP_DRV_LOG(INFO,
 483                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 " l0: %u l1: %u l2: %u\n",
 484                     max_lvl + 1,
 485                     (uint64_t)num_data_pages * TF_EM_PAGE_SIZE,
 486                     num_data_pages,
 487                     page_cnt[TF_PT_LVL_0],
 488                     page_cnt[TF_PT_LVL_1],
 489                     page_cnt[TF_PT_LVL_2]);
 490
 491         return 0;
 492 }
 493
 494 /**
 495  * Unregisters EM Ctx in Firmware
 496  *
 497  * [in] tfp
 498  *   Pointer to a TruFlow handle
 499  *
 500  * [in] tbl_scope_cb
 501  *   Pointer to a table scope control block
 502  *
 503  * [in] dir
 504  *   Receive or transmit direction
 505  */
 506 static void
 507 tf_em_ctx_unreg(struct tf *tfp,
 508                 struct tf_tbl_scope_cb *tbl_scope_cb,
 509                 int dir)
 510 {
 511         struct hcapi_cfa_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 512         struct hcapi_cfa_em_table *tbl;
 513         int i;
 514
 515         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 516                 tbl = &ctxp->em_tables[i];
 517
 518                 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
 519                         tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
 520                         tf_em_free_page_table(tbl);
 521                 }
 522         }
 523 }
 524
 525 /**
 526  * Registers EM Ctx in Firmware
 527  *
 528  * [in] tfp
 529  *   Pointer to a TruFlow handle
 530  *
 531  * [in] tbl_scope_cb
 532  *   Pointer to a table scope control block
 533  *
 534  * [in] dir
 535  *   Receive or transmit direction
 536  *
 537  * Returns:
 538  *   0       - Success
 539  *   -ENOMEM - Out of Memory
 540  */
 541 static int
 542 tf_em_ctx_reg(struct tf *tfp,
 543               struct tf_tbl_scope_cb *tbl_scope_cb,
 544               int dir)
 545 {
 546         struct hcapi_cfa_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 547         struct hcapi_cfa_em_table *tbl;
 548         int rc;
 549         int i;
 550
 551         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 552                 tbl = &ctxp->em_tables[i];
 553
 554                 if (tbl->num_entries && tbl->entry_size) {
 555                         rc = tf_em_size_table(tbl);
 556
 557                         if (rc)
 558                                 goto cleanup;
 559
 560                         rc = tf_em_alloc_page_table(tbl);
 561                         if (rc)
 562                                 goto cleanup;
 563
 564                         tf_em_setup_page_table(tbl);
 565                         rc = tf_msg_em_mem_rgtr(tfp,
 566                                                 tbl->num_lvl - 1,
 567                                                 TF_EM_PAGE_SIZE_ENUM,
 568                                                 tbl->l0_dma_addr,
 569                                                 &tbl->ctx_id);
 570                         if (rc)
 571                                 goto cleanup;
 572                 }
 573         }
 574         return rc;
 575
 576 cleanup:
 577         tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
 578         return rc;
 579 }
 580
 581
 582 /**
 583  * Validates EM number of entries requested
 584  *
 585  * [in] tbl_scope_cb
 586  *   Pointer to table scope control block to be populated
 587  *
 588  * [in] parms
 589  *   Pointer to input parameters
 590  *
 591  * Returns:
 592  *   0       - Success
 593  *   -EINVAL - Parameter error
 594  */
 595 static int
 596 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 597                            struct tf_alloc_tbl_scope_parms *parms)
 598 {
 599         uint32_t cnt;
 600
 601         if (parms->rx_mem_size_in_mb != 0) {
 602                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 603                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 604                                      + 1);
 605                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 606                                         TF_MEGABYTE) / (key_b + action_b);
 607
 608                 if (num_entries < TF_EM_MIN_ENTRIES) {
 609                         TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 610                                     "%uMB\n",
 611                                     parms->rx_mem_size_in_mb);
 612                         return -EINVAL;
 613                 }
 614
 615                 cnt = TF_EM_MIN_ENTRIES;
 616                 while (num_entries > cnt &&
 617                        cnt <= TF_EM_MAX_ENTRIES)
 618                         cnt *= 2;
 619
 620                 if (cnt > TF_EM_MAX_ENTRIES) {
 621                         TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 622                                     "%u\n",
 623                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 624                         return -EINVAL;
 625                 }
 626
 627                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 628         } else {
 629                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 630                     TF_EM_MIN_ENTRIES ||
 631                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 632                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 633                         TFP_DRV_LOG(ERR,
 634                                     "EEM: Invalid number of Rx flows "
 635                                     "requested:%u max:%u\n",
 636                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 637                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 638                         return -EINVAL;
 639                 }
 640
 641                 /* must be a power-of-2 supported value
 642                  * in the range 32K - 128M
 643                  */
 644                 cnt = TF_EM_MIN_ENTRIES;
 645                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 646                        cnt <= TF_EM_MAX_ENTRIES)
 647                         cnt *= 2;
 648
 649                 if (cnt > TF_EM_MAX_ENTRIES) {
 650                         TFP_DRV_LOG(ERR,
 651                                     "EEM: Invalid number of Rx requested: %u\n",
 652                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 653                         return -EINVAL;
 654                 }
 655         }
 656
 657         if (parms->tx_mem_size_in_mb != 0) {
 658                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 659                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 660                                      + 1);
 661                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 662                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 663                         (key_b + action_b);
 664
 665                 if (num_entries < TF_EM_MIN_ENTRIES) {
 666                         TFP_DRV_LOG(ERR,
 667                                     "EEM: Insufficient memory requested:%uMB\n",
 668                                     parms->rx_mem_size_in_mb);
 669                         return -EINVAL;
 670                 }
 671
 672                 cnt = TF_EM_MIN_ENTRIES;
 673                 while (num_entries > cnt &&
 674                        cnt <= TF_EM_MAX_ENTRIES)
 675                         cnt *= 2;
 676
 677                 if (cnt > TF_EM_MAX_ENTRIES) {
 678                         TFP_DRV_LOG(ERR,
 679                                     "EEM: Invalid number of Tx requested: %u\n",
 680                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 681                         return -EINVAL;
 682                 }
 683
 684                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 685         } else {
 686                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 687                     TF_EM_MIN_ENTRIES ||
 688                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 689                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 690                         TFP_DRV_LOG(ERR,
 691                                     "EEM: Invalid number of Tx flows "
 692                                     "requested:%u max:%u\n",
 693                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 694                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 695                         return -EINVAL;
 696                 }
 697
 698                 cnt = TF_EM_MIN_ENTRIES;
 699                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 700                        cnt <= TF_EM_MAX_ENTRIES)
 701                         cnt *= 2;
 702
 703                 if (cnt > TF_EM_MAX_ENTRIES) {
 704                         TFP_DRV_LOG(ERR,
 705                                     "EEM: Invalid number of Tx requested: %u\n",
 706                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 707                         return -EINVAL;
 708                 }
 709         }
 710
 711         if (parms->rx_num_flows_in_k != 0 &&
 712             (parms->rx_max_key_sz_in_bits / 8 == 0)) {
 713                 TFP_DRV_LOG(ERR,
 714                             "EEM: Rx key size required: %u\n",
 715                             (parms->rx_max_key_sz_in_bits));
 716                 return -EINVAL;
 717         }
 718
 719         if (parms->tx_num_flows_in_k != 0 &&
 720             (parms->tx_max_key_sz_in_bits / 8 == 0)) {
 721                 TFP_DRV_LOG(ERR,
 722                             "EEM: Tx key size required: %u\n",
 723                             (parms->tx_max_key_sz_in_bits));
 724                 return -EINVAL;
 725         }
 726         /* Rx */
 727         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 728                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 729         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 730                 parms->rx_max_key_sz_in_bits / 8;
 731
 732         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 733                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 734         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 735                 parms->rx_max_key_sz_in_bits / 8;
 736
 737         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 738                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 739         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 740                 parms->rx_max_action_entry_sz_in_bits / 8;
 741
 742         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries = 0;
 743
 744         /* Tx */
 745         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 746                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 747         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 748                 parms->tx_max_key_sz_in_bits / 8;
 749
 750         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 751                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 752         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 753                 parms->tx_max_key_sz_in_bits / 8;
 754
 755         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 756                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 757         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 758                 parms->tx_max_action_entry_sz_in_bits / 8;
 759
 760         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries = 0;
 761
 762         return 0;
 763 }
 764
 765 /** insert EEM entry API
 766  *
 767  * returns:
 768  *  0
 769  *  TF_ERR          - unable to get lock
 770  *
 771  * insert callback returns:
 772  *   0
 773  *   TF_ERR_EM_DUP  - key is already in table
 774  */
 775 static int
 776 tf_insert_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 777                     struct tf_insert_em_entry_parms *parms)
 778 {
 779         uint32_t mask;
 780         uint32_t key0_hash;
 781         uint32_t key1_hash;
 782         uint32_t key0_index;
 783         uint32_t key1_index;
 784         struct cfa_p4_eem_64b_entry key_entry;
 785         uint32_t index;
 786         enum hcapi_cfa_em_table_type table_type;
 787         uint32_t gfid;
 788         struct hcapi_cfa_hwop op;
 789         struct hcapi_cfa_key_tbl key_tbl;
 790         struct hcapi_cfa_key_data key_obj;
 791         struct hcapi_cfa_key_loc key_loc;
 792         uint64_t big_hash;
 793         int rc;
 794
 795         /* Get mask to use on hash */
 796         mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
 797
 798         if (!mask)
 799                 return -EINVAL;
 800
 801 #ifdef TF_EEM_DEBUG
 802         dump_raw((uint8_t *)parms->key, TF_HW_EM_KEY_MAX_SIZE + 4, "In Key");
 803 #endif
 804
 805         big_hash = hcapi_cfa_key_hash((uint64_t *)parms->key,
 806                                       (TF_HW_EM_KEY_MAX_SIZE + 4) * 8);
 807         key0_hash = (uint32_t)(big_hash >> 32);
 808         key1_hash = (uint32_t)(big_hash & 0xFFFFFFFF);
 809
 810         key0_index = key0_hash & mask;
 811         key1_index = key1_hash & mask;
 812
 813 #ifdef TF_EEM_DEBUG
 814         TFP_DRV_LOG(DEBUG, "Key0 hash:0x%08x\n", key0_hash);
 815         TFP_DRV_LOG(DEBUG, "Key1 hash:0x%08x\n", key1_hash);
 816 #endif
 817         /*
 818          * Use the "result" arg to populate all of the key entry then
 819          * store the byte swapped "raw" entry in a local copy ready
 820          * for insertion in to the table.
 821          */
 822         tf_em_create_key_entry((struct cfa_p4_eem_entry_hdr *)parms->em_record,
 823                                 ((uint8_t *)parms->key),
 824                                 &key_entry);
 825
 826         /*
 827          * Try to add to Key0 table, if that does not work then
 828          * try the key1 table.
 829          */
 830         index = key0_index;
 831         op.opcode = HCAPI_CFA_HWOPS_ADD;
 832         key_tbl.base0 = (uint8_t *)
 833                 &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE];
 834         key_tbl.page_size = TF_EM_PAGE_SIZE;
 835         key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 836         key_obj.data = (uint8_t *)&key_entry;
 837         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 838
 839         rc = hcapi_cfa_key_hw_op(&op,
 840                                  &key_tbl,
 841                                  &key_obj,
 842                                  &key_loc);
 843
 844         if (rc == 0) {
 845                 table_type = TF_KEY0_TABLE;
 846         } else {
 847                 index = key1_index;
 848
 849                 key_tbl.base0 = (uint8_t *)
 850                 &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY1_TABLE];
 851                 key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 852
 853                 rc = hcapi_cfa_key_hw_op(&op,
 854                                          &key_tbl,
 855                                          &key_obj,
 856                                          &key_loc);
 857                 if (rc != 0)
 858                         return rc;
 859
 860                 table_type = TF_KEY1_TABLE;
 861         }
 862
 863         TF_SET_GFID(gfid,
 864                     index,
 865                     table_type);
 866         TF_SET_FLOW_ID(parms->flow_id,
 867                        gfid,
 868                        TF_GFID_TABLE_EXTERNAL,
 869                        parms->dir);
 870         TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
 871                                      0,
 872                                      0,
 873                                      0,
 874                                      index,
 875                                      0,
 876                                      table_type);
 877
 878         return 0;
 879 }
 880
 881 /** delete EEM hash entry API
 882  *
 883  * returns:
 884  *   0
 885  *   -EINVAL      - parameter error
 886  *   TF_NO_SESSION    - bad session ID
 887  *   TF_ERR_TBL_SCOPE - invalid table scope
 888  *   TF_ERR_TBL_IF    - invalid table interface
 889  *
 890  * insert callback returns
 891  *   0
 892  *   TF_NO_EM_MATCH - entry not found
 893  */
 894 static int
 895 tf_delete_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 896                     struct tf_delete_em_entry_parms *parms)
 897 {
 898         enum hcapi_cfa_em_table_type hash_type;
 899         uint32_t index;
 900         struct hcapi_cfa_hwop op;
 901         struct hcapi_cfa_key_tbl key_tbl;
 902         struct hcapi_cfa_key_data key_obj;
 903         struct hcapi_cfa_key_loc key_loc;
 904         int rc;
 905
 906         if (parms->flow_handle == 0)
 907                 return -EINVAL;
 908
 909         TF_GET_HASH_TYPE_FROM_FLOW_HANDLE(parms->flow_handle, hash_type);
 910         TF_GET_INDEX_FROM_FLOW_HANDLE(parms->flow_handle, index);
 911
 912         op.opcode = HCAPI_CFA_HWOPS_DEL;
 913         key_tbl.base0 = (uint8_t *)
 914         &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[(hash_type == 0 ?
 915                                                           TF_KEY0_TABLE :
 916                                                           TF_KEY1_TABLE)];
 917         key_tbl.page_size = TF_EM_PAGE_SIZE;
 918         key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 919         key_obj.data = NULL;
 920         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 921
 922         rc = hcapi_cfa_key_hw_op(&op,
 923                                  &key_tbl,
 924                                  &key_obj,
 925                                  &key_loc);
 926
 927         if (!rc)
 928                 return rc;
 929
 930         return 0;
 931 }
 932
 933 /** insert EM hash entry API
 934  *
 935  *    returns:
 936  *    0       - Success
 937  *    -EINVAL - Error
 938  */
 939 int
 940 tf_em_insert_ext_entry(struct tf *tfp __rte_unused,
 941                        struct tf_insert_em_entry_parms *parms)
 942 {
 943         struct tf_tbl_scope_cb *tbl_scope_cb;
 944
 945         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 946         if (tbl_scope_cb == NULL) {
 947                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 948                 return -EINVAL;
 949         }
 950
 951         return tf_insert_eem_entry(tbl_scope_cb, parms);
 952 }
 953
 954 /** Delete EM hash entry API
 955  *
 956  *    returns:
 957  *    0       - Success
 958  *    -EINVAL - Error
 959  */
 960 int
 961 tf_em_delete_ext_entry(struct tf *tfp __rte_unused,
 962                        struct tf_delete_em_entry_parms *parms)
 963 {
 964         struct tf_tbl_scope_cb *tbl_scope_cb;
 965
 966         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 967         if (tbl_scope_cb == NULL) {
 968                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 969                 return -EINVAL;
 970         }
 971
 972         return tf_delete_eem_entry(tbl_scope_cb, parms);
 973 }
 974
 975 int
 976 tf_em_ext_host_alloc(struct tf *tfp,
 977                      struct tf_alloc_tbl_scope_parms *parms)
 978 {
 979         int rc;
 980         enum tf_dir dir;
 981         struct tf_tbl_scope_cb *tbl_scope_cb;
 982         struct hcapi_cfa_em_table *em_tables;
 983         struct tf_free_tbl_scope_parms free_parms;
 984         struct tf_rm_allocate_parms aparms = { 0 };
 985         struct tf_rm_free_parms fparms = { 0 };
 986
 987         /* Get Table Scope control block from the session pool */
 988         aparms.rm_db = eem_db[TF_DIR_RX];
 989         aparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
 990         aparms.index = (uint32_t *)&parms->tbl_scope_id;
 991         rc = tf_rm_allocate(&aparms);
 992         if (rc) {
 993                 TFP_DRV_LOG(ERR,
 994                             "Failed to allocate table scope\n");
 995                 return rc;
 996         }
 997
 998         tbl_scope_cb = &tbl_scopes[parms->tbl_scope_id];
 999         tbl_scope_cb->index = parms->tbl_scope_id;
1000         tbl_scope_cb->tbl_scope_id = parms->tbl_scope_id;
1001
1002         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1003                 rc = tf_msg_em_qcaps(tfp,
1004                                      dir,
1005                                      &tbl_scope_cb->em_caps[dir]);
1006                 if (rc) {
1007                         TFP_DRV_LOG(ERR,
1008                                     "EEM: Unable to query for EEM capability,"
1009                                     " rc:%s\n",
1010                                     strerror(-rc));
1011                         goto cleanup;
1012                 }
1013         }
1014
1015         /*
1016          * Validate and setup table sizes
1017          */
1018         if (tf_em_validate_num_entries(tbl_scope_cb, parms))
1019                 goto cleanup;
1020
1021         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1022                 /*
1023                  * Allocate tables and signal configuration to FW
1024                  */
1025                 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
1026                 if (rc) {
1027                         TFP_DRV_LOG(ERR,
1028                                     "EEM: Unable to register for EEM ctx,"
1029                                     " rc:%s\n",
1030                                     strerror(-rc));
1031                         goto cleanup;
1032                 }
1033
1034                 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
1035                 rc = tf_msg_em_cfg(tfp,
1036                                    em_tables[TF_KEY0_TABLE].num_entries,
1037                                    em_tables[TF_KEY0_TABLE].ctx_id,
1038                                    em_tables[TF_KEY1_TABLE].ctx_id,
1039                                    em_tables[TF_RECORD_TABLE].ctx_id,
1040                                    em_tables[TF_EFC_TABLE].ctx_id,
1041                                    parms->hw_flow_cache_flush_timer,
1042                                    dir);
1043                 if (rc) {
1044                         TFP_DRV_LOG(ERR,
1045                                     "TBL: Unable to configure EEM in firmware"
1046                                     " rc:%s\n",
1047                                     strerror(-rc));
1048                         goto cleanup_full;
1049                 }
1050
1051                 rc = tf_msg_em_op(tfp,
1052                                   dir,
1053                                   HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
1054
1055                 if (rc) {
1056                         TFP_DRV_LOG(ERR,
1057                                     "EEM: Unable to enable EEM in firmware"
1058                                     " rc:%s\n",
1059                                     strerror(-rc));
1060                         goto cleanup_full;
1061                 }
1062
1063                 /* Allocate the pool of offsets of the external memory.
1064                  * Initially, this is a single fixed size pool for all external
1065                  * actions related to a single table scope.
1066                  */
1067                 rc = tf_create_tbl_pool_external(dir,
1068                                             tbl_scope_cb,
1069                                             em_tables[TF_RECORD_TABLE].num_entries,
1070                                             em_tables[TF_RECORD_TABLE].entry_size);
1071                 if (rc) {
1072                         TFP_DRV_LOG(ERR,
1073                                     "%s TBL: Unable to allocate idx pools %s\n",
1074                                     tf_dir_2_str(dir),
1075                                     strerror(-rc));
1076                         goto cleanup_full;
1077                 }
1078         }
1079
1080         return 0;
1081
1082 cleanup_full:
1083         free_parms.tbl_scope_id = parms->tbl_scope_id;
1084         tf_em_ext_host_free(tfp, &free_parms);
1085         return -EINVAL;
1086
1087 cleanup:
1088         /* Free Table control block */
1089         fparms.rm_db = eem_db[TF_DIR_RX];
1090         fparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
1091         fparms.index = parms->tbl_scope_id;
1092         tf_rm_free(&fparms);
1093         return -EINVAL;
1094 }
1095
1096 int
1097 tf_em_ext_host_free(struct tf *tfp,
1098                     struct tf_free_tbl_scope_parms *parms)
1099 {
1100         int rc = 0;
1101         enum tf_dir  dir;
1102         struct tf_tbl_scope_cb *tbl_scope_cb;
1103         struct tf_rm_free_parms aparms = { 0 };
1104
1105         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
1106
1107         if (tbl_scope_cb == NULL) {
1108                 TFP_DRV_LOG(ERR, "Table scope error\n");
1109                 return -EINVAL;
1110         }
1111
1112         /* Free Table control block */
1113         aparms.rm_db = eem_db[TF_DIR_RX];
1114         aparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
1115         aparms.index = parms->tbl_scope_id;
1116         rc = tf_rm_free(&aparms);
1117         if (rc) {
1118                 TFP_DRV_LOG(ERR,
1119                             "Failed to free table scope\n");
1120         }
1121
1122         /* free table scope locks */
1123         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1124                 /* Free associated external pools
1125                  */
1126                 tf_destroy_tbl_pool_external(dir,
1127                                              tbl_scope_cb);
1128                 tf_msg_em_op(tfp,
1129                              dir,
1130                              HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
1131
1132                 /* free table scope and all associated resources */
1133                 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
1134         }
1135
1136         tbl_scopes[parms->tbl_scope_id].tbl_scope_id = TF_TBL_SCOPE_INVALID;
1137         return rc;
1138 }
1139
1140 /**
1141  * Sets the specified external table type element.
1142  *
1143  * This API sets the specified element data
1144  *
1145  * [in] tfp
1146  *   Pointer to TF handle
1147  *
1148  * [in] parms
1149  *   Pointer to table set parameters
1150  *
1151  * Returns
1152  *   - (0) if successful.
1153  *   - (-EINVAL) on failure.
1154  */
1155 int tf_tbl_ext_host_set(struct tf *tfp,
1156                         struct tf_tbl_set_parms *parms)
1157 {
1158         int rc = 0;
1159         struct tf_tbl_scope_cb *tbl_scope_cb;
1160         uint32_t tbl_scope_id;
1161         struct hcapi_cfa_hwop op;
1162         struct hcapi_cfa_key_tbl key_tbl;
1163         struct hcapi_cfa_key_data key_obj;
1164         struct hcapi_cfa_key_loc key_loc;
1165
1166         TF_CHECK_PARMS2(tfp, parms);
1167
1168         if (parms->data == NULL) {
1169                 TFP_DRV_LOG(ERR,
1170                             "%s, invalid parms->data\n",
1171                             tf_dir_2_str(parms->dir));
1172                 return -EINVAL;
1173         }
1174
1175         tbl_scope_id = parms->tbl_scope_id;
1176
1177         if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1178                 TFP_DRV_LOG(ERR,
1179                             "%s, Table scope not allocated\n",
1180                             tf_dir_2_str(parms->dir));
1181                 return -EINVAL;
1182         }
1183
1184         /* Get the table scope control block associated with the
1185          * external pool
1186          */
1187         tbl_scope_cb = tbl_scope_cb_find(tbl_scope_id);
1188
1189         if (tbl_scope_cb == NULL) {
1190                 TFP_DRV_LOG(ERR,
1191                             "%s, table scope error\n",
1192                             tf_dir_2_str(parms->dir));
1193                 return -EINVAL;
1194         }
1195
1196         op.opcode = HCAPI_CFA_HWOPS_PUT;
1197         key_tbl.base0 =
1198                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_RECORD_TABLE];
1199         key_tbl.page_size = TF_EM_PAGE_SIZE;
1200         key_obj.offset = parms->idx;
1201         key_obj.data = parms->data;
1202         key_obj.size = parms->data_sz_in_bytes;
1203
1204         rc = hcapi_cfa_key_hw_op(&op,
1205                                  &key_tbl,
1206                                  &key_obj,
1207                                  &key_loc);
1208
1209         return rc;
1210 }