drivers/net/bnxt/tf_core/tf_em_host.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <math.h>
   8 #include <sys/param.h>
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12
  13 #include "tf_core.h"
  14 #include "tf_util.h"
  15 #include "tf_common.h"
  16 #include "tf_em.h"
  17 #include "tf_em_common.h"
  18 #include "tf_msg.h"
  19 #include "tfp.h"
  20 #include "lookup3.h"
  21 #include "tf_ext_flow_handle.h"
  22
  23 #include "bnxt.h"
  24
  25
  26 #define PTU_PTE_VALID          0x1UL
  27 #define PTU_PTE_LAST           0x2UL
  28 #define PTU_PTE_NEXT_TO_LAST   0x4UL
  29
  30 /* Number of pointers per page_size */
  31 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  32
  33 #define TF_EM_PG_SZ_4K        (1 << 12)
  34 #define TF_EM_PG_SZ_8K        (1 << 13)
  35 #define TF_EM_PG_SZ_64K       (1 << 16)
  36 #define TF_EM_PG_SZ_256K      (1 << 18)
  37 #define TF_EM_PG_SZ_1M        (1 << 20)
  38 #define TF_EM_PG_SZ_2M        (1 << 21)
  39 #define TF_EM_PG_SZ_4M        (1 << 22)
  40 #define TF_EM_PG_SZ_1G        (1 << 30)
  41
  42 #define TF_EM_CTX_ID_INVALID   0xFFFF
  43
  44 #define TF_EM_MIN_ENTRIES     (1 << 15) /* 32K */
  45 #define TF_EM_MAX_ENTRIES     (1 << 27) /* 128M */
  46
  47 /**
  48  * EM DBs.
  49  */
  50 extern void *eem_db[TF_DIR_MAX];
  51 #define TF_EEM_DB_TBL_SCOPE 1
  52
  53 extern struct tf_tbl_scope_cb tbl_scopes[TF_NUM_TBL_SCOPE];
  54
  55 /**
  56  * Function to free a page table
  57  *
  58  * [in] tp
  59  *   Pointer to the page table to free
  60  */
  61 static void
  62 tf_em_free_pg_tbl(struct hcapi_cfa_em_page_tbl *tp)
  63 {
  64         uint32_t i;
  65
  66         for (i = 0; i < tp->pg_count; i++) {
  67                 if (!tp->pg_va_tbl[i]) {
  68                         TFP_DRV_LOG(WARNING,
  69                                     "No mapping for page: %d table: %016" PRIu64 "\n",
  70                                     i,
  71                                     (uint64_t)(uintptr_t)tp);
  72                         continue;
  73                 }
  74
  75                 tfp_free(tp->pg_va_tbl[i]);
  76                 tp->pg_va_tbl[i] = NULL;
  77         }
  78
  79         tp->pg_count = 0;
  80         tfp_free(tp->pg_va_tbl);
  81         tp->pg_va_tbl = NULL;
  82         tfp_free(tp->pg_pa_tbl);
  83         tp->pg_pa_tbl = NULL;
  84 }
  85
  86 /**
  87  * Function to free an EM table
  88  *
  89  * [in] tbl
  90  *   Pointer to the EM table to free
  91  */
  92 static void
  93 tf_em_free_page_table(struct hcapi_cfa_em_table *tbl)
  94 {
  95         struct hcapi_cfa_em_page_tbl *tp;
  96         int i;
  97
  98         for (i = 0; i < tbl->num_lvl; i++) {
  99                 tp = &tbl->pg_tbl[i];
 100                 TFP_DRV_LOG(INFO,
 101                            "EEM: Freeing page table: size %u lvl %d cnt %u\n",
 102                            TF_EM_PAGE_SIZE,
 103                             i,
 104                             tp->pg_count);
 105
 106                 tf_em_free_pg_tbl(tp);
 107         }
 108
 109         tbl->l0_addr = NULL;
 110         tbl->l0_dma_addr = 0;
 111         tbl->num_lvl = 0;
 112         tbl->num_data_pages = 0;
 113 }
 114
 115 /**
 116  * Allocation of page tables
 117  *
 118  * [in] tfp
 119  *   Pointer to a TruFlow handle
 120  *
 121  * [in] pg_count
 122  *   Page count to allocate
 123  *
 124  * [in] pg_size
 125  *   Size of each page
 126  *
 127  * Returns:
 128  *   0       - Success
 129  *   -ENOMEM - Out of memory
 130  */
 131 static int
 132 tf_em_alloc_pg_tbl(struct hcapi_cfa_em_page_tbl *tp,
 133                    uint32_t pg_count,
 134                    uint32_t pg_size)
 135 {
 136         uint32_t i;
 137         struct tfp_calloc_parms parms;
 138
 139         parms.nitems = pg_count;
 140         parms.size = sizeof(void *);
 141         parms.alignment = 0;
 142
 143         if (tfp_calloc(&parms) != 0)
 144                 return -ENOMEM;
 145
 146         tp->pg_va_tbl = parms.mem_va;
 147
 148         if (tfp_calloc(&parms) != 0) {
 149                 tfp_free(tp->pg_va_tbl);
 150                 return -ENOMEM;
 151         }
 152
 153         tp->pg_pa_tbl = parms.mem_va;
 154
 155         tp->pg_count = 0;
 156         tp->pg_size = pg_size;
 157
 158         for (i = 0; i < pg_count; i++) {
 159                 parms.nitems = 1;
 160                 parms.size = pg_size;
 161                 parms.alignment = TF_EM_PAGE_ALIGNMENT;
 162
 163                 if (tfp_calloc(&parms) != 0)
 164                         goto cleanup;
 165
 166                 tp->pg_pa_tbl[i] = (uintptr_t)parms.mem_pa;
 167                 tp->pg_va_tbl[i] = parms.mem_va;
 168
 169                 memset(tp->pg_va_tbl[i], 0, pg_size);
 170                 tp->pg_count++;
 171         }
 172
 173         return 0;
 174
 175 cleanup:
 176         tf_em_free_pg_tbl(tp);
 177         return -ENOMEM;
 178 }
 179
 180 /**
 181  * Allocates EM page tables
 182  *
 183  * [in] tbl
 184  *   Table to allocate pages for
 185  *
 186  * Returns:
 187  *   0       - Success
 188  *   -ENOMEM - Out of memory
 189  */
 190 static int
 191 tf_em_alloc_page_table(struct hcapi_cfa_em_table *tbl)
 192 {
 193         struct hcapi_cfa_em_page_tbl *tp;
 194         int rc = 0;
 195         int i;
 196         uint32_t j;
 197
 198         for (i = 0; i < tbl->num_lvl; i++) {
 199                 tp = &tbl->pg_tbl[i];
 200
 201                 rc = tf_em_alloc_pg_tbl(tp,
 202                                         tbl->page_cnt[i],
 203                                         TF_EM_PAGE_SIZE);
 204                 if (rc) {
 205                         TFP_DRV_LOG(WARNING,
 206                                 "Failed to allocate page table: lvl: %d, rc:%s\n",
 207                                 i,
 208                                 strerror(-rc));
 209                         goto cleanup;
 210                 }
 211
 212                 for (j = 0; j < tp->pg_count; j++) {
 213                         TFP_DRV_LOG(INFO,
 214                                 "EEM: Allocated page table: size %u lvl %d cnt"
 215                                 " %u VA:%p PA:%p\n",
 216                                 TF_EM_PAGE_SIZE,
 217                                 i,
 218                                 tp->pg_count,
 219                                 (void *)(uintptr_t)tp->pg_va_tbl[j],
 220                                 (void *)(uintptr_t)tp->pg_pa_tbl[j]);
 221                 }
 222         }
 223         return rc;
 224
 225 cleanup:
 226         tf_em_free_page_table(tbl);
 227         return rc;
 228 }
 229
 230 /**
 231  * Links EM page tables
 232  *
 233  * [in] tp
 234  *   Pointer to page table
 235  *
 236  * [in] tp_next
 237  *   Pointer to the next page table
 238  *
 239  * [in] set_pte_last
 240  *   Flag controlling if the page table is last
 241  */
 242 static void
 243 tf_em_link_page_table(struct hcapi_cfa_em_page_tbl *tp,
 244                       struct hcapi_cfa_em_page_tbl *tp_next,
 245                       bool set_pte_last)
 246 {
 247         uint64_t *pg_pa = tp_next->pg_pa_tbl;
 248         uint64_t *pg_va;
 249         uint64_t valid;
 250         uint32_t k = 0;
 251         uint32_t i;
 252         uint32_t j;
 253
 254         for (i = 0; i < tp->pg_count; i++) {
 255                 pg_va = tp->pg_va_tbl[i];
 256
 257                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
 258                         if (k == tp_next->pg_count - 2 && set_pte_last)
 259                                 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
 260                         else if (k == tp_next->pg_count - 1 && set_pte_last)
 261                                 valid = PTU_PTE_LAST | PTU_PTE_VALID;
 262                         else
 263                                 valid = PTU_PTE_VALID;
 264
 265                         pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
 266                         if (++k >= tp_next->pg_count)
 267                                 return;
 268                 }
 269         }
 270 }
 271
 272 /**
 273  * Setup a EM page table
 274  *
 275  * [in] tbl
 276  *   Pointer to EM page table
 277  */
 278 static void
 279 tf_em_setup_page_table(struct hcapi_cfa_em_table *tbl)
 280 {
 281         struct hcapi_cfa_em_page_tbl *tp_next;
 282         struct hcapi_cfa_em_page_tbl *tp;
 283         bool set_pte_last = 0;
 284         int i;
 285
 286         for (i = 0; i < tbl->num_lvl - 1; i++) {
 287                 tp = &tbl->pg_tbl[i];
 288                 tp_next = &tbl->pg_tbl[i + 1];
 289                 if (i == tbl->num_lvl - 2)
 290                         set_pte_last = 1;
 291                 tf_em_link_page_table(tp, tp_next, set_pte_last);
 292         }
 293
 294         tbl->l0_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_va_tbl[0];
 295         tbl->l0_dma_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_pa_tbl[0];
 296 }
 297
 298 /**
 299  * Given the page size, size of each data item (entry size),
 300  * and the total number of entries needed, determine the number
 301  * of page table levels and the number of data pages required.
 302  *
 303  * [in] page_size
 304  *   Page size
 305  *
 306  * [in] entry_size
 307  *   Entry size
 308  *
 309  * [in] num_entries
 310  *   Number of entries needed
 311  *
 312  * [out] num_data_pages
 313  *   Number of pages required
 314  *
 315  * Returns:
 316  *   Success  - Number of EM page levels required
 317  *   -ENOMEM  - Out of memory
 318  */
 319 static int
 320 tf_em_size_page_tbl_lvl(uint32_t page_size,
 321                         uint32_t entry_size,
 322                         uint32_t num_entries,
 323                         uint64_t *num_data_pages)
 324 {
 325         uint64_t lvl_data_size = page_size;
 326         int lvl = TF_PT_LVL_0;
 327         uint64_t data_size;
 328
 329         *num_data_pages = 0;
 330         data_size = (uint64_t)num_entries * entry_size;
 331
 332         while (lvl_data_size < data_size) {
 333                 lvl++;
 334
 335                 if (lvl == TF_PT_LVL_1)
 336                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 337                                 page_size;
 338                 else if (lvl == TF_PT_LVL_2)
 339                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 340                                 MAX_PAGE_PTRS(page_size) * page_size;
 341                 else
 342                         return -ENOMEM;
 343         }
 344
 345         *num_data_pages = roundup(data_size, page_size) / page_size;
 346
 347         return lvl;
 348 }
 349
 350 /**
 351  * Return the number of page table pages needed to
 352  * reference the given number of next level pages.
 353  *
 354  * [in] num_pages
 355  *   Number of EM pages
 356  *
 357  * [in] page_size
 358  *   Size of each EM page
 359  *
 360  * Returns:
 361  *   Number of EM page table pages
 362  */
 363 static uint32_t
 364 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 365                      uint32_t page_size)
 366 {
 367         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 368                        MAX_PAGE_PTRS(page_size);
 369         return 0;
 370 }
 371
 372 /**
 373  * Given the number of data pages, page_size and the maximum
 374  * number of page table levels (already determined), size
 375  * the number of page table pages required at each level.
 376  *
 377  * [in] max_lvl
 378  *   Max number of levels
 379  *
 380  * [in] num_data_pages
 381  *   Number of EM data pages
 382  *
 383  * [in] page_size
 384  *   Size of an EM page
 385  *
 386  * [out] *page_cnt
 387  *   EM page count
 388  */
 389 static void
 390 tf_em_size_page_tbls(int max_lvl,
 391                      uint64_t num_data_pages,
 392                      uint32_t page_size,
 393                      uint32_t *page_cnt)
 394 {
 395         if (max_lvl == TF_PT_LVL_0) {
 396                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 397         } else if (max_lvl == TF_PT_LVL_1) {
 398                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 399                 page_cnt[TF_PT_LVL_0] =
 400                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 401         } else if (max_lvl == TF_PT_LVL_2) {
 402                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 403                 page_cnt[TF_PT_LVL_1] =
 404                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 405                 page_cnt[TF_PT_LVL_0] =
 406                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 407         } else {
 408                 return;
 409         }
 410 }
 411
 412 /**
 413  * Size the EM table based on capabilities
 414  *
 415  * [in] tbl
 416  *   EM table to size
 417  *
 418  * Returns:
 419  *   0        - Success
 420  *   - EINVAL - Parameter error
 421  *   - ENOMEM - Out of memory
 422  */
 423 static int
 424 tf_em_size_table(struct hcapi_cfa_em_table *tbl)
 425 {
 426         uint64_t num_data_pages;
 427         uint32_t *page_cnt;
 428         int max_lvl;
 429         uint32_t num_entries;
 430         uint32_t cnt = TF_EM_MIN_ENTRIES;
 431
 432         /* Ignore entry if both size and number are zero */
 433         if (!tbl->entry_size && !tbl->num_entries)
 434                 return 0;
 435
 436         /* If only one is set then error */
 437         if (!tbl->entry_size || !tbl->num_entries)
 438                 return -EINVAL;
 439
 440         /* Determine number of page table levels and the number
 441          * of data pages needed to process the given eem table.
 442          */
 443         if (tbl->type == TF_RECORD_TABLE) {
 444                 /*
 445                  * For action records just a memory size is provided. Work
 446                  * backwards to resolve to number of entries
 447                  */
 448                 num_entries = tbl->num_entries / tbl->entry_size;
 449                 if (num_entries < TF_EM_MIN_ENTRIES) {
 450                         num_entries = TF_EM_MIN_ENTRIES;
 451                 } else {
 452                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 453                                 cnt *= 2;
 454                         num_entries = cnt;
 455                 }
 456         } else {
 457                 num_entries = tbl->num_entries;
 458         }
 459
 460         max_lvl = tf_em_size_page_tbl_lvl(TF_EM_PAGE_SIZE,
 461                                           tbl->entry_size,
 462                                           tbl->num_entries,
 463                                           &num_data_pages);
 464         if (max_lvl < 0) {
 465                 TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 466                 TFP_DRV_LOG(WARNING,
 467                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 468                             tbl->type, (uint64_t)num_entries * tbl->entry_size,
 469                             TF_EM_PAGE_SIZE);
 470                 return -ENOMEM;
 471         }
 472
 473         tbl->num_lvl = max_lvl + 1;
 474         tbl->num_data_pages = num_data_pages;
 475
 476         /* Determine the number of pages needed at each level */
 477         page_cnt = tbl->page_cnt;
 478         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 479         tf_em_size_page_tbls(max_lvl, num_data_pages, TF_EM_PAGE_SIZE,
 480                                 page_cnt);
 481
 482         TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 483         TFP_DRV_LOG(INFO,
 484                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 " l0: %u l1: %u l2: %u\n",
 485                     max_lvl + 1,
 486                     (uint64_t)num_data_pages * TF_EM_PAGE_SIZE,
 487                     num_data_pages,
 488                     page_cnt[TF_PT_LVL_0],
 489                     page_cnt[TF_PT_LVL_1],
 490                     page_cnt[TF_PT_LVL_2]);
 491
 492         return 0;
 493 }
 494
 495 /**
 496  * Unregisters EM Ctx in Firmware
 497  *
 498  * [in] tfp
 499  *   Pointer to a TruFlow handle
 500  *
 501  * [in] tbl_scope_cb
 502  *   Pointer to a table scope control block
 503  *
 504  * [in] dir
 505  *   Receive or transmit direction
 506  */
 507 static void
 508 tf_em_ctx_unreg(struct tf *tfp,
 509                 struct tf_tbl_scope_cb *tbl_scope_cb,
 510                 int dir)
 511 {
 512         struct hcapi_cfa_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 513         struct hcapi_cfa_em_table *tbl;
 514         int i;
 515
 516         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 517                 tbl = &ctxp->em_tables[i];
 518
 519                 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
 520                         tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
 521                         tf_em_free_page_table(tbl);
 522                 }
 523         }
 524 }
 525
 526 /**
 527  * Registers EM Ctx in Firmware
 528  *
 529  * [in] tfp
 530  *   Pointer to a TruFlow handle
 531  *
 532  * [in] tbl_scope_cb
 533  *   Pointer to a table scope control block
 534  *
 535  * [in] dir
 536  *   Receive or transmit direction
 537  *
 538  * Returns:
 539  *   0       - Success
 540  *   -ENOMEM - Out of Memory
 541  */
 542 static int
 543 tf_em_ctx_reg(struct tf *tfp,
 544               struct tf_tbl_scope_cb *tbl_scope_cb,
 545               int dir)
 546 {
 547         struct hcapi_cfa_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 548         struct hcapi_cfa_em_table *tbl;
 549         int rc;
 550         int i;
 551
 552         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 553                 tbl = &ctxp->em_tables[i];
 554
 555                 if (tbl->num_entries && tbl->entry_size) {
 556                         rc = tf_em_size_table(tbl);
 557
 558                         if (rc)
 559                                 goto cleanup;
 560
 561                         rc = tf_em_alloc_page_table(tbl);
 562                         if (rc)
 563                                 goto cleanup;
 564
 565                         tf_em_setup_page_table(tbl);
 566                         rc = tf_msg_em_mem_rgtr(tfp,
 567                                                 tbl->num_lvl - 1,
 568                                                 TF_EM_PAGE_SIZE_ENUM,
 569                                                 tbl->l0_dma_addr,
 570                                                 &tbl->ctx_id);
 571                         if (rc)
 572                                 goto cleanup;
 573                 }
 574         }
 575         return rc;
 576
 577 cleanup:
 578         tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
 579         return rc;
 580 }
 581
 582
 583 /**
 584  * Validates EM number of entries requested
 585  *
 586  * [in] tbl_scope_cb
 587  *   Pointer to table scope control block to be populated
 588  *
 589  * [in] parms
 590  *   Pointer to input parameters
 591  *
 592  * Returns:
 593  *   0       - Success
 594  *   -EINVAL - Parameter error
 595  */
 596 static int
 597 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 598                            struct tf_alloc_tbl_scope_parms *parms)
 599 {
 600         uint32_t cnt;
 601
 602         if (parms->rx_mem_size_in_mb != 0) {
 603                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 604                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 605                                      + 1);
 606                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 607                                         TF_MEGABYTE) / (key_b + action_b);
 608
 609                 if (num_entries < TF_EM_MIN_ENTRIES) {
 610                         TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 611                                     "%uMB\n",
 612                                     parms->rx_mem_size_in_mb);
 613                         return -EINVAL;
 614                 }
 615
 616                 cnt = TF_EM_MIN_ENTRIES;
 617                 while (num_entries > cnt &&
 618                        cnt <= TF_EM_MAX_ENTRIES)
 619                         cnt *= 2;
 620
 621                 if (cnt > TF_EM_MAX_ENTRIES) {
 622                         TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 623                                     "%u\n",
 624                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 625                         return -EINVAL;
 626                 }
 627
 628                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 629         } else {
 630                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 631                     TF_EM_MIN_ENTRIES ||
 632                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 633                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 634                         TFP_DRV_LOG(ERR,
 635                                     "EEM: Invalid number of Rx flows "
 636                                     "requested:%u max:%u\n",
 637                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 638                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 639                         return -EINVAL;
 640                 }
 641
 642                 /* must be a power-of-2 supported value
 643                  * in the range 32K - 128M
 644                  */
 645                 cnt = TF_EM_MIN_ENTRIES;
 646                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 647                        cnt <= TF_EM_MAX_ENTRIES)
 648                         cnt *= 2;
 649
 650                 if (cnt > TF_EM_MAX_ENTRIES) {
 651                         TFP_DRV_LOG(ERR,
 652                                     "EEM: Invalid number of Rx requested: %u\n",
 653                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 654                         return -EINVAL;
 655                 }
 656         }
 657
 658         if (parms->tx_mem_size_in_mb != 0) {
 659                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 660                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 661                                      + 1);
 662                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 663                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 664                         (key_b + action_b);
 665
 666                 if (num_entries < TF_EM_MIN_ENTRIES) {
 667                         TFP_DRV_LOG(ERR,
 668                                     "EEM: Insufficient memory requested:%uMB\n",
 669                                     parms->rx_mem_size_in_mb);
 670                         return -EINVAL;
 671                 }
 672
 673                 cnt = TF_EM_MIN_ENTRIES;
 674                 while (num_entries > cnt &&
 675                        cnt <= TF_EM_MAX_ENTRIES)
 676                         cnt *= 2;
 677
 678                 if (cnt > TF_EM_MAX_ENTRIES) {
 679                         TFP_DRV_LOG(ERR,
 680                                     "EEM: Invalid number of Tx requested: %u\n",
 681                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 682                         return -EINVAL;
 683                 }
 684
 685                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 686         } else {
 687                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 688                     TF_EM_MIN_ENTRIES ||
 689                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 690                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 691                         TFP_DRV_LOG(ERR,
 692                                     "EEM: Invalid number of Tx flows "
 693                                     "requested:%u max:%u\n",
 694                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 695                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 696                         return -EINVAL;
 697                 }
 698
 699                 cnt = TF_EM_MIN_ENTRIES;
 700                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 701                        cnt <= TF_EM_MAX_ENTRIES)
 702                         cnt *= 2;
 703
 704                 if (cnt > TF_EM_MAX_ENTRIES) {
 705                         TFP_DRV_LOG(ERR,
 706                                     "EEM: Invalid number of Tx requested: %u\n",
 707                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 708                         return -EINVAL;
 709                 }
 710         }
 711
 712         if (parms->rx_num_flows_in_k != 0 &&
 713             (parms->rx_max_key_sz_in_bits / 8 == 0)) {
 714                 TFP_DRV_LOG(ERR,
 715                             "EEM: Rx key size required: %u\n",
 716                             (parms->rx_max_key_sz_in_bits));
 717                 return -EINVAL;
 718         }
 719
 720         if (parms->tx_num_flows_in_k != 0 &&
 721             (parms->tx_max_key_sz_in_bits / 8 == 0)) {
 722                 TFP_DRV_LOG(ERR,
 723                             "EEM: Tx key size required: %u\n",
 724                             (parms->tx_max_key_sz_in_bits));
 725                 return -EINVAL;
 726         }
 727         /* Rx */
 728         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 729                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 730         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 731                 parms->rx_max_key_sz_in_bits / 8;
 732
 733         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 734                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 735         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 736                 parms->rx_max_key_sz_in_bits / 8;
 737
 738         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 739                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 740         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 741                 parms->rx_max_action_entry_sz_in_bits / 8;
 742
 743         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries = 0;
 744
 745         /* Tx */
 746         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 747                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 748         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 749                 parms->tx_max_key_sz_in_bits / 8;
 750
 751         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 752                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 753         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 754                 parms->tx_max_key_sz_in_bits / 8;
 755
 756         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 757                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 758         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 759                 parms->tx_max_action_entry_sz_in_bits / 8;
 760
 761         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries = 0;
 762
 763         return 0;
 764 }
 765
 766 /** insert EEM entry API
 767  *
 768  * returns:
 769  *  0
 770  *  TF_ERR          - unable to get lock
 771  *
 772  * insert callback returns:
 773  *   0
 774  *   TF_ERR_EM_DUP  - key is already in table
 775  */
 776 static int
 777 tf_insert_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 778                     struct tf_insert_em_entry_parms *parms)
 779 {
 780         uint32_t mask;
 781         uint32_t key0_hash;
 782         uint32_t key1_hash;
 783         uint32_t key0_index;
 784         uint32_t key1_index;
 785         struct cfa_p4_eem_64b_entry key_entry;
 786         uint32_t index;
 787         enum hcapi_cfa_em_table_type table_type;
 788         uint32_t gfid;
 789         struct hcapi_cfa_hwop op;
 790         struct hcapi_cfa_key_tbl key_tbl;
 791         struct hcapi_cfa_key_data key_obj;
 792         struct hcapi_cfa_key_loc key_loc;
 793         uint64_t big_hash;
 794         int rc;
 795
 796         /* Get mask to use on hash */
 797         mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
 798
 799         if (!mask)
 800                 return -EINVAL;
 801
 802 #ifdef TF_EEM_DEBUG
 803         dump_raw((uint8_t *)parms->key, TF_HW_EM_KEY_MAX_SIZE + 4, "In Key");
 804 #endif
 805
 806         big_hash = hcapi_cfa_key_hash((uint64_t *)parms->key,
 807                                       (TF_HW_EM_KEY_MAX_SIZE + 4) * 8);
 808         key0_hash = (uint32_t)(big_hash >> 32);
 809         key1_hash = (uint32_t)(big_hash & 0xFFFFFFFF);
 810
 811         key0_index = key0_hash & mask;
 812         key1_index = key1_hash & mask;
 813
 814 #ifdef TF_EEM_DEBUG
 815         TFP_DRV_LOG(DEBUG, "Key0 hash:0x%08x\n", key0_hash);
 816         TFP_DRV_LOG(DEBUG, "Key1 hash:0x%08x\n", key1_hash);
 817 #endif
 818         /*
 819          * Use the "result" arg to populate all of the key entry then
 820          * store the byte swapped "raw" entry in a local copy ready
 821          * for insertion in to the table.
 822          */
 823         tf_em_create_key_entry((struct cfa_p4_eem_entry_hdr *)parms->em_record,
 824                                 ((uint8_t *)parms->key),
 825                                 &key_entry);
 826
 827         /*
 828          * Try to add to Key0 table, if that does not work then
 829          * try the key1 table.
 830          */
 831         index = key0_index;
 832         op.opcode = HCAPI_CFA_HWOPS_ADD;
 833         key_tbl.base0 = (uint8_t *)
 834                 &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE];
 835         key_obj.offset = (index * TF_EM_KEY_RECORD_SIZE) % TF_EM_PAGE_SIZE;
 836         key_obj.data = (uint8_t *)&key_entry;
 837         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 838
 839         rc = hcapi_cfa_key_hw_op(&op,
 840                                  &key_tbl,
 841                                  &key_obj,
 842                                  &key_loc);
 843
 844         if (rc == 0) {
 845                 table_type = TF_KEY0_TABLE;
 846         } else {
 847                 index = key1_index;
 848
 849                 key_tbl.base0 = (uint8_t *)
 850                 &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY1_TABLE];
 851                 key_obj.offset =
 852                         (index * TF_EM_KEY_RECORD_SIZE) % TF_EM_PAGE_SIZE;
 853
 854                 rc = hcapi_cfa_key_hw_op(&op,
 855                                          &key_tbl,
 856                                          &key_obj,
 857                                          &key_loc);
 858                 if (rc != 0)
 859                         return rc;
 860
 861                 table_type = TF_KEY1_TABLE;
 862         }
 863
 864         TF_SET_GFID(gfid,
 865                     index,
 866                     table_type);
 867         TF_SET_FLOW_ID(parms->flow_id,
 868                        gfid,
 869                        TF_GFID_TABLE_EXTERNAL,
 870                        parms->dir);
 871         TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
 872                                      0,
 873                                      0,
 874                                      0,
 875                                      index,
 876                                      0,
 877                                      table_type);
 878
 879         return 0;
 880 }
 881
 882 /** delete EEM hash entry API
 883  *
 884  * returns:
 885  *   0
 886  *   -EINVAL      - parameter error
 887  *   TF_NO_SESSION    - bad session ID
 888  *   TF_ERR_TBL_SCOPE - invalid table scope
 889  *   TF_ERR_TBL_IF    - invalid table interface
 890  *
 891  * insert callback returns
 892  *   0
 893  *   TF_NO_EM_MATCH - entry not found
 894  */
 895 static int
 896 tf_delete_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 897                     struct tf_delete_em_entry_parms *parms)
 898 {
 899         enum hcapi_cfa_em_table_type hash_type;
 900         uint32_t index;
 901         struct hcapi_cfa_hwop op;
 902         struct hcapi_cfa_key_tbl key_tbl;
 903         struct hcapi_cfa_key_data key_obj;
 904         struct hcapi_cfa_key_loc key_loc;
 905         int rc;
 906
 907         if (parms->flow_handle == 0)
 908                 return -EINVAL;
 909
 910         TF_GET_HASH_TYPE_FROM_FLOW_HANDLE(parms->flow_handle, hash_type);
 911         TF_GET_INDEX_FROM_FLOW_HANDLE(parms->flow_handle, index);
 912
 913         op.opcode = HCAPI_CFA_HWOPS_DEL;
 914         key_tbl.base0 = (uint8_t *)
 915         &tbl_scope_cb->em_ctx_info[parms->dir].em_tables[(hash_type == 0 ?
 916                                                           TF_KEY0_TABLE :
 917                                                           TF_KEY1_TABLE)];
 918         key_obj.offset = (index * TF_EM_KEY_RECORD_SIZE) % TF_EM_PAGE_SIZE;
 919         key_obj.data = NULL;
 920         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 921
 922         rc = hcapi_cfa_key_hw_op(&op,
 923                                  &key_tbl,
 924                                  &key_obj,
 925                                  &key_loc);
 926
 927         if (!rc)
 928                 return rc;
 929
 930         return 0;
 931 }
 932
 933 /** insert EM hash entry API
 934  *
 935  *    returns:
 936  *    0       - Success
 937  *    -EINVAL - Error
 938  */
 939 int
 940 tf_em_insert_ext_entry(struct tf *tfp __rte_unused,
 941                        struct tf_insert_em_entry_parms *parms)
 942 {
 943         struct tf_tbl_scope_cb *tbl_scope_cb;
 944
 945         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 946         if (tbl_scope_cb == NULL) {
 947                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 948                 return -EINVAL;
 949         }
 950
 951         return tf_insert_eem_entry(tbl_scope_cb, parms);
 952 }
 953
 954 /** Delete EM hash entry API
 955  *
 956  *    returns:
 957  *    0       - Success
 958  *    -EINVAL - Error
 959  */
 960 int
 961 tf_em_delete_ext_entry(struct tf *tfp __rte_unused,
 962                        struct tf_delete_em_entry_parms *parms)
 963 {
 964         struct tf_tbl_scope_cb *tbl_scope_cb;
 965
 966         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 967         if (tbl_scope_cb == NULL) {
 968                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 969                 return -EINVAL;
 970         }
 971
 972         return tf_delete_eem_entry(tbl_scope_cb, parms);
 973 }
 974
 975 int
 976 tf_em_ext_host_alloc(struct tf *tfp,
 977                      struct tf_alloc_tbl_scope_parms *parms)
 978 {
 979         int rc;
 980         enum tf_dir dir;
 981         struct tf_tbl_scope_cb *tbl_scope_cb;
 982         struct hcapi_cfa_em_table *em_tables;
 983         struct tf_free_tbl_scope_parms free_parms;
 984         struct tf_rm_allocate_parms aparms = { 0 };
 985         struct tf_rm_free_parms fparms = { 0 };
 986
 987         /* Get Table Scope control block from the session pool */
 988         aparms.rm_db = eem_db[TF_DIR_RX];
 989         aparms.db_index = TF_EEM_DB_TBL_SCOPE;
 990         aparms.index = (uint32_t *)&parms->tbl_scope_id;
 991         rc = tf_rm_allocate(&aparms);
 992         if (rc) {
 993                 TFP_DRV_LOG(ERR,
 994                             "Failed to allocate table scope\n");
 995                 return rc;
 996         }
 997
 998         tbl_scope_cb = &tbl_scopes[parms->tbl_scope_id];
 999         tbl_scope_cb->index = parms->tbl_scope_id;
1000         tbl_scope_cb->tbl_scope_id = parms->tbl_scope_id;
1001
1002         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1003                 rc = tf_msg_em_qcaps(tfp,
1004                                      dir,
1005                                      &tbl_scope_cb->em_caps[dir]);
1006                 if (rc) {
1007                         TFP_DRV_LOG(ERR,
1008                                     "EEM: Unable to query for EEM capability,"
1009                                     " rc:%s\n",
1010                                     strerror(-rc));
1011                         goto cleanup;
1012                 }
1013         }
1014
1015         /*
1016          * Validate and setup table sizes
1017          */
1018         if (tf_em_validate_num_entries(tbl_scope_cb, parms))
1019                 goto cleanup;
1020
1021         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1022                 /*
1023                  * Allocate tables and signal configuration to FW
1024                  */
1025                 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
1026                 if (rc) {
1027                         TFP_DRV_LOG(ERR,
1028                                     "EEM: Unable to register for EEM ctx,"
1029                                     " rc:%s\n",
1030                                     strerror(-rc));
1031                         goto cleanup;
1032                 }
1033
1034                 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
1035                 rc = tf_msg_em_cfg(tfp,
1036                                    em_tables[TF_KEY0_TABLE].num_entries,
1037                                    em_tables[TF_KEY0_TABLE].ctx_id,
1038                                    em_tables[TF_KEY1_TABLE].ctx_id,
1039                                    em_tables[TF_RECORD_TABLE].ctx_id,
1040                                    em_tables[TF_EFC_TABLE].ctx_id,
1041                                    parms->hw_flow_cache_flush_timer,
1042                                    dir);
1043                 if (rc) {
1044                         TFP_DRV_LOG(ERR,
1045                                     "TBL: Unable to configure EEM in firmware"
1046                                     " rc:%s\n",
1047                                     strerror(-rc));
1048                         goto cleanup_full;
1049                 }
1050
1051                 rc = tf_msg_em_op(tfp,
1052                                   dir,
1053                                   HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
1054
1055                 if (rc) {
1056                         TFP_DRV_LOG(ERR,
1057                                     "EEM: Unable to enable EEM in firmware"
1058                                     " rc:%s\n",
1059                                     strerror(-rc));
1060                         goto cleanup_full;
1061                 }
1062
1063                 /* Allocate the pool of offsets of the external memory.
1064                  * Initially, this is a single fixed size pool for all external
1065                  * actions related to a single table scope.
1066                  */
1067                 rc = tf_create_tbl_pool_external(dir,
1068                                             tbl_scope_cb,
1069                                             em_tables[TF_RECORD_TABLE].num_entries,
1070                                             em_tables[TF_RECORD_TABLE].entry_size);
1071                 if (rc) {
1072                         TFP_DRV_LOG(ERR,
1073                                     "%s TBL: Unable to allocate idx pools %s\n",
1074                                     tf_dir_2_str(dir),
1075                                     strerror(-rc));
1076                         goto cleanup_full;
1077                 }
1078         }
1079
1080         return 0;
1081
1082 cleanup_full:
1083         free_parms.tbl_scope_id = parms->tbl_scope_id;
1084         tf_em_ext_host_free(tfp, &free_parms);
1085         return -EINVAL;
1086
1087 cleanup:
1088         /* Free Table control block */
1089         fparms.rm_db = eem_db[TF_DIR_RX];
1090         fparms.db_index = TF_EEM_DB_TBL_SCOPE;
1091         fparms.index = parms->tbl_scope_id;
1092         tf_rm_free(&fparms);
1093         return -EINVAL;
1094 }
1095
1096 int
1097 tf_em_ext_host_free(struct tf *tfp,
1098                     struct tf_free_tbl_scope_parms *parms)
1099 {
1100         int rc = 0;
1101         enum tf_dir  dir;
1102         struct tf_tbl_scope_cb *tbl_scope_cb;
1103         struct tf_rm_free_parms aparms = { 0 };
1104
1105         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
1106
1107         if (tbl_scope_cb == NULL) {
1108                 TFP_DRV_LOG(ERR, "Table scope error\n");
1109                 return -EINVAL;
1110         }
1111
1112         /* Free Table control block */
1113         aparms.rm_db = eem_db[TF_DIR_RX];
1114         aparms.db_index = TF_EEM_DB_TBL_SCOPE;
1115         aparms.index = parms->tbl_scope_id;
1116         rc = tf_rm_free(&aparms);
1117         if (rc) {
1118                 TFP_DRV_LOG(ERR,
1119                             "Failed to free table scope\n");
1120         }
1121
1122         /* free table scope locks */
1123         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1124                 /* Free associated external pools
1125                  */
1126                 tf_destroy_tbl_pool_external(dir,
1127                                              tbl_scope_cb);
1128                 tf_msg_em_op(tfp,
1129                              dir,
1130                              HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
1131
1132                 /* free table scope and all associated resources */
1133                 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
1134         }
1135
1136         tbl_scopes[parms->tbl_scope_id].tbl_scope_id = -1;
1137         return rc;
1138 }