drivers/net/bnxt/tf_core/tf_em_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <math.h>
   8 #include <sys/param.h>
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12
  13 #include "tf_core.h"
  14 #include "tf_util.h"
  15 #include "tf_common.h"
  16 #include "tf_em.h"
  17 #include "tf_em_common.h"
  18 #include "tf_msg.h"
  19 #include "tfp.h"
  20 #include "tf_device.h"
  21 #include "tf_ext_flow_handle.h"
  22 #include "cfa_resource_types.h"
  23
  24 #include "bnxt.h"
  25
  26 /* Number of pointers per page_size */
  27 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  28
  29 /**
  30  * EM DBs.
  31  */
  32 void *eem_db[TF_DIR_MAX];
  33
  34 /**
  35  * Init flag, set on bind and cleared on unbind
  36  */
  37 static uint8_t init;
  38
  39 /**
  40  * Host or system
  41  */
  42 static enum tf_mem_type mem_type;
  43
  44 /** Table scope array */
  45 struct tf_tbl_scope_cb tbl_scopes[TF_NUM_TBL_SCOPE];
  46
  47 /** Table scope reversal table
  48  *
  49  * Table scope are allocated from 15 to 0 within HCAPI RM.  Because of the
  50  * association between PFs and legacy table scopes, reverse table scope ids.
  51  * 15 indicates 0, 14 indicates 1, etc... The application will only see the 0
  52  * based number.  The firmware will only use the 0 based number.  Only HCAPI RM
  53  * and Truflow RM believe the number is 15.  When HCAPI RM support allocation
  54  * from low to high is supported, this adjust function can be removed.
  55  */
  56 const uint32_t tbl_scope_reverse[TF_NUM_TBL_SCOPE] = {
  57         15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
  58
  59 static uint32_t
  60 tf_tbl_scope_adjust(uint32_t tbl_scope_id)
  61 {
  62         if (tbl_scope_id < TF_NUM_TBL_SCOPE)
  63                 return tbl_scope_reverse[tbl_scope_id];
  64         else
  65                 return TF_TBL_SCOPE_INVALID;
  66 };
  67
  68
  69 /* API defined in tf_em.h */
  70 struct tf_tbl_scope_cb *
  71 tbl_scope_cb_find(uint32_t tbl_scope_id)
  72 {
  73         int i;
  74         struct tf_rm_is_allocated_parms parms = { 0 };
  75         int allocated;
  76         uint32_t rm_tbl_scope_id;
  77
  78         rm_tbl_scope_id = tf_tbl_scope_adjust(tbl_scope_id);
  79
  80         if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID)
  81                 return NULL;
  82
  83         /* Check that id is valid */
  84         parms.rm_db = eem_db[TF_DIR_RX];
  85         parms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
  86         parms.index = rm_tbl_scope_id;
  87         parms.allocated = &allocated;
  88
  89         i = tf_rm_is_allocated(&parms);
  90
  91         if (i < 0 || allocated != TF_RM_ALLOCATED_ENTRY_IN_USE)
  92                 return NULL;
  93
  94         for (i = 0; i < TF_NUM_TBL_SCOPE; i++) {
  95                 if (tbl_scopes[i].tbl_scope_id == tbl_scope_id)
  96                         return &tbl_scopes[i];
  97         }
  98
  99         return NULL;
 100 }
 101
 102 int tf_tbl_scope_alloc(uint32_t *tbl_scope_id)
 103 {
 104         int rc;
 105         struct tf_rm_allocate_parms parms = { 0 };
 106         uint32_t rm_tbl_scope_id;
 107         uint32_t usr_tbl_scope_id = TF_TBL_SCOPE_INVALID;
 108
 109         /* Get Table Scope control block from the session pool */
 110         parms.rm_db = eem_db[TF_DIR_RX];
 111         parms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
 112         parms.index = &rm_tbl_scope_id;
 113
 114         rc = tf_rm_allocate(&parms);
 115         if (rc) {
 116                 TFP_DRV_LOG(ERR,
 117                             "Failed to allocate table scope rc:%s\n",
 118                             strerror(-rc));
 119                 return rc;
 120         }
 121
 122         usr_tbl_scope_id = tf_tbl_scope_adjust(rm_tbl_scope_id);
 123
 124         if (usr_tbl_scope_id == TF_TBL_SCOPE_INVALID) {
 125                 TFP_DRV_LOG(ERR,
 126                             "Invalid table scope allocated id:%d\n",
 127                             (int)rm_tbl_scope_id);
 128                 return -EINVAL;
 129         }
 130         *tbl_scope_id = usr_tbl_scope_id;
 131         return 0;
 132 };
 133
 134 int tf_tbl_scope_free(uint32_t tbl_scope_id)
 135 {
 136         struct tf_rm_free_parms parms = { 0 };
 137         uint32_t rm_tbl_scope_id;
 138         uint32_t rc;
 139
 140         rm_tbl_scope_id = tf_tbl_scope_adjust(tbl_scope_id);
 141
 142         if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID) {
 143                 TFP_DRV_LOG(ERR,
 144                             "Invalid table scope allocated id:%d\n",
 145                             (int)tbl_scope_id);
 146                 return -EINVAL;
 147         }
 148
 149         parms.rm_db = eem_db[TF_DIR_RX];
 150         parms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
 151         parms.index = rm_tbl_scope_id;
 152
 153         rc = tf_rm_free(&parms);
 154         return rc;
 155 };
 156
 157 int
 158 tf_create_tbl_pool_external(enum tf_dir dir,
 159                             struct tf_tbl_scope_cb *tbl_scope_cb,
 160                             uint32_t num_entries,
 161                             uint32_t entry_sz_bytes)
 162 {
 163         struct tfp_calloc_parms parms;
 164         uint32_t i;
 165         int32_t j;
 166         int rc = 0;
 167         struct stack *pool = &tbl_scope_cb->ext_act_pool[dir];
 168
 169         parms.nitems = num_entries;
 170         parms.size = sizeof(uint32_t);
 171         parms.alignment = 0;
 172
 173         if (tfp_calloc(&parms) != 0) {
 174                 TFP_DRV_LOG(ERR, "%s: TBL: external pool failure %s\n",
 175                             tf_dir_2_str(dir), strerror(ENOMEM));
 176                 return -ENOMEM;
 177         }
 178
 179         /* Create empty stack
 180          */
 181         rc = stack_init(num_entries, parms.mem_va, pool);
 182
 183         if (rc != 0) {
 184                 TFP_DRV_LOG(ERR, "%s: TBL: stack init failure %s\n",
 185                             tf_dir_2_str(dir), strerror(-rc));
 186                 goto cleanup;
 187         }
 188
 189         /* Save the  malloced memory address so that it can
 190          * be freed when the table scope is freed.
 191          */
 192         tbl_scope_cb->ext_act_pool_mem[dir] = (uint32_t *)parms.mem_va;
 193
 194         /* Fill pool with indexes in reverse
 195          */
 196         j = (num_entries - 1) * entry_sz_bytes;
 197
 198         for (i = 0; i < num_entries; i++) {
 199                 rc = stack_push(pool, j);
 200                 if (rc != 0) {
 201                         TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
 202                                     tf_dir_2_str(dir), strerror(-rc));
 203                         goto cleanup;
 204                 }
 205
 206                 if (j < 0) {
 207                         TFP_DRV_LOG(ERR, "%d TBL: invalid offset (%d)\n",
 208                                     dir, j);
 209                         goto cleanup;
 210                 }
 211                 j -= entry_sz_bytes;
 212         }
 213
 214         if (!stack_is_full(pool)) {
 215                 rc = -EINVAL;
 216                 TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
 217                             tf_dir_2_str(dir), strerror(-rc));
 218                 goto cleanup;
 219         }
 220         return 0;
 221 cleanup:
 222         tfp_free((void *)parms.mem_va);
 223         return rc;
 224 }
 225
 226 /**
 227  * Destroy External Tbl pool of memory indexes.
 228  *
 229  * [in] dir
 230  *   direction
 231  * [in] tbl_scope_cb
 232  *   pointer to the table scope
 233  */
 234 void
 235 tf_destroy_tbl_pool_external(enum tf_dir dir,
 236                              struct tf_tbl_scope_cb *tbl_scope_cb)
 237 {
 238         uint32_t *ext_act_pool_mem =
 239                 tbl_scope_cb->ext_act_pool_mem[dir];
 240
 241         tfp_free(ext_act_pool_mem);
 242 }
 243
 244 /**
 245  * Allocate External Tbl entry from the scope pool.
 246  *
 247  * [in] tfp
 248  *   Pointer to Truflow Handle
 249  * [in] parms
 250  *   Allocation parameters
 251  *
 252  * Return:
 253  *  0       - Success, entry allocated - no search support
 254  *  -ENOMEM -EINVAL -EOPNOTSUPP
 255  *          - Failure, entry not allocated, out of resources
 256  */
 257 int
 258 tf_tbl_ext_alloc(struct tf *tfp,
 259                  struct tf_tbl_alloc_parms *parms)
 260 {
 261         int rc;
 262         uint32_t index;
 263         struct tf_tbl_scope_cb *tbl_scope_cb;
 264         struct stack *pool;
 265
 266         TF_CHECK_PARMS2(tfp, parms);
 267
 268         /* Get the pool info from the table scope
 269          */
 270         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 271
 272         if (tbl_scope_cb == NULL) {
 273                 TFP_DRV_LOG(ERR,
 274                             "%s, table scope not allocated\n",
 275                             tf_dir_2_str(parms->dir));
 276                 return -EINVAL;
 277         }
 278         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
 279
 280         /* Allocate an element
 281          */
 282         rc = stack_pop(pool, &index);
 283
 284         if (rc != 0) {
 285                 TFP_DRV_LOG(ERR,
 286                    "%s, Allocation failed, type:%d\n",
 287                    tf_dir_2_str(parms->dir),
 288                    parms->type);
 289                 return rc;
 290         }
 291
 292         *parms->idx = index;
 293         return rc;
 294 }
 295
 296 /**
 297  * Free External Tbl entry to the scope pool.
 298  *
 299  * [in] tfp
 300  *   Pointer to Truflow Handle
 301  * [in] parms
 302  *   Allocation parameters
 303  *
 304  * Return:
 305  *  0       - Success, entry freed
 306  *
 307  * - Failure, entry not successfully freed for these reasons
 308  *  -ENOMEM
 309  *  -EOPNOTSUPP
 310  *  -EINVAL
 311  */
 312 int
 313 tf_tbl_ext_free(struct tf *tfp,
 314                 struct tf_tbl_free_parms *parms)
 315 {
 316         int rc = 0;
 317         uint32_t index;
 318         struct tf_tbl_scope_cb *tbl_scope_cb;
 319         struct stack *pool;
 320
 321         TF_CHECK_PARMS2(tfp, parms);
 322
 323         /* Get the pool info from the table scope
 324          */
 325         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 326
 327         if (tbl_scope_cb == NULL) {
 328                 TFP_DRV_LOG(ERR,
 329                             "%s, table scope error\n",
 330                             tf_dir_2_str(parms->dir));
 331                 return -EINVAL;
 332         }
 333         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
 334
 335         index = parms->idx;
 336
 337         rc = stack_push(pool, index);
 338
 339         if (rc != 0) {
 340                 TFP_DRV_LOG(ERR,
 341                    "%s, consistency error, stack full, type:%d, idx:%d\n",
 342                    tf_dir_2_str(parms->dir),
 343                    parms->type,
 344                    index);
 345         }
 346         return rc;
 347 }
 348
 349 uint32_t
 350 tf_em_get_key_mask(int num_entries)
 351 {
 352         uint32_t mask = num_entries - 1;
 353
 354         if (num_entries & TF_EM_MAX_MASK)
 355                 return 0;
 356
 357         if (num_entries > TF_EM_MAX_ENTRY)
 358                 return 0;
 359
 360         return mask;
 361 }
 362
 363 void
 364 tf_em_create_key_entry(struct cfa_p4_eem_entry_hdr *result,
 365                        uint8_t *in_key,
 366                        struct cfa_p4_eem_64b_entry *key_entry)
 367 {
 368         key_entry->hdr.word1 = result->word1;
 369         key_entry->hdr.pointer = result->pointer;
 370         memcpy(key_entry->key, in_key, TF_HW_EM_KEY_MAX_SIZE + 4);
 371 }
 372
 373
 374 /**
 375  * Return the number of page table pages needed to
 376  * reference the given number of next level pages.
 377  *
 378  * [in] num_pages
 379  *   Number of EM pages
 380  *
 381  * [in] page_size
 382  *   Size of each EM page
 383  *
 384  * Returns:
 385  *   Number of EM page table pages
 386  */
 387 static uint32_t
 388 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 389                      uint32_t page_size)
 390 {
 391         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 392                        MAX_PAGE_PTRS(page_size);
 393         return 0;
 394 }
 395
 396 /**
 397  * Given the number of data pages, page_size and the maximum
 398  * number of page table levels (already determined), size
 399  * the number of page table pages required at each level.
 400  *
 401  * [in] max_lvl
 402  *   Max number of levels
 403  *
 404  * [in] num_data_pages
 405  *   Number of EM data pages
 406  *
 407  * [in] page_size
 408  *   Size of an EM page
 409  *
 410  * [out] *page_cnt
 411  *   EM page count
 412  */
 413 static void
 414 tf_em_size_page_tbls(int max_lvl,
 415                      uint64_t num_data_pages,
 416                      uint32_t page_size,
 417                      uint32_t *page_cnt)
 418 {
 419         if (max_lvl == TF_PT_LVL_0) {
 420                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 421         } else if (max_lvl == TF_PT_LVL_1) {
 422                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 423                 page_cnt[TF_PT_LVL_0] =
 424                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 425         } else if (max_lvl == TF_PT_LVL_2) {
 426                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 427                 page_cnt[TF_PT_LVL_1] =
 428                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 429                 page_cnt[TF_PT_LVL_0] =
 430                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 431         } else {
 432                 return;
 433         }
 434 }
 435
 436 /**
 437  * Given the page size, size of each data item (entry size),
 438  * and the total number of entries needed, determine the number
 439  * of page table levels and the number of data pages required.
 440  *
 441  * [in] page_size
 442  *   Page size
 443  *
 444  * [in] entry_size
 445  *   Entry size
 446  *
 447  * [in] num_entries
 448  *   Number of entries needed
 449  *
 450  * [out] num_data_pages
 451  *   Number of pages required
 452  *
 453  * Returns:
 454  *   Success  - Number of EM page levels required
 455  *   -ENOMEM  - Out of memory
 456  */
 457 static int
 458 tf_em_size_page_tbl_lvl(uint32_t page_size,
 459                         uint32_t entry_size,
 460                         uint32_t num_entries,
 461                         uint64_t *num_data_pages)
 462 {
 463         uint64_t lvl_data_size = page_size;
 464         int lvl = TF_PT_LVL_0;
 465         uint64_t data_size;
 466
 467         *num_data_pages = 0;
 468         data_size = (uint64_t)num_entries * entry_size;
 469
 470         while (lvl_data_size < data_size) {
 471                 lvl++;
 472
 473                 if (lvl == TF_PT_LVL_1)
 474                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 475                                 page_size;
 476                 else if (lvl == TF_PT_LVL_2)
 477                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 478                                 MAX_PAGE_PTRS(page_size) * page_size;
 479                 else
 480                         return -ENOMEM;
 481         }
 482
 483         *num_data_pages = roundup(data_size, page_size) / page_size;
 484
 485         return lvl;
 486 }
 487
 488 /**
 489  * Size the EM table based on capabilities
 490  *
 491  * [in] tbl
 492  *   EM table to size
 493  *
 494  * Returns:
 495  *   0        - Success
 496  *   - EINVAL - Parameter error
 497  *   - ENOMEM - Out of memory
 498  */
 499 int
 500 tf_em_size_table(struct hcapi_cfa_em_table *tbl,
 501                  uint32_t page_size)
 502 {
 503         uint64_t num_data_pages;
 504         uint32_t *page_cnt;
 505         int max_lvl;
 506         uint32_t num_entries;
 507         uint32_t cnt = TF_EM_MIN_ENTRIES;
 508
 509         /* Ignore entry if both size and number are zero */
 510         if (!tbl->entry_size && !tbl->num_entries)
 511                 return 0;
 512
 513         /* If only one is set then error */
 514         if (!tbl->entry_size || !tbl->num_entries)
 515                 return -EINVAL;
 516
 517         /* Determine number of page table levels and the number
 518          * of data pages needed to process the given eem table.
 519          */
 520         if (tbl->type == TF_RECORD_TABLE) {
 521                 /*
 522                  * For action records just a memory size is provided. Work
 523                  * backwards to resolve to number of entries
 524                  */
 525                 num_entries = tbl->num_entries / tbl->entry_size;
 526                 if (num_entries < TF_EM_MIN_ENTRIES) {
 527                         num_entries = TF_EM_MIN_ENTRIES;
 528                 } else {
 529                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 530                                 cnt *= 2;
 531                         num_entries = cnt;
 532                 }
 533         } else {
 534                 num_entries = tbl->num_entries;
 535         }
 536
 537         max_lvl = tf_em_size_page_tbl_lvl(page_size,
 538                                           tbl->entry_size,
 539                                           tbl->num_entries,
 540                                           &num_data_pages);
 541         if (max_lvl < 0) {
 542                 TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 543                 TFP_DRV_LOG(WARNING,
 544                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 545                             tbl->type, (uint64_t)num_entries * tbl->entry_size,
 546                             page_size);
 547                 return -ENOMEM;
 548         }
 549
 550         tbl->num_lvl = max_lvl + 1;
 551         tbl->num_data_pages = num_data_pages;
 552
 553         /* Determine the number of pages needed at each level */
 554         page_cnt = tbl->page_cnt;
 555         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 556         tf_em_size_page_tbls(max_lvl, num_data_pages, page_size,
 557                                 page_cnt);
 558
 559         TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 560         TFP_DRV_LOG(INFO,
 561                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 \
 562                     " l0: %u l1: %u l2: %u\n",
 563                     max_lvl + 1,
 564                     (uint64_t)num_data_pages * page_size,
 565                     num_data_pages,
 566                     page_cnt[TF_PT_LVL_0],
 567                     page_cnt[TF_PT_LVL_1],
 568                     page_cnt[TF_PT_LVL_2]);
 569
 570         return 0;
 571 }
 572
 573 /**
 574  * Validates EM number of entries requested
 575  *
 576  * [in] tbl_scope_cb
 577  *   Pointer to table scope control block to be populated
 578  *
 579  * [in] parms
 580  *   Pointer to input parameters
 581  *
 582  * Returns:
 583  *   0       - Success
 584  *   -EINVAL - Parameter error
 585  */
 586 int
 587 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 588                            struct tf_alloc_tbl_scope_parms *parms)
 589 {
 590         uint32_t cnt;
 591
 592         if (parms->rx_mem_size_in_mb != 0) {
 593                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 594                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 595                                      + 1);
 596                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 597                                         TF_MEGABYTE) / (key_b + action_b);
 598
 599                 if (num_entries < TF_EM_MIN_ENTRIES) {
 600                         TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 601                                     "%uMB\n",
 602                                     parms->rx_mem_size_in_mb);
 603                         return -EINVAL;
 604                 }
 605
 606                 cnt = TF_EM_MIN_ENTRIES;
 607                 while (num_entries > cnt &&
 608                        cnt <= TF_EM_MAX_ENTRIES)
 609                         cnt *= 2;
 610
 611                 if (cnt > TF_EM_MAX_ENTRIES) {
 612                         TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 613                                     "%u\n",
 614                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 615                         return -EINVAL;
 616                 }
 617
 618                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 619         } else {
 620                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 621                     TF_EM_MIN_ENTRIES ||
 622                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 623                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 624                         TFP_DRV_LOG(ERR,
 625                                     "EEM: Invalid number of Rx flows "
 626                                     "requested:%u max:%u\n",
 627                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 628                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 629                         return -EINVAL;
 630                 }
 631
 632                 /* must be a power-of-2 supported value
 633                  * in the range 32K - 128M
 634                  */
 635                 cnt = TF_EM_MIN_ENTRIES;
 636                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 637                        cnt <= TF_EM_MAX_ENTRIES)
 638                         cnt *= 2;
 639
 640                 if (cnt > TF_EM_MAX_ENTRIES) {
 641                         TFP_DRV_LOG(ERR,
 642                                     "EEM: Invalid number of Rx requested: %u\n",
 643                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 644                         return -EINVAL;
 645                 }
 646         }
 647
 648         if (parms->tx_mem_size_in_mb != 0) {
 649                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 650                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 651                                      + 1);
 652                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 653                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 654                         (key_b + action_b);
 655
 656                 if (num_entries < TF_EM_MIN_ENTRIES) {
 657                         TFP_DRV_LOG(ERR,
 658                                     "EEM: Insufficient memory requested:%uMB\n",
 659                                     parms->rx_mem_size_in_mb);
 660                         return -EINVAL;
 661                 }
 662
 663                 cnt = TF_EM_MIN_ENTRIES;
 664                 while (num_entries > cnt &&
 665                        cnt <= TF_EM_MAX_ENTRIES)
 666                         cnt *= 2;
 667
 668                 if (cnt > TF_EM_MAX_ENTRIES) {
 669                         TFP_DRV_LOG(ERR,
 670                                     "EEM: Invalid number of Tx requested: %u\n",
 671                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 672                         return -EINVAL;
 673                 }
 674
 675                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 676         } else {
 677                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 678                     TF_EM_MIN_ENTRIES ||
 679                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 680                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 681                         TFP_DRV_LOG(ERR,
 682                                     "EEM: Invalid number of Tx flows "
 683                                     "requested:%u max:%u\n",
 684                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 685                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 686                         return -EINVAL;
 687                 }
 688
 689                 cnt = TF_EM_MIN_ENTRIES;
 690                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 691                        cnt <= TF_EM_MAX_ENTRIES)
 692                         cnt *= 2;
 693
 694                 if (cnt > TF_EM_MAX_ENTRIES) {
 695                         TFP_DRV_LOG(ERR,
 696                                     "EEM: Invalid number of Tx requested: %u\n",
 697                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 698                         return -EINVAL;
 699                 }
 700         }
 701
 702         if (parms->rx_num_flows_in_k != 0 &&
 703             parms->rx_max_key_sz_in_bits / 8 == 0) {
 704                 TFP_DRV_LOG(ERR,
 705                             "EEM: Rx key size required: %u\n",
 706                             (parms->rx_max_key_sz_in_bits));
 707                 return -EINVAL;
 708         }
 709
 710         if (parms->tx_num_flows_in_k != 0 &&
 711             parms->tx_max_key_sz_in_bits / 8 == 0) {
 712                 TFP_DRV_LOG(ERR,
 713                             "EEM: Tx key size required: %u\n",
 714                             (parms->tx_max_key_sz_in_bits));
 715                 return -EINVAL;
 716         }
 717         /* Rx */
 718         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 719                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 720         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 721                 parms->rx_max_key_sz_in_bits / 8;
 722
 723         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 724                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 725         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 726                 parms->rx_max_key_sz_in_bits / 8;
 727
 728         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 729                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 730         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 731                 parms->rx_max_action_entry_sz_in_bits / 8;
 732
 733         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries = 0;
 734
 735         /* Tx */
 736         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 737                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 738         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 739                 parms->tx_max_key_sz_in_bits / 8;
 740
 741         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 742                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 743         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 744                 parms->tx_max_key_sz_in_bits / 8;
 745
 746         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 747                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 748         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 749                 parms->tx_max_action_entry_sz_in_bits / 8;
 750
 751         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries = 0;
 752
 753         return 0;
 754 }
 755
 756 /** insert EEM entry API
 757  *
 758  * returns:
 759  *  0
 760  *  TF_ERR          - unable to get lock
 761  *
 762  * insert callback returns:
 763  *   0
 764  *   TF_ERR_EM_DUP  - key is already in table
 765  */
 766 static int
 767 tf_insert_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 768                     struct tf_insert_em_entry_parms *parms)
 769 {
 770         uint32_t mask;
 771         uint32_t key0_hash;
 772         uint32_t key1_hash;
 773         uint32_t key0_index;
 774         uint32_t key1_index;
 775         struct cfa_p4_eem_64b_entry key_entry;
 776         uint32_t index;
 777         enum hcapi_cfa_em_table_type table_type;
 778         uint32_t gfid;
 779         struct hcapi_cfa_hwop op;
 780         struct hcapi_cfa_key_tbl key_tbl;
 781         struct hcapi_cfa_key_data key_obj;
 782         struct hcapi_cfa_key_loc key_loc;
 783         uint64_t big_hash;
 784         int rc;
 785
 786 #if (TF_EM_SYSMEM_DELAY_EXPORT == 1)
 787         if (!tbl_scope_cb->valid) {
 788                 rc = offload_system_mmap(tbl_scope_cb);
 789
 790                 if (rc) {
 791                         struct tf_rm_free_parms fparms = { 0 };
 792                         uint32_t rm_tbl_scope_id;
 793
 794                         TFP_DRV_LOG(ERR,
 795                                     "System alloc mmap failed\n");
 796
 797                         rm_tbl_scope_id =
 798                                 tf_tbl_scope_adjust(parms->tbl_scope_id);
 799
 800                         if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID)
 801                                 return -EINVAL;
 802
 803                         /* Free Table control block */
 804                         fparms.rm_db = eem_db[TF_DIR_RX];
 805                         fparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
 806                         fparms.index = parms->tbl_scope_id;
 807                         tf_rm_free(&fparms);
 808                         return -EINVAL;
 809                 }
 810
 811                 tbl_scope_cb->valid = true;
 812         }
 813 #endif
 814         /* Get mask to use on hash */
 815         mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
 816
 817         if (!mask)
 818                 return -EINVAL;
 819
 820 #ifdef TF_EEM_DEBUG
 821         dump_raw((uint8_t *)parms->key, TF_HW_EM_KEY_MAX_SIZE + 4, "In Key");
 822 #endif
 823
 824         big_hash = hcapi_cfa_key_hash((uint64_t *)parms->key,
 825                                       (TF_HW_EM_KEY_MAX_SIZE + 4) * 8);
 826         key0_hash = (uint32_t)(big_hash >> 32);
 827         key1_hash = (uint32_t)(big_hash & 0xFFFFFFFF);
 828
 829         key0_index = key0_hash & mask;
 830         key1_index = key1_hash & mask;
 831
 832 #ifdef TF_EEM_DEBUG
 833         TFP_DRV_LOG(DEBUG, "Key0 hash:0x%08x\n", key0_hash);
 834         TFP_DRV_LOG(DEBUG, "Key1 hash:0x%08x\n", key1_hash);
 835 #endif
 836         /*
 837          * Use the "result" arg to populate all of the key entry then
 838          * store the byte swapped "raw" entry in a local copy ready
 839          * for insertion in to the table.
 840          */
 841         tf_em_create_key_entry((struct cfa_p4_eem_entry_hdr *)parms->em_record,
 842                                 ((uint8_t *)parms->key),
 843                                 &key_entry);
 844
 845         /*
 846          * Try to add to Key0 table, if that does not work then
 847          * try the key1 table.
 848          */
 849         index = key0_index;
 850         op.opcode = HCAPI_CFA_HWOPS_ADD;
 851         key_tbl.base0 =
 852                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE];
 853         key_tbl.page_size = TF_EM_PAGE_SIZE;
 854         key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 855         key_obj.data = (uint8_t *)&key_entry;
 856         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 857
 858         rc = hcapi_cfa_key_hw_op(&op,
 859                                  &key_tbl,
 860                                  &key_obj,
 861                                  &key_loc);
 862
 863         if (rc == 0) {
 864                 table_type = TF_KEY0_TABLE;
 865         } else {
 866                 index = key1_index;
 867
 868                 key_tbl.base0 =
 869                         (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY1_TABLE];
 870                 key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 871
 872                 rc = hcapi_cfa_key_hw_op(&op,
 873                                          &key_tbl,
 874                                          &key_obj,
 875                                          &key_loc);
 876                 if (rc != 0)
 877                         return rc;
 878
 879                 table_type = TF_KEY1_TABLE;
 880         }
 881
 882         TF_SET_GFID(gfid,
 883                     index,
 884                     table_type);
 885         TF_SET_FLOW_ID(parms->flow_id,
 886                        gfid,
 887                        TF_GFID_TABLE_EXTERNAL,
 888                        parms->dir);
 889         TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
 890                                      0,
 891                                      0,
 892                                      0,
 893                                      index,
 894                                      0,
 895                                      table_type);
 896
 897         return 0;
 898 }
 899
 900 /** delete EEM hash entry API
 901  *
 902  * returns:
 903  *   0
 904  *   -EINVAL      - parameter error
 905  *   TF_NO_SESSION    - bad session ID
 906  *   TF_ERR_TBL_SCOPE - invalid table scope
 907  *   TF_ERR_TBL_IF    - invalid table interface
 908  *
 909  * insert callback returns
 910  *   0
 911  *   TF_NO_EM_MATCH - entry not found
 912  */
 913 static int
 914 tf_delete_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 915                     struct tf_delete_em_entry_parms *parms)
 916 {
 917         enum hcapi_cfa_em_table_type hash_type;
 918         uint32_t index;
 919         struct hcapi_cfa_hwop op;
 920         struct hcapi_cfa_key_tbl key_tbl;
 921         struct hcapi_cfa_key_data key_obj;
 922         struct hcapi_cfa_key_loc key_loc;
 923         int rc;
 924
 925         TF_GET_HASH_TYPE_FROM_FLOW_HANDLE(parms->flow_handle, hash_type);
 926         TF_GET_INDEX_FROM_FLOW_HANDLE(parms->flow_handle, index);
 927
 928         op.opcode = HCAPI_CFA_HWOPS_DEL;
 929         key_tbl.base0 =
 930                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables
 931                         [(hash_type == 0 ? TF_KEY0_TABLE : TF_KEY1_TABLE)];
 932         key_tbl.page_size = TF_EM_PAGE_SIZE;
 933         key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 934         key_obj.data = NULL;
 935         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 936
 937         rc = hcapi_cfa_key_hw_op(&op,
 938                                  &key_tbl,
 939                                  &key_obj,
 940                                  &key_loc);
 941
 942         if (!rc)
 943                 return rc;
 944
 945         return 0;
 946 }
 947
 948 /** insert EM hash entry API
 949  *
 950  *    returns:
 951  *    0       - Success
 952  *    -EINVAL - Error
 953  */
 954 int
 955 tf_em_insert_ext_entry(struct tf *tfp __rte_unused,
 956                        struct tf_insert_em_entry_parms *parms)
 957 {
 958         struct tf_tbl_scope_cb *tbl_scope_cb;
 959
 960         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 961         if (tbl_scope_cb == NULL) {
 962                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 963                 return -EINVAL;
 964         }
 965
 966         return tf_insert_eem_entry
 967                 (tbl_scope_cb,
 968                 parms);
 969 }
 970
 971 /** Delete EM hash entry API
 972  *
 973  *    returns:
 974  *    0       - Success
 975  *    -EINVAL - Error
 976  */
 977 int
 978 tf_em_delete_ext_entry(struct tf *tfp __rte_unused,
 979                        struct tf_delete_em_entry_parms *parms)
 980 {
 981         struct tf_tbl_scope_cb *tbl_scope_cb;
 982
 983         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 984         if (tbl_scope_cb == NULL) {
 985                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 986                 return -EINVAL;
 987         }
 988
 989         return tf_delete_eem_entry(tbl_scope_cb, parms);
 990 }
 991
 992
 993 int
 994 tf_em_ext_common_bind(struct tf *tfp,
 995                       struct tf_em_cfg_parms *parms)
 996 {
 997         int rc;
 998         int i;
 999         struct tf_rm_create_db_parms db_cfg = { 0 };
1000         uint8_t db_exists = 0;
1001
1002         TF_CHECK_PARMS2(tfp, parms);
1003
1004         if (init) {
1005                 TFP_DRV_LOG(ERR,
1006                             "EM Ext DB already initialized\n");
1007                 return -EINVAL;
1008         }
1009
1010         db_cfg.type = TF_DEVICE_MODULE_TYPE_EM;
1011         db_cfg.num_elements = parms->num_elements;
1012         db_cfg.cfg = parms->cfg;
1013
1014         for (i = 0; i < TF_DIR_MAX; i++) {
1015                 db_cfg.dir = i;
1016                 db_cfg.alloc_cnt = parms->resources->em_cnt[i].cnt;
1017
1018                 /* Check if we got any request to support EEM, if so
1019                  * we build an EM Ext DB holding Table Scopes.
1020                  */
1021                 if (db_cfg.alloc_cnt[TF_EM_TBL_TYPE_TBL_SCOPE] == 0)
1022                         continue;
1023
1024                 db_cfg.rm_db = &eem_db[i];
1025                 rc = tf_rm_create_db(tfp, &db_cfg);
1026                 if (rc) {
1027                         TFP_DRV_LOG(ERR,
1028                                     "%s: EM Ext DB creation failed\n",
1029                                     tf_dir_2_str(i));
1030
1031                         return rc;
1032                 }
1033                 db_exists = 1;
1034         }
1035
1036         if (db_exists)
1037                 init = 1;
1038
1039         mem_type = parms->mem_type;
1040
1041         return 0;
1042 }
1043
1044 int
1045 tf_em_ext_common_unbind(struct tf *tfp)
1046 {
1047         int rc;
1048         int i;
1049         struct tf_rm_free_db_parms fparms = { 0 };
1050
1051         TF_CHECK_PARMS1(tfp);
1052
1053         /* Bail if nothing has been initialized */
1054         if (!init) {
1055                 TFP_DRV_LOG(INFO,
1056                             "No EM Ext DBs created\n");
1057                 return 0;
1058         }
1059
1060         for (i = 0; i < TF_DIR_MAX; i++) {
1061                 fparms.dir = i;
1062                 fparms.rm_db = eem_db[i];
1063                 rc = tf_rm_free_db(tfp, &fparms);
1064                 if (rc)
1065                         return rc;
1066
1067                 eem_db[i] = NULL;
1068         }
1069
1070         init = 0;
1071
1072         return 0;
1073 }
1074
1075 /**
1076  * Sets the specified external table type element.
1077  *
1078  * This API sets the specified element data
1079  *
1080  * [in] tfp
1081  *   Pointer to TF handle
1082  *
1083  * [in] parms
1084  *   Pointer to table set parameters
1085  *
1086  * Returns
1087  *   - (0) if successful.
1088  *   - (-EINVAL) on failure.
1089  */
1090 int tf_tbl_ext_common_set(struct tf *tfp,
1091                           struct tf_tbl_set_parms *parms)
1092 {
1093         int rc = 0;
1094         struct tf_tbl_scope_cb *tbl_scope_cb;
1095         uint32_t tbl_scope_id;
1096         struct hcapi_cfa_hwop op;
1097         struct hcapi_cfa_key_tbl key_tbl;
1098         struct hcapi_cfa_key_data key_obj;
1099         struct hcapi_cfa_key_loc key_loc;
1100
1101         TF_CHECK_PARMS2(tfp, parms);
1102
1103         if (parms->data == NULL) {
1104                 TFP_DRV_LOG(ERR,
1105                             "%s, invalid parms->data\n",
1106                             tf_dir_2_str(parms->dir));
1107                 return -EINVAL;
1108         }
1109
1110         tbl_scope_id = parms->tbl_scope_id;
1111
1112         if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1113                 TFP_DRV_LOG(ERR,
1114                             "%s, Table scope not allocated\n",
1115                             tf_dir_2_str(parms->dir));
1116                 return -EINVAL;
1117         }
1118
1119         /* Get the table scope control block associated with the
1120          * external pool
1121          */
1122         tbl_scope_cb = tbl_scope_cb_find(tbl_scope_id);
1123
1124         if (tbl_scope_cb == NULL) {
1125                 TFP_DRV_LOG(ERR,
1126                             "%s, table scope error\n",
1127                             tf_dir_2_str(parms->dir));
1128                 return -EINVAL;
1129         }
1130
1131 #if (TF_EM_SYSMEM_DELAY_EXPORT == 1)
1132         if (!tbl_scope_cb->valid) {
1133                 rc = offload_system_mmap(tbl_scope_cb);
1134
1135                 if (rc) {
1136                         struct tf_rm_free_parms fparms = { 0 };
1137                         uint32_t rm_tbl_scope_id;
1138
1139                         /* TODO: support allocation of table scope from
1140                          * min in HCAPI RM.  For now call adjust function
1141                          * on value obtained from RM.
1142                          */
1143                         rm_tbl_scope_id =
1144                                 tf_tbl_scope_adjust(parms->tbl_scope_id);
1145
1146                         if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID)
1147                                 return -EINVAL;
1148
1149                         TFP_DRV_LOG(ERR,
1150                                     "System alloc mmap failed\n");
1151                         /* Free Table control block */
1152                         fparms.rm_db = eem_db[TF_DIR_RX];
1153                         fparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
1154                         fparms.index = rm_tbl_scope_id;
1155                         tf_rm_free(&fparms);
1156                         return -EINVAL;
1157                 }
1158
1159                 tbl_scope_cb->valid = true;
1160         }
1161 #endif
1162
1163         op.opcode = HCAPI_CFA_HWOPS_PUT;
1164         key_tbl.base0 =
1165                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_RECORD_TABLE];
1166         key_tbl.page_size = TF_EM_PAGE_SIZE;
1167         key_obj.offset = parms->idx;
1168         key_obj.data = parms->data;
1169         key_obj.size = parms->data_sz_in_bytes;
1170
1171         rc = hcapi_cfa_key_hw_op(&op,
1172                                  &key_tbl,
1173                                  &key_obj,
1174                                  &key_loc);
1175
1176         return rc;
1177 }
1178
1179 int
1180 tf_em_ext_common_alloc(struct tf *tfp,
1181                        struct tf_alloc_tbl_scope_parms *parms)
1182 {
1183         return tf_em_ext_alloc(tfp, parms);
1184 }
1185
1186 int
1187 tf_em_ext_common_free(struct tf *tfp,
1188                       struct tf_free_tbl_scope_parms *parms)
1189 {
1190         return tf_em_ext_free(tfp, parms);
1191 }