drivers/net/bnxt/tf_core/tf_em_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <math.h>
   8 #include <sys/param.h>
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12
  13 #include "tf_core.h"
  14 #include "tf_util.h"
  15 #include "tf_common.h"
  16 #include "tf_em.h"
  17 #include "tf_em_common.h"
  18 #include "tf_msg.h"
  19 #include "tfp.h"
  20 #include "tf_device.h"
  21 #include "tf_ext_flow_handle.h"
  22 #include "cfa_resource_types.h"
  23
  24 #include "bnxt.h"
  25
  26 /* Number of pointers per page_size */
  27 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  28
  29 /**
  30  * EM DBs.
  31  */
  32 void *eem_db[TF_DIR_MAX];
  33
  34 /**
  35  * Init flag, set on bind and cleared on unbind
  36  */
  37 static uint8_t init;
  38
  39 /**
  40  * Host or system
  41  */
  42 static enum tf_mem_type mem_type;
  43
  44 /** Table scope array */
  45 struct tf_tbl_scope_cb tbl_scopes[TF_NUM_TBL_SCOPE];
  46
  47 /** Table scope reversal table
  48  *
  49  * Table scope are allocated from 15 to 0 within HCAPI RM.  Because of the
  50  * association between PFs and legacy table scopes, reverse table scope ids.
  51  * 15 indicates 0, 14 indicates 1, etc... The application will only see the 0
  52  * based number.  The firmware will only use the 0 based number.  Only HCAPI RM
  53  * and Truflow RM believe the number is 15.  When HCAPI RM support allocation
  54  * from low to high is supported, this adjust function can be removed.
  55  */
  56 const uint32_t tbl_scope_reverse[TF_NUM_TBL_SCOPE] = {
  57         15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
  58
  59 static uint32_t
  60 tf_tbl_scope_adjust(uint32_t tbl_scope_id)
  61 {
  62         if (tbl_scope_id < TF_NUM_TBL_SCOPE)
  63                 return tbl_scope_reverse[tbl_scope_id];
  64         else
  65                 return TF_TBL_SCOPE_INVALID;
  66 };
  67
  68
  69 /* API defined in tf_em.h */
  70 struct tf_tbl_scope_cb *
  71 tbl_scope_cb_find(uint32_t tbl_scope_id)
  72 {
  73         int i;
  74         struct tf_rm_is_allocated_parms parms = { 0 };
  75         int allocated;
  76         uint32_t rm_tbl_scope_id;
  77
  78         rm_tbl_scope_id = tf_tbl_scope_adjust(tbl_scope_id);
  79
  80         if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID)
  81                 return NULL;
  82
  83         /* Check that id is valid */
  84         parms.rm_db = eem_db[TF_DIR_RX];
  85         parms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
  86         parms.index = rm_tbl_scope_id;
  87         parms.allocated = &allocated;
  88
  89         i = tf_rm_is_allocated(&parms);
  90
  91         if (i < 0 || allocated != TF_RM_ALLOCATED_ENTRY_IN_USE)
  92                 return NULL;
  93
  94         for (i = 0; i < TF_NUM_TBL_SCOPE; i++) {
  95                 if (tbl_scopes[i].tbl_scope_id == tbl_scope_id)
  96                         return &tbl_scopes[i];
  97         }
  98
  99         return NULL;
 100 }
 101
 102 int tf_tbl_scope_alloc(uint32_t *tbl_scope_id)
 103 {
 104         int rc;
 105         struct tf_rm_allocate_parms parms = { 0 };
 106         uint32_t rm_tbl_scope_id;
 107         uint32_t usr_tbl_scope_id = TF_TBL_SCOPE_INVALID;
 108
 109         /* Get Table Scope control block from the session pool */
 110         parms.rm_db = eem_db[TF_DIR_RX];
 111         parms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
 112         parms.index = &rm_tbl_scope_id;
 113
 114         rc = tf_rm_allocate(&parms);
 115         if (rc) {
 116                 TFP_DRV_LOG(ERR,
 117                             "Failed to allocate table scope rc:%s\n",
 118                             strerror(-rc));
 119                 return rc;
 120         }
 121
 122         usr_tbl_scope_id = tf_tbl_scope_adjust(rm_tbl_scope_id);
 123
 124         if (usr_tbl_scope_id == TF_TBL_SCOPE_INVALID) {
 125                 TFP_DRV_LOG(ERR,
 126                             "Invalid table scope allocated id:%d\n",
 127                             (int)rm_tbl_scope_id);
 128                 return -EINVAL;
 129         }
 130         *tbl_scope_id = usr_tbl_scope_id;
 131         return 0;
 132 };
 133
 134 int tf_tbl_scope_free(uint32_t tbl_scope_id)
 135 {
 136         struct tf_rm_free_parms parms = { 0 };
 137         uint32_t rm_tbl_scope_id;
 138         uint32_t rc;
 139
 140         rm_tbl_scope_id = tf_tbl_scope_adjust(tbl_scope_id);
 141
 142         if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID) {
 143                 TFP_DRV_LOG(ERR,
 144                             "Invalid table scope allocated id:%d\n",
 145                             (int)tbl_scope_id);
 146                 return -EINVAL;
 147         }
 148
 149         parms.rm_db = eem_db[TF_DIR_RX];
 150         parms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
 151         parms.index = rm_tbl_scope_id;
 152
 153         rc = tf_rm_free(&parms);
 154         return rc;
 155 };
 156
 157 int
 158 tf_create_tbl_pool_external(enum tf_dir dir,
 159                             struct tf_tbl_scope_cb *tbl_scope_cb,
 160                             uint32_t num_entries,
 161                             uint32_t entry_sz_bytes)
 162 {
 163         struct tfp_calloc_parms parms;
 164         uint32_t i;
 165         int32_t j;
 166         int rc = 0;
 167         struct stack *pool = &tbl_scope_cb->ext_act_pool[dir];
 168
 169         parms.nitems = num_entries;
 170         parms.size = sizeof(uint32_t);
 171         parms.alignment = 0;
 172
 173         if (tfp_calloc(&parms) != 0) {
 174                 TFP_DRV_LOG(ERR, "%s: TBL: external pool failure %s\n",
 175                             tf_dir_2_str(dir), strerror(ENOMEM));
 176                 return -ENOMEM;
 177         }
 178
 179         /* Create empty stack
 180          */
 181         rc = stack_init(num_entries, parms.mem_va, pool);
 182
 183         if (rc != 0) {
 184                 TFP_DRV_LOG(ERR, "%s: TBL: stack init failure %s\n",
 185                             tf_dir_2_str(dir), strerror(-rc));
 186                 goto cleanup;
 187         }
 188
 189         /* Save the  malloced memory address so that it can
 190          * be freed when the table scope is freed.
 191          */
 192         tbl_scope_cb->ext_act_pool_mem[dir] = (uint32_t *)parms.mem_va;
 193
 194         /* Fill pool with indexes in reverse
 195          */
 196         j = (num_entries - 1) * entry_sz_bytes;
 197
 198         for (i = 0; i < num_entries; i++) {
 199                 rc = stack_push(pool, j);
 200                 if (rc != 0) {
 201                         TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
 202                                     tf_dir_2_str(dir), strerror(-rc));
 203                         goto cleanup;
 204                 }
 205
 206                 if (j < 0) {
 207                         TFP_DRV_LOG(ERR, "%d TBL: invalid offset (%d)\n",
 208                                     dir, j);
 209                         goto cleanup;
 210                 }
 211                 j -= entry_sz_bytes;
 212         }
 213
 214         if (!stack_is_full(pool)) {
 215                 rc = -EINVAL;
 216                 TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
 217                             tf_dir_2_str(dir), strerror(-rc));
 218                 goto cleanup;
 219         }
 220         return 0;
 221 cleanup:
 222         tfp_free((void *)parms.mem_va);
 223         return rc;
 224 }
 225
 226 /**
 227  * Destroy External Tbl pool of memory indexes.
 228  *
 229  * [in] dir
 230  *   direction
 231  * [in] tbl_scope_cb
 232  *   pointer to the table scope
 233  */
 234 void
 235 tf_destroy_tbl_pool_external(enum tf_dir dir,
 236                              struct tf_tbl_scope_cb *tbl_scope_cb)
 237 {
 238         uint32_t *ext_act_pool_mem =
 239                 tbl_scope_cb->ext_act_pool_mem[dir];
 240
 241         tfp_free(ext_act_pool_mem);
 242 }
 243
 244 /**
 245  * Allocate External Tbl entry from the scope pool.
 246  *
 247  * [in] tfp
 248  *   Pointer to Truflow Handle
 249  * [in] parms
 250  *   Allocation parameters
 251  *
 252  * Return:
 253  *  0       - Success, entry allocated - no search support
 254  *  -ENOMEM -EINVAL -EOPNOTSUPP
 255  *          - Failure, entry not allocated, out of resources
 256  */
 257 int
 258 tf_tbl_ext_alloc(struct tf *tfp,
 259                  struct tf_tbl_alloc_parms *parms)
 260 {
 261         int rc;
 262         uint32_t index;
 263         struct tf_tbl_scope_cb *tbl_scope_cb;
 264         struct stack *pool;
 265
 266         TF_CHECK_PARMS2(tfp, parms);
 267
 268         /* Get the pool info from the table scope
 269          */
 270         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 271
 272         if (tbl_scope_cb == NULL) {
 273                 TFP_DRV_LOG(ERR,
 274                             "%s, table scope not allocated\n",
 275                             tf_dir_2_str(parms->dir));
 276                 return -EINVAL;
 277         }
 278         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
 279
 280         /* Allocate an element
 281          */
 282         rc = stack_pop(pool, &index);
 283
 284         if (rc != 0) {
 285                 TFP_DRV_LOG(ERR,
 286                    "%s, Allocation failed, type:%d\n",
 287                    tf_dir_2_str(parms->dir),
 288                    parms->type);
 289                 return rc;
 290         }
 291
 292         *parms->idx = index;
 293         return rc;
 294 }
 295
 296 /**
 297  * Free External Tbl entry to the scope pool.
 298  *
 299  * [in] tfp
 300  *   Pointer to Truflow Handle
 301  * [in] parms
 302  *   Allocation parameters
 303  *
 304  * Return:
 305  *  0       - Success, entry freed
 306  *
 307  * - Failure, entry not successfully freed for these reasons
 308  *  -ENOMEM
 309  *  -EOPNOTSUPP
 310  *  -EINVAL
 311  */
 312 int
 313 tf_tbl_ext_free(struct tf *tfp,
 314                 struct tf_tbl_free_parms *parms)
 315 {
 316         int rc = 0;
 317         uint32_t index;
 318         struct tf_tbl_scope_cb *tbl_scope_cb;
 319         struct stack *pool;
 320
 321         TF_CHECK_PARMS2(tfp, parms);
 322
 323         /* Get the pool info from the table scope
 324          */
 325         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 326
 327         if (tbl_scope_cb == NULL) {
 328                 TFP_DRV_LOG(ERR,
 329                             "%s, table scope error\n",
 330                             tf_dir_2_str(parms->dir));
 331                 return -EINVAL;
 332         }
 333         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
 334
 335         index = parms->idx;
 336
 337         rc = stack_push(pool, index);
 338
 339         if (rc != 0) {
 340                 TFP_DRV_LOG(ERR,
 341                    "%s, consistency error, stack full, type:%d, idx:%d\n",
 342                    tf_dir_2_str(parms->dir),
 343                    parms->type,
 344                    index);
 345         }
 346         return rc;
 347 }
 348
 349 uint32_t
 350 tf_em_get_key_mask(int num_entries)
 351 {
 352         uint32_t mask = num_entries - 1;
 353
 354         if (num_entries & TF_EM_MAX_MASK)
 355                 return 0;
 356
 357         if (num_entries > TF_EM_MAX_ENTRY)
 358                 return 0;
 359
 360         return mask;
 361 }
 362
 363 void
 364 tf_em_create_key_entry(struct cfa_p4_eem_entry_hdr *result,
 365                        uint8_t *in_key,
 366                        struct cfa_p4_eem_64b_entry *key_entry)
 367 {
 368         key_entry->hdr.word1 = result->word1;
 369         key_entry->hdr.pointer = result->pointer;
 370         memcpy(key_entry->key, in_key, TF_HW_EM_KEY_MAX_SIZE + 4);
 371 }
 372
 373
 374 /**
 375  * Return the number of page table pages needed to
 376  * reference the given number of next level pages.
 377  *
 378  * [in] num_pages
 379  *   Number of EM pages
 380  *
 381  * [in] page_size
 382  *   Size of each EM page
 383  *
 384  * Returns:
 385  *   Number of EM page table pages
 386  */
 387 static uint32_t
 388 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 389                      uint32_t page_size)
 390 {
 391         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 392                        MAX_PAGE_PTRS(page_size);
 393         return 0;
 394 }
 395
 396 /**
 397  * Given the number of data pages, page_size and the maximum
 398  * number of page table levels (already determined), size
 399  * the number of page table pages required at each level.
 400  *
 401  * [in] max_lvl
 402  *   Max number of levels
 403  *
 404  * [in] num_data_pages
 405  *   Number of EM data pages
 406  *
 407  * [in] page_size
 408  *   Size of an EM page
 409  *
 410  * [out] *page_cnt
 411  *   EM page count
 412  */
 413 static void
 414 tf_em_size_page_tbls(int max_lvl,
 415                      uint64_t num_data_pages,
 416                      uint32_t page_size,
 417                      uint32_t *page_cnt)
 418 {
 419         if (max_lvl == TF_PT_LVL_0) {
 420                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 421         } else if (max_lvl == TF_PT_LVL_1) {
 422                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 423                 page_cnt[TF_PT_LVL_0] =
 424                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 425         } else if (max_lvl == TF_PT_LVL_2) {
 426                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 427                 page_cnt[TF_PT_LVL_1] =
 428                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 429                 page_cnt[TF_PT_LVL_0] =
 430                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 431         } else {
 432                 return;
 433         }
 434 }
 435
 436 /**
 437  * Given the page size, size of each data item (entry size),
 438  * and the total number of entries needed, determine the number
 439  * of page table levels and the number of data pages required.
 440  *
 441  * [in] page_size
 442  *   Page size
 443  *
 444  * [in] entry_size
 445  *   Entry size
 446  *
 447  * [in] num_entries
 448  *   Number of entries needed
 449  *
 450  * [out] num_data_pages
 451  *   Number of pages required
 452  *
 453  * Returns:
 454  *   Success  - Number of EM page levels required
 455  *   -ENOMEM  - Out of memory
 456  */
 457 static int
 458 tf_em_size_page_tbl_lvl(uint32_t page_size,
 459                         uint32_t entry_size,
 460                         uint32_t num_entries,
 461                         uint64_t *num_data_pages)
 462 {
 463         uint64_t lvl_data_size = page_size;
 464         int lvl = TF_PT_LVL_0;
 465         uint64_t data_size;
 466
 467         *num_data_pages = 0;
 468         data_size = (uint64_t)num_entries * entry_size;
 469
 470         while (lvl_data_size < data_size) {
 471                 lvl++;
 472
 473                 if (lvl == TF_PT_LVL_1)
 474                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 475                                 page_size;
 476                 else if (lvl == TF_PT_LVL_2)
 477                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 478                                 MAX_PAGE_PTRS(page_size) * page_size;
 479                 else
 480                         return -ENOMEM;
 481         }
 482
 483         *num_data_pages = roundup(data_size, page_size) / page_size;
 484
 485         return lvl;
 486 }
 487
 488 /**
 489  * Size the EM table based on capabilities
 490  *
 491  * [in] tbl
 492  *   EM table to size
 493  *
 494  * Returns:
 495  *   0        - Success
 496  *   - EINVAL - Parameter error
 497  *   - ENOMEM - Out of memory
 498  */
 499 int
 500 tf_em_size_table(struct hcapi_cfa_em_table *tbl,
 501                  uint32_t page_size)
 502 {
 503         uint64_t num_data_pages;
 504         uint32_t *page_cnt;
 505         int max_lvl;
 506         uint32_t num_entries;
 507         uint32_t cnt = TF_EM_MIN_ENTRIES;
 508
 509         /* Ignore entry if both size and number are zero */
 510         if (!tbl->entry_size && !tbl->num_entries)
 511                 return 0;
 512
 513         /* If only one is set then error */
 514         if (!tbl->entry_size || !tbl->num_entries)
 515                 return -EINVAL;
 516
 517         /* Determine number of page table levels and the number
 518          * of data pages needed to process the given eem table.
 519          */
 520         if (tbl->type == TF_RECORD_TABLE) {
 521                 /*
 522                  * For action records just a memory size is provided. Work
 523                  * backwards to resolve to number of entries
 524                  */
 525                 num_entries = tbl->num_entries / tbl->entry_size;
 526                 if (num_entries < TF_EM_MIN_ENTRIES) {
 527                         num_entries = TF_EM_MIN_ENTRIES;
 528                 } else {
 529                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 530                                 cnt *= 2;
 531                         num_entries = cnt;
 532                 }
 533         } else {
 534                 num_entries = tbl->num_entries;
 535         }
 536
 537         max_lvl = tf_em_size_page_tbl_lvl(page_size,
 538                                           tbl->entry_size,
 539                                           tbl->num_entries,
 540                                           &num_data_pages);
 541         if (max_lvl < 0) {
 542                 TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 543                 TFP_DRV_LOG(WARNING,
 544                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 545                             tbl->type, (uint64_t)num_entries * tbl->entry_size,
 546                             page_size);
 547                 return -ENOMEM;
 548         }
 549
 550         tbl->num_lvl = max_lvl + 1;
 551         tbl->num_data_pages = num_data_pages;
 552
 553         /* Determine the number of pages needed at each level */
 554         page_cnt = tbl->page_cnt;
 555         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 556         tf_em_size_page_tbls(max_lvl, num_data_pages, page_size,
 557                                 page_cnt);
 558
 559         TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 560         TFP_DRV_LOG(INFO,
 561                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 \
 562                     " l0: %u l1: %u l2: %u\n",
 563                     max_lvl + 1,
 564                     (uint64_t)num_data_pages * page_size,
 565                     num_data_pages,
 566                     page_cnt[TF_PT_LVL_0],
 567                     page_cnt[TF_PT_LVL_1],
 568                     page_cnt[TF_PT_LVL_2]);
 569
 570         return 0;
 571 }
 572
 573 /**
 574  * Validates EM number of entries requested
 575  *
 576  * [in] tbl_scope_cb
 577  *   Pointer to table scope control block to be populated
 578  *
 579  * [in] parms
 580  *   Pointer to input parameters
 581  *
 582  * Returns:
 583  *   0       - Success
 584  *   -EINVAL - Parameter error
 585  */
 586 int
 587 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 588                            struct tf_alloc_tbl_scope_parms *parms)
 589 {
 590         uint32_t cnt;
 591
 592         if (parms->rx_mem_size_in_mb != 0) {
 593                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 594                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 595                                      + 1);
 596                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 597                                         TF_MEGABYTE) / (key_b + action_b);
 598
 599                 if (num_entries < TF_EM_MIN_ENTRIES) {
 600                         TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 601                                     "%uMB\n",
 602                                     parms->rx_mem_size_in_mb);
 603                         return -EINVAL;
 604                 }
 605
 606                 cnt = TF_EM_MIN_ENTRIES;
 607                 while (num_entries > cnt &&
 608                        cnt <= TF_EM_MAX_ENTRIES)
 609                         cnt *= 2;
 610
 611                 if (cnt > TF_EM_MAX_ENTRIES) {
 612                         TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 613                                     "%u\n",
 614                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 615                         return -EINVAL;
 616                 }
 617
 618                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 619         } else {
 620                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 621                     TF_EM_MIN_ENTRIES ||
 622                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 623                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 624                         TFP_DRV_LOG(ERR,
 625                                     "EEM: Invalid number of Rx flows "
 626                                     "requested:%u max:%u\n",
 627                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 628                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 629                         return -EINVAL;
 630                 }
 631
 632                 /* must be a power-of-2 supported value
 633                  * in the range 32K - 128M
 634                  */
 635                 cnt = TF_EM_MIN_ENTRIES;
 636                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 637                        cnt <= TF_EM_MAX_ENTRIES)
 638                         cnt *= 2;
 639
 640                 if (cnt > TF_EM_MAX_ENTRIES) {
 641                         TFP_DRV_LOG(ERR,
 642                                     "EEM: Invalid number of Rx requested: %u\n",
 643                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 644                         return -EINVAL;
 645                 }
 646         }
 647
 648         if (parms->tx_mem_size_in_mb != 0) {
 649                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 650                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 651                                      + 1);
 652                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 653                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 654                         (key_b + action_b);
 655
 656                 if (num_entries < TF_EM_MIN_ENTRIES) {
 657                         TFP_DRV_LOG(ERR,
 658                                     "EEM: Insufficient memory requested:%uMB\n",
 659                                     parms->rx_mem_size_in_mb);
 660                         return -EINVAL;
 661                 }
 662
 663                 cnt = TF_EM_MIN_ENTRIES;
 664                 while (num_entries > cnt &&
 665                        cnt <= TF_EM_MAX_ENTRIES)
 666                         cnt *= 2;
 667
 668                 if (cnt > TF_EM_MAX_ENTRIES) {
 669                         TFP_DRV_LOG(ERR,
 670                                     "EEM: Invalid number of Tx requested: %u\n",
 671                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 672                         return -EINVAL;
 673                 }
 674
 675                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 676         } else {
 677                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 678                     TF_EM_MIN_ENTRIES ||
 679                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 680                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 681                         TFP_DRV_LOG(ERR,
 682                                     "EEM: Invalid number of Tx flows "
 683                                     "requested:%u max:%u\n",
 684                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 685                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 686                         return -EINVAL;
 687                 }
 688
 689                 cnt = TF_EM_MIN_ENTRIES;
 690                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 691                        cnt <= TF_EM_MAX_ENTRIES)
 692                         cnt *= 2;
 693
 694                 if (cnt > TF_EM_MAX_ENTRIES) {
 695                         TFP_DRV_LOG(ERR,
 696                                     "EEM: Invalid number of Tx requested: %u\n",
 697                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 698                         return -EINVAL;
 699                 }
 700         }
 701
 702         if (parms->rx_num_flows_in_k != 0 &&
 703             parms->rx_max_key_sz_in_bits / 8 == 0) {
 704                 TFP_DRV_LOG(ERR,
 705                             "EEM: Rx key size required: %u\n",
 706                             (parms->rx_max_key_sz_in_bits));
 707                 return -EINVAL;
 708         }
 709
 710         if (parms->tx_num_flows_in_k != 0 &&
 711             parms->tx_max_key_sz_in_bits / 8 == 0) {
 712                 TFP_DRV_LOG(ERR,
 713                             "EEM: Tx key size required: %u\n",
 714                             (parms->tx_max_key_sz_in_bits));
 715                 return -EINVAL;
 716         }
 717         /* Rx */
 718         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 719                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 720         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 721                 parms->rx_max_key_sz_in_bits / 8;
 722
 723         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 724                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 725         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 726                 parms->rx_max_key_sz_in_bits / 8;
 727
 728         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 729                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 730         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 731                 parms->rx_max_action_entry_sz_in_bits / 8;
 732
 733         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries = 0;
 734
 735         /* Tx */
 736         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 737                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 738         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 739                 parms->tx_max_key_sz_in_bits / 8;
 740
 741         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 742                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 743         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 744                 parms->tx_max_key_sz_in_bits / 8;
 745
 746         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 747                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 748         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 749                 parms->tx_max_action_entry_sz_in_bits / 8;
 750
 751         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries = 0;
 752
 753         return 0;
 754 }
 755
 756 /** insert EEM entry API
 757  *
 758  * returns:
 759  *  0
 760  *  TF_ERR          - unable to get lock
 761  *
 762  * insert callback returns:
 763  *   0
 764  *   TF_ERR_EM_DUP  - key is already in table
 765  */
 766 static int
 767 tf_insert_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 768                     struct tf_insert_em_entry_parms *parms)
 769 {
 770         uint32_t mask;
 771         uint32_t key0_hash;
 772         uint32_t key1_hash;
 773         uint32_t key0_index;
 774         uint32_t key1_index;
 775         struct cfa_p4_eem_64b_entry key_entry;
 776         uint32_t index;
 777         enum hcapi_cfa_em_table_type table_type;
 778         uint32_t gfid;
 779         struct hcapi_cfa_hwop op;
 780         struct hcapi_cfa_key_tbl key_tbl;
 781         struct hcapi_cfa_key_data key_obj;
 782         struct hcapi_cfa_key_loc key_loc;
 783         uint64_t big_hash;
 784         int rc;
 785
 786         /* Get mask to use on hash */
 787         mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
 788
 789         if (!mask)
 790                 return -EINVAL;
 791
 792 #ifdef TF_EEM_DEBUG
 793         dump_raw((uint8_t *)parms->key, TF_HW_EM_KEY_MAX_SIZE + 4, "In Key");
 794 #endif
 795
 796         big_hash = hcapi_cfa_key_hash((uint64_t *)parms->key,
 797                                       (TF_HW_EM_KEY_MAX_SIZE + 4) * 8);
 798         key0_hash = (uint32_t)(big_hash >> 32);
 799         key1_hash = (uint32_t)(big_hash & 0xFFFFFFFF);
 800
 801         key0_index = key0_hash & mask;
 802         key1_index = key1_hash & mask;
 803
 804 #ifdef TF_EEM_DEBUG
 805         TFP_DRV_LOG(DEBUG, "Key0 hash:0x%08x\n", key0_hash);
 806         TFP_DRV_LOG(DEBUG, "Key1 hash:0x%08x\n", key1_hash);
 807 #endif
 808         /*
 809          * Use the "result" arg to populate all of the key entry then
 810          * store the byte swapped "raw" entry in a local copy ready
 811          * for insertion in to the table.
 812          */
 813         tf_em_create_key_entry((struct cfa_p4_eem_entry_hdr *)parms->em_record,
 814                                 ((uint8_t *)parms->key),
 815                                 &key_entry);
 816
 817         /*
 818          * Try to add to Key0 table, if that does not work then
 819          * try the key1 table.
 820          */
 821         index = key0_index;
 822         op.opcode = HCAPI_CFA_HWOPS_ADD;
 823         key_tbl.base0 =
 824                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE];
 825         key_tbl.page_size = TF_EM_PAGE_SIZE;
 826         key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 827         key_obj.data = (uint8_t *)&key_entry;
 828         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 829
 830         rc = hcapi_cfa_key_hw_op(&op,
 831                                  &key_tbl,
 832                                  &key_obj,
 833                                  &key_loc);
 834
 835         if (rc == 0) {
 836                 table_type = TF_KEY0_TABLE;
 837         } else {
 838                 index = key1_index;
 839
 840                 key_tbl.base0 =
 841                         (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY1_TABLE];
 842                 key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 843
 844                 rc = hcapi_cfa_key_hw_op(&op,
 845                                          &key_tbl,
 846                                          &key_obj,
 847                                          &key_loc);
 848                 if (rc != 0)
 849                         return rc;
 850
 851                 table_type = TF_KEY1_TABLE;
 852         }
 853
 854         TF_SET_GFID(gfid,
 855                     index,
 856                     table_type);
 857         TF_SET_FLOW_ID(parms->flow_id,
 858                        gfid,
 859                        TF_GFID_TABLE_EXTERNAL,
 860                        parms->dir);
 861         TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
 862                                      0,
 863                                      0,
 864                                      0,
 865                                      index,
 866                                      0,
 867                                      table_type);
 868
 869         return 0;
 870 }
 871
 872 /** delete EEM hash entry API
 873  *
 874  * returns:
 875  *   0
 876  *   -EINVAL      - parameter error
 877  *   TF_NO_SESSION    - bad session ID
 878  *   TF_ERR_TBL_SCOPE - invalid table scope
 879  *   TF_ERR_TBL_IF    - invalid table interface
 880  *
 881  * insert callback returns
 882  *   0
 883  *   TF_NO_EM_MATCH - entry not found
 884  */
 885 static int
 886 tf_delete_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 887                     struct tf_delete_em_entry_parms *parms)
 888 {
 889         enum hcapi_cfa_em_table_type hash_type;
 890         uint32_t index;
 891         struct hcapi_cfa_hwop op;
 892         struct hcapi_cfa_key_tbl key_tbl;
 893         struct hcapi_cfa_key_data key_obj;
 894         struct hcapi_cfa_key_loc key_loc;
 895         int rc;
 896
 897         TF_GET_HASH_TYPE_FROM_FLOW_HANDLE(parms->flow_handle, hash_type);
 898         TF_GET_INDEX_FROM_FLOW_HANDLE(parms->flow_handle, index);
 899
 900         op.opcode = HCAPI_CFA_HWOPS_DEL;
 901         key_tbl.base0 =
 902                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables
 903                         [(hash_type == 0 ? TF_KEY0_TABLE : TF_KEY1_TABLE)];
 904         key_tbl.page_size = TF_EM_PAGE_SIZE;
 905         key_obj.offset = index * TF_EM_KEY_RECORD_SIZE;
 906         key_obj.data = NULL;
 907         key_obj.size = TF_EM_KEY_RECORD_SIZE;
 908
 909         rc = hcapi_cfa_key_hw_op(&op,
 910                                  &key_tbl,
 911                                  &key_obj,
 912                                  &key_loc);
 913
 914         if (!rc)
 915                 return rc;
 916
 917         return 0;
 918 }
 919
 920 /** insert EM hash entry API
 921  *
 922  *    returns:
 923  *    0       - Success
 924  *    -EINVAL - Error
 925  */
 926 int
 927 tf_em_insert_ext_entry(struct tf *tfp __rte_unused,
 928                        struct tf_insert_em_entry_parms *parms)
 929 {
 930         struct tf_tbl_scope_cb *tbl_scope_cb;
 931
 932         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 933         if (tbl_scope_cb == NULL) {
 934                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 935                 return -EINVAL;
 936         }
 937
 938         return tf_insert_eem_entry
 939                 (tbl_scope_cb,
 940                 parms);
 941 }
 942
 943 /** Delete EM hash entry API
 944  *
 945  *    returns:
 946  *    0       - Success
 947  *    -EINVAL - Error
 948  */
 949 int
 950 tf_em_delete_ext_entry(struct tf *tfp __rte_unused,
 951                        struct tf_delete_em_entry_parms *parms)
 952 {
 953         struct tf_tbl_scope_cb *tbl_scope_cb;
 954
 955         tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
 956         if (tbl_scope_cb == NULL) {
 957                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 958                 return -EINVAL;
 959         }
 960
 961         return tf_delete_eem_entry(tbl_scope_cb, parms);
 962 }
 963
 964
 965 int
 966 tf_em_ext_common_bind(struct tf *tfp,
 967                       struct tf_em_cfg_parms *parms)
 968 {
 969         int rc;
 970         int i;
 971         struct tf_rm_create_db_parms db_cfg = { 0 };
 972         uint8_t db_exists = 0;
 973
 974         TF_CHECK_PARMS2(tfp, parms);
 975
 976         if (init) {
 977                 TFP_DRV_LOG(ERR,
 978                             "EM Ext DB already initialized\n");
 979                 return -EINVAL;
 980         }
 981
 982         db_cfg.type = TF_DEVICE_MODULE_TYPE_EM;
 983         db_cfg.num_elements = parms->num_elements;
 984         db_cfg.cfg = parms->cfg;
 985
 986         for (i = 0; i < TF_DIR_MAX; i++) {
 987                 db_cfg.dir = i;
 988                 db_cfg.alloc_cnt = parms->resources->em_cnt[i].cnt;
 989
 990                 /* Check if we got any request to support EEM, if so
 991                  * we build an EM Ext DB holding Table Scopes.
 992                  */
 993                 if (db_cfg.alloc_cnt[TF_EM_TBL_TYPE_TBL_SCOPE] == 0)
 994                         continue;
 995
 996                 db_cfg.rm_db = &eem_db[i];
 997                 rc = tf_rm_create_db(tfp, &db_cfg);
 998                 if (rc) {
 999                         TFP_DRV_LOG(ERR,
1000                                     "%s: EM Ext DB creation failed\n",
1001                                     tf_dir_2_str(i));
1002
1003                         return rc;
1004                 }
1005                 db_exists = 1;
1006         }
1007
1008         if (db_exists)
1009                 init = 1;
1010
1011         mem_type = parms->mem_type;
1012
1013         return 0;
1014 }
1015
1016 int
1017 tf_em_ext_common_unbind(struct tf *tfp)
1018 {
1019         int rc;
1020         int i;
1021         struct tf_rm_free_db_parms fparms = { 0 };
1022
1023         TF_CHECK_PARMS1(tfp);
1024
1025         /* Bail if nothing has been initialized */
1026         if (!init) {
1027                 TFP_DRV_LOG(INFO,
1028                             "No EM Ext DBs created\n");
1029                 return 0;
1030         }
1031
1032         for (i = 0; i < TF_DIR_MAX; i++) {
1033                 fparms.dir = i;
1034                 fparms.rm_db = eem_db[i];
1035                 rc = tf_rm_free_db(tfp, &fparms);
1036                 if (rc)
1037                         return rc;
1038
1039                 eem_db[i] = NULL;
1040         }
1041
1042         init = 0;
1043
1044         return 0;
1045 }
1046
1047 /**
1048  * Sets the specified external table type element.
1049  *
1050  * This API sets the specified element data
1051  *
1052  * [in] tfp
1053  *   Pointer to TF handle
1054  *
1055  * [in] parms
1056  *   Pointer to table set parameters
1057  *
1058  * Returns
1059  *   - (0) if successful.
1060  *   - (-EINVAL) on failure.
1061  */
1062 int tf_tbl_ext_common_set(struct tf *tfp,
1063                           struct tf_tbl_set_parms *parms)
1064 {
1065         int rc = 0;
1066         struct tf_tbl_scope_cb *tbl_scope_cb;
1067         uint32_t tbl_scope_id;
1068         struct hcapi_cfa_hwop op;
1069         struct hcapi_cfa_key_tbl key_tbl;
1070         struct hcapi_cfa_key_data key_obj;
1071         struct hcapi_cfa_key_loc key_loc;
1072
1073         TF_CHECK_PARMS2(tfp, parms);
1074
1075         if (parms->data == NULL) {
1076                 TFP_DRV_LOG(ERR,
1077                             "%s, invalid parms->data\n",
1078                             tf_dir_2_str(parms->dir));
1079                 return -EINVAL;
1080         }
1081
1082         tbl_scope_id = parms->tbl_scope_id;
1083
1084         if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1085                 TFP_DRV_LOG(ERR,
1086                             "%s, Table scope not allocated\n",
1087                             tf_dir_2_str(parms->dir));
1088                 return -EINVAL;
1089         }
1090
1091         /* Get the table scope control block associated with the
1092          * external pool
1093          */
1094         tbl_scope_cb = tbl_scope_cb_find(tbl_scope_id);
1095
1096         if (tbl_scope_cb == NULL) {
1097                 TFP_DRV_LOG(ERR,
1098                             "%s, table scope error\n",
1099                             tf_dir_2_str(parms->dir));
1100                 return -EINVAL;
1101         }
1102
1103         op.opcode = HCAPI_CFA_HWOPS_PUT;
1104         key_tbl.base0 =
1105                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_RECORD_TABLE];
1106         key_tbl.page_size = TF_EM_PAGE_SIZE;
1107         key_obj.offset = parms->idx;
1108         key_obj.data = parms->data;
1109         key_obj.size = parms->data_sz_in_bytes;
1110
1111         rc = hcapi_cfa_key_hw_op(&op,
1112                                  &key_tbl,
1113                                  &key_obj,
1114                                  &key_loc);
1115
1116         return rc;
1117 }
1118
1119 int
1120 tf_em_ext_common_alloc(struct tf *tfp,
1121                        struct tf_alloc_tbl_scope_parms *parms)
1122 {
1123         return tf_em_ext_alloc(tfp, parms);
1124 }
1125
1126 int
1127 tf_em_ext_common_free(struct tf *tfp,
1128                       struct tf_free_tbl_scope_parms *parms)
1129 {
1130         return tf_em_ext_free(tfp, parms);
1131 }