drivers/net/bnxt/tf_core/tf_em_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2021 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <math.h>
   8 #include <sys/param.h>
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12
  13 #include "tf_core.h"
  14 #include "tf_util.h"
  15 #include "tf_common.h"
  16 #include "tf_em.h"
  17 #include "tf_em_common.h"
  18 #include "tf_msg.h"
  19 #include "tfp.h"
  20 #include "tf_device.h"
  21 #include "tf_ext_flow_handle.h"
  22 #include "hcapi_cfa.h"
  23
  24 #include "bnxt.h"
  25
  26 /* Number of pointers per page_size */
  27 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  28
  29 /**
  30  * Host or system
  31  */
  32 static enum tf_mem_type mem_type;
  33
  34 /* API defined in tf_em.h */
  35 int
  36 tf_create_tbl_pool_external(enum tf_dir dir,
  37                             struct tf_tbl_scope_cb *tbl_scope_cb,
  38                             uint32_t num_entries,
  39                             uint32_t entry_sz_bytes)
  40 {
  41         struct tfp_calloc_parms parms;
  42         uint32_t i;
  43         int32_t j;
  44         int rc = 0;
  45         struct stack *pool = &tbl_scope_cb->ext_act_pool[dir];
  46
  47         parms.nitems = num_entries;
  48         parms.size = sizeof(uint32_t);
  49         parms.alignment = 0;
  50
  51         if (tfp_calloc(&parms) != 0) {
  52                 TFP_DRV_LOG(ERR, "%s: TBL: external pool failure %s\n",
  53                             tf_dir_2_str(dir), strerror(ENOMEM));
  54                 return -ENOMEM;
  55         }
  56
  57         /* Create empty stack
  58          */
  59         rc = stack_init(num_entries, parms.mem_va, pool);
  60
  61         if (rc != 0) {
  62                 TFP_DRV_LOG(ERR, "%s: TBL: stack init failure %s\n",
  63                             tf_dir_2_str(dir), strerror(-rc));
  64                 goto cleanup;
  65         }
  66
  67         /* Save the  malloced memory address so that it can
  68          * be freed when the table scope is freed.
  69          */
  70         tbl_scope_cb->ext_act_pool_mem[dir] = (uint32_t *)parms.mem_va;
  71
  72         /* Fill pool with indexes in reverse
  73          */
  74         j = (num_entries - 1) * entry_sz_bytes;
  75
  76         for (i = 0; i < num_entries; i++) {
  77                 rc = stack_push(pool, j);
  78                 if (rc != 0) {
  79                         TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
  80                                     tf_dir_2_str(dir), strerror(-rc));
  81                         goto cleanup;
  82                 }
  83
  84                 if (j < 0) {
  85                         TFP_DRV_LOG(ERR, "%d TBL: invalid offset (%d)\n",
  86                                     dir, j);
  87                         goto cleanup;
  88                 }
  89                 j -= entry_sz_bytes;
  90         }
  91
  92         if (!stack_is_full(pool)) {
  93                 rc = -EINVAL;
  94                 TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
  95                             tf_dir_2_str(dir), strerror(-rc));
  96                 goto cleanup;
  97         }
  98         return 0;
  99 cleanup:
 100         tfp_free((void *)parms.mem_va);
 101         return rc;
 102 }
 103
 104 /**
 105  * Destroy External Tbl pool of memory indexes.
 106  *
 107  * [in] dir
 108  *   direction
 109  * [in] tbl_scope_cb
 110  *   pointer to the table scope
 111  */
 112 void
 113 tf_destroy_tbl_pool_external(enum tf_dir dir,
 114                              struct tf_tbl_scope_cb *tbl_scope_cb)
 115 {
 116         uint32_t *ext_act_pool_mem =
 117                 tbl_scope_cb->ext_act_pool_mem[dir];
 118
 119         tfp_free(ext_act_pool_mem);
 120 }
 121
 122 /**
 123  * Looks up table scope control block using tbl_scope_id from tf_session.
 124  *
 125  * [in] tfp
 126  *   Pointer to Truflow Handle
 127  * [in] tbl_scope_id
 128  *   table scope id
 129  *
 130  * Return:
 131  *  - Pointer to the tf_tbl_scope_cb, if found.
 132  *  - (NULL) on failure, not found.
 133  */
 134 struct tf_tbl_scope_cb *
 135 tf_em_ext_common_tbl_scope_find(struct tf *tfp,
 136                         uint32_t tbl_scope_id)
 137 {
 138         int rc;
 139         struct em_ext_db *ext_db;
 140         void *ext_ptr = NULL;
 141         struct tf_tbl_scope_cb *tbl_scope_cb = NULL;
 142         struct ll_entry *entry;
 143
 144         rc = tf_session_get_em_ext_db(tfp, &ext_ptr);
 145         if (rc)
 146                 return NULL;
 147
 148         ext_db = (struct em_ext_db *)ext_ptr;
 149
 150         for (entry = ext_db->tbl_scope_ll.head; entry != NULL;
 151                         entry = entry->next) {
 152                 tbl_scope_cb = (struct tf_tbl_scope_cb *)entry;
 153                 if (tbl_scope_cb->tbl_scope_id == tbl_scope_id)
 154                         return tbl_scope_cb;
 155         }
 156
 157         return NULL;
 158 }
 159
 160 /**
 161  * Allocate External Tbl entry from the scope pool.
 162  *
 163  * [in] tfp
 164  *   Pointer to Truflow Handle
 165  * [in] parms
 166  *   Allocation parameters
 167  *
 168  * Return:
 169  *  0       - Success, entry allocated - no search support
 170  *  -ENOMEM -EINVAL -EOPNOTSUPP
 171  *          - Failure, entry not allocated, out of resources
 172  */
 173 int
 174 tf_tbl_ext_alloc(struct tf *tfp,
 175                  struct tf_tbl_alloc_parms *parms)
 176 {
 177         int rc;
 178         uint32_t index;
 179         struct tf_tbl_scope_cb *tbl_scope_cb;
 180         struct stack *pool;
 181
 182         TF_CHECK_PARMS2(tfp, parms);
 183
 184         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
 185         if (tbl_scope_cb == NULL) {
 186                 TFP_DRV_LOG(ERR,
 187                             "%s, table scope not allocated\n",
 188                             tf_dir_2_str(parms->dir));
 189                 return -EINVAL;
 190         }
 191
 192         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
 193
 194         /* Allocate an element
 195          */
 196         rc = stack_pop(pool, &index);
 197
 198         if (rc != 0) {
 199                 TFP_DRV_LOG(ERR,
 200                    "%s, Allocation failed, type:%d\n",
 201                    tf_dir_2_str(parms->dir),
 202                    parms->type);
 203                 return rc;
 204         }
 205
 206         *parms->idx = index;
 207         return rc;
 208 }
 209
 210 /**
 211  * Free External Tbl entry to the scope pool.
 212  *
 213  * [in] tfp
 214  *   Pointer to Truflow Handle
 215  * [in] parms
 216  *   Allocation parameters
 217  *
 218  * Return:
 219  *  0       - Success, entry freed
 220  *
 221  * - Failure, entry not successfully freed for these reasons
 222  *  -ENOMEM
 223  *  -EOPNOTSUPP
 224  *  -EINVAL
 225  */
 226 int
 227 tf_tbl_ext_free(struct tf *tfp,
 228                 struct tf_tbl_free_parms *parms)
 229 {
 230         int rc = 0;
 231         uint32_t index;
 232         struct tf_tbl_scope_cb *tbl_scope_cb;
 233         struct stack *pool;
 234
 235         TF_CHECK_PARMS2(tfp, parms);
 236
 237         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
 238         if (tbl_scope_cb == NULL) {
 239                 TFP_DRV_LOG(ERR,
 240                             "%s, table scope error\n",
 241                             tf_dir_2_str(parms->dir));
 242                 return -EINVAL;
 243         }
 244         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
 245
 246         index = parms->idx;
 247
 248         rc = stack_push(pool, index);
 249
 250         if (rc != 0) {
 251                 TFP_DRV_LOG(ERR,
 252                    "%s, consistency error, stack full, type:%d, idx:%d\n",
 253                    tf_dir_2_str(parms->dir),
 254                    parms->type,
 255                    index);
 256         }
 257         return rc;
 258 }
 259
 260 uint32_t
 261 tf_em_get_key_mask(int num_entries)
 262 {
 263         uint32_t mask = num_entries - 1;
 264
 265         if (num_entries & TF_EM_MAX_MASK)
 266                 return 0;
 267
 268         if (num_entries > TF_EM_MAX_ENTRY)
 269                 return 0;
 270
 271         return mask;
 272 }
 273
 274 void
 275 tf_em_create_key_entry(struct cfa_p4_eem_entry_hdr *result,
 276                        uint8_t *in_key,
 277                        struct cfa_p4_eem_64b_entry *key_entry)
 278 {
 279         key_entry->hdr.word1 = result->word1;
 280         key_entry->hdr.pointer = result->pointer;
 281         memcpy(key_entry->key, in_key, TF_P4_HW_EM_KEY_MAX_SIZE + 4);
 282 }
 283
 284
 285 /**
 286  * Return the number of page table pages needed to
 287  * reference the given number of next level pages.
 288  *
 289  * [in] num_pages
 290  *   Number of EM pages
 291  *
 292  * [in] page_size
 293  *   Size of each EM page
 294  *
 295  * Returns:
 296  *   Number of EM page table pages
 297  */
 298 static uint32_t
 299 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 300                      uint32_t page_size)
 301 {
 302         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 303                        MAX_PAGE_PTRS(page_size);
 304         return 0;
 305 }
 306
 307 /**
 308  * Given the number of data pages, page_size and the maximum
 309  * number of page table levels (already determined), size
 310  * the number of page table pages required at each level.
 311  *
 312  * [in] max_lvl
 313  *   Max number of levels
 314  *
 315  * [in] num_data_pages
 316  *   Number of EM data pages
 317  *
 318  * [in] page_size
 319  *   Size of an EM page
 320  *
 321  * [out] *page_cnt
 322  *   EM page count
 323  */
 324 static void
 325 tf_em_size_page_tbls(int max_lvl,
 326                      uint64_t num_data_pages,
 327                      uint32_t page_size,
 328                      uint32_t *page_cnt)
 329 {
 330         if (max_lvl == TF_PT_LVL_0) {
 331                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 332         } else if (max_lvl == TF_PT_LVL_1) {
 333                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 334                 page_cnt[TF_PT_LVL_0] =
 335                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 336         } else if (max_lvl == TF_PT_LVL_2) {
 337                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 338                 page_cnt[TF_PT_LVL_1] =
 339                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 340                 page_cnt[TF_PT_LVL_0] =
 341                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 342         } else {
 343                 return;
 344         }
 345 }
 346
 347 /**
 348  * Given the page size, size of each data item (entry size),
 349  * and the total number of entries needed, determine the number
 350  * of page table levels and the number of data pages required.
 351  *
 352  * [in] page_size
 353  *   Page size
 354  *
 355  * [in] entry_size
 356  *   Entry size
 357  *
 358  * [in] num_entries
 359  *   Number of entries needed
 360  *
 361  * [out] num_data_pages
 362  *   Number of pages required
 363  *
 364  * Returns:
 365  *   Success  - Number of EM page levels required
 366  *   -ENOMEM  - Out of memory
 367  */
 368 static int
 369 tf_em_size_page_tbl_lvl(uint32_t page_size,
 370                         uint32_t entry_size,
 371                         uint32_t num_entries,
 372                         uint64_t *num_data_pages)
 373 {
 374         uint64_t lvl_data_size = page_size;
 375         int lvl = TF_PT_LVL_0;
 376         uint64_t data_size;
 377
 378         *num_data_pages = 0;
 379         data_size = (uint64_t)num_entries * entry_size;
 380
 381         while (lvl_data_size < data_size) {
 382                 lvl++;
 383
 384                 if (lvl == TF_PT_LVL_1)
 385                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 386                                 page_size;
 387                 else if (lvl == TF_PT_LVL_2)
 388                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 389                                 MAX_PAGE_PTRS(page_size) * page_size;
 390                 else
 391                         return -ENOMEM;
 392         }
 393
 394         *num_data_pages = roundup(data_size, page_size) / page_size;
 395
 396         return lvl;
 397 }
 398
 399 /**
 400  * Size the EM table based on capabilities
 401  *
 402  * [in] tbl
 403  *   EM table to size
 404  *
 405  * Returns:
 406  *   0        - Success
 407  *   - EINVAL - Parameter error
 408  *   - ENOMEM - Out of memory
 409  */
 410 int
 411 tf_em_size_table(struct hcapi_cfa_em_table *tbl,
 412                  uint32_t page_size)
 413 {
 414         uint64_t num_data_pages;
 415         uint32_t *page_cnt;
 416         int max_lvl;
 417         uint32_t num_entries;
 418         uint32_t cnt = TF_EM_MIN_ENTRIES;
 419
 420         /* Ignore entry if both size and number are zero */
 421         if (!tbl->entry_size && !tbl->num_entries)
 422                 return 0;
 423
 424         /* If only one is set then error */
 425         if (!tbl->entry_size || !tbl->num_entries)
 426                 return -EINVAL;
 427
 428         /* Determine number of page table levels and the number
 429          * of data pages needed to process the given eem table.
 430          */
 431         if (tbl->type == TF_RECORD_TABLE) {
 432                 /*
 433                  * For action records just a memory size is provided. Work
 434                  * backwards to resolve to number of entries
 435                  */
 436                 num_entries = tbl->num_entries / tbl->entry_size;
 437                 if (num_entries < TF_EM_MIN_ENTRIES) {
 438                         num_entries = TF_EM_MIN_ENTRIES;
 439                 } else {
 440                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 441                                 cnt *= 2;
 442                         num_entries = cnt;
 443                 }
 444         } else {
 445                 num_entries = tbl->num_entries;
 446         }
 447
 448         max_lvl = tf_em_size_page_tbl_lvl(page_size,
 449                                           tbl->entry_size,
 450                                           tbl->num_entries,
 451                                           &num_data_pages);
 452         if (max_lvl < 0) {
 453                 TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 454                 TFP_DRV_LOG(WARNING,
 455                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 456                             tbl->type, (uint64_t)num_entries * tbl->entry_size,
 457                             page_size);
 458                 return -ENOMEM;
 459         }
 460
 461         tbl->num_lvl = max_lvl + 1;
 462         tbl->num_data_pages = num_data_pages;
 463
 464         /* Determine the number of pages needed at each level */
 465         page_cnt = tbl->page_cnt;
 466         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 467         tf_em_size_page_tbls(max_lvl, num_data_pages, page_size,
 468                                 page_cnt);
 469
 470         TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 471         TFP_DRV_LOG(INFO,
 472                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 \
 473                     " l0: %u l1: %u l2: %u\n",
 474                     max_lvl + 1,
 475                     (uint64_t)num_data_pages * page_size,
 476                     num_data_pages,
 477                     page_cnt[TF_PT_LVL_0],
 478                     page_cnt[TF_PT_LVL_1],
 479                     page_cnt[TF_PT_LVL_2]);
 480
 481         return 0;
 482 }
 483
 484 /**
 485  * Validates EM number of entries requested
 486  *
 487  * [in] tbl_scope_cb
 488  *   Pointer to table scope control block to be populated
 489  *
 490  * [in] parms
 491  *   Pointer to input parameters
 492  *
 493  * Returns:
 494  *   0       - Success
 495  *   -EINVAL - Parameter error
 496  */
 497 int
 498 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 499                            struct tf_alloc_tbl_scope_parms *parms)
 500 {
 501         uint32_t cnt;
 502
 503         if (parms->rx_mem_size_in_mb != 0) {
 504                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 505                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 506                                      + 1);
 507                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 508                                         TF_MEGABYTE) / (key_b + action_b);
 509
 510                 if (num_entries < TF_EM_MIN_ENTRIES) {
 511                         TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 512                                     "%uMB\n",
 513                                     parms->rx_mem_size_in_mb);
 514                         return -EINVAL;
 515                 }
 516
 517                 cnt = TF_EM_MIN_ENTRIES;
 518                 while (num_entries > cnt &&
 519                        cnt <= TF_EM_MAX_ENTRIES)
 520                         cnt *= 2;
 521
 522                 if (cnt > TF_EM_MAX_ENTRIES) {
 523                         TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 524                                     "%u\n",
 525                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 526                         return -EINVAL;
 527                 }
 528
 529                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 530         } else {
 531                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 532                     TF_EM_MIN_ENTRIES ||
 533                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 534                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 535                         TFP_DRV_LOG(ERR,
 536                                     "EEM: Invalid number of Rx flows "
 537                                     "requested:%u max:%u\n",
 538                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 539                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 540                         return -EINVAL;
 541                 }
 542
 543                 /* must be a power-of-2 supported value
 544                  * in the range 32K - 128M
 545                  */
 546                 cnt = TF_EM_MIN_ENTRIES;
 547                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 548                        cnt <= TF_EM_MAX_ENTRIES)
 549                         cnt *= 2;
 550
 551                 if (cnt > TF_EM_MAX_ENTRIES) {
 552                         TFP_DRV_LOG(ERR,
 553                                     "EEM: Invalid number of Rx requested: %u\n",
 554                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 555                         return -EINVAL;
 556                 }
 557         }
 558
 559         if (parms->tx_mem_size_in_mb != 0) {
 560                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 561                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 562                                      + 1);
 563                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 564                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 565                         (key_b + action_b);
 566
 567                 if (num_entries < TF_EM_MIN_ENTRIES) {
 568                         TFP_DRV_LOG(ERR,
 569                                     "EEM: Insufficient memory requested:%uMB\n",
 570                                     parms->rx_mem_size_in_mb);
 571                         return -EINVAL;
 572                 }
 573
 574                 cnt = TF_EM_MIN_ENTRIES;
 575                 while (num_entries > cnt &&
 576                        cnt <= TF_EM_MAX_ENTRIES)
 577                         cnt *= 2;
 578
 579                 if (cnt > TF_EM_MAX_ENTRIES) {
 580                         TFP_DRV_LOG(ERR,
 581                                     "EEM: Invalid number of Tx requested: %u\n",
 582                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 583                         return -EINVAL;
 584                 }
 585
 586                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 587         } else {
 588                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 589                     TF_EM_MIN_ENTRIES ||
 590                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 591                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 592                         TFP_DRV_LOG(ERR,
 593                                     "EEM: Invalid number of Tx flows "
 594                                     "requested:%u max:%u\n",
 595                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 596                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 597                         return -EINVAL;
 598                 }
 599
 600                 cnt = TF_EM_MIN_ENTRIES;
 601                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 602                        cnt <= TF_EM_MAX_ENTRIES)
 603                         cnt *= 2;
 604
 605                 if (cnt > TF_EM_MAX_ENTRIES) {
 606                         TFP_DRV_LOG(ERR,
 607                                     "EEM: Invalid number of Tx requested: %u\n",
 608                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 609                         return -EINVAL;
 610                 }
 611         }
 612
 613         if (parms->rx_num_flows_in_k != 0 &&
 614             parms->rx_max_key_sz_in_bits / 8 == 0) {
 615                 TFP_DRV_LOG(ERR,
 616                             "EEM: Rx key size required: %u\n",
 617                             (parms->rx_max_key_sz_in_bits));
 618                 return -EINVAL;
 619         }
 620
 621         if (parms->tx_num_flows_in_k != 0 &&
 622             parms->tx_max_key_sz_in_bits / 8 == 0) {
 623                 TFP_DRV_LOG(ERR,
 624                             "EEM: Tx key size required: %u\n",
 625                             (parms->tx_max_key_sz_in_bits));
 626                 return -EINVAL;
 627         }
 628         /* Rx */
 629         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 630                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 631         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 632                 parms->rx_max_key_sz_in_bits / 8;
 633
 634         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 635                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 636         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 637                 parms->rx_max_key_sz_in_bits / 8;
 638
 639         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 640                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 641         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 642                 parms->rx_max_action_entry_sz_in_bits / 8;
 643
 644         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries =
 645                 0;
 646
 647         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_ACTION_TABLE].num_entries =
 648                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 649         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_ACTION_TABLE].entry_size =
 650                 parms->rx_max_action_entry_sz_in_bits / 8;
 651
 652         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EM_LKUP_TABLE].num_entries =
 653                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 654         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EM_LKUP_TABLE].entry_size =
 655                 parms->rx_max_key_sz_in_bits / 8;
 656
 657         /* Tx */
 658         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 659                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 660         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 661                 parms->tx_max_key_sz_in_bits / 8;
 662
 663         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 664                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 665         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 666                 parms->tx_max_key_sz_in_bits / 8;
 667
 668         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 669                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 670         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 671                 parms->tx_max_action_entry_sz_in_bits / 8;
 672
 673         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries =
 674                 0;
 675
 676         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_ACTION_TABLE].num_entries =
 677                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 678         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_ACTION_TABLE].entry_size =
 679                 parms->tx_max_action_entry_sz_in_bits / 8;
 680
 681         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EM_LKUP_TABLE].num_entries =
 682                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 683         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EM_LKUP_TABLE].entry_size =
 684                 parms->tx_max_key_sz_in_bits / 8;
 685
 686         return 0;
 687 }
 688
 689 /** insert EEM entry API
 690  *
 691  * returns:
 692  *  0
 693  *  TF_ERR          - unable to get lock
 694  *
 695  * insert callback returns:
 696  *   0
 697  *   TF_ERR_EM_DUP  - key is already in table
 698  */
 699 static int
 700 tf_insert_eem_entry(struct tf_dev_info *dev,
 701                     struct tf_tbl_scope_cb *tbl_scope_cb,
 702                     struct tf_insert_em_entry_parms *parms)
 703 {
 704         uint32_t mask;
 705         uint32_t key0_hash;
 706         uint32_t key1_hash;
 707         uint32_t key0_index;
 708         uint32_t key1_index;
 709         struct cfa_p4_eem_64b_entry key_entry;
 710         uint32_t index;
 711         enum hcapi_cfa_em_table_type table_type;
 712         uint32_t gfid;
 713         struct hcapi_cfa_hwop op;
 714         struct hcapi_cfa_key_tbl key_tbl;
 715         struct hcapi_cfa_key_data key_obj;
 716         struct hcapi_cfa_key_loc key_loc;
 717         uint64_t big_hash;
 718         int rc;
 719
 720         /* Get mask to use on hash */
 721         mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
 722
 723         if (!mask)
 724                 return -EINVAL;
 725
 726         if (dev->ops->tf_dev_cfa_key_hash == NULL)
 727                 return -EINVAL;
 728
 729         big_hash = dev->ops->tf_dev_cfa_key_hash((uint64_t *)parms->key,
 730                                          (TF_P4_HW_EM_KEY_MAX_SIZE + 4) * 8);
 731         key0_hash = (uint32_t)(big_hash >> 32);
 732         key1_hash = (uint32_t)(big_hash & 0xFFFFFFFF);
 733
 734         key0_index = key0_hash & mask;
 735         key1_index = key1_hash & mask;
 736
 737         /*
 738          * Use the "result" arg to populate all of the key entry then
 739          * store the byte swapped "raw" entry in a local copy ready
 740          * for insertion in to the table.
 741          */
 742         tf_em_create_key_entry((struct cfa_p4_eem_entry_hdr *)parms->em_record,
 743                                 ((uint8_t *)parms->key),
 744                                 &key_entry);
 745
 746         /*
 747          * Try to add to Key0 table, if that does not work then
 748          * try the key1 table.
 749          */
 750         index = key0_index;
 751         op.opcode = HCAPI_CFA_HWOPS_ADD;
 752         key_tbl.base0 =
 753                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE];
 754         key_tbl.page_size = TF_EM_PAGE_SIZE;
 755         key_obj.offset = index * TF_P4_EM_KEY_RECORD_SIZE;
 756         key_obj.data = (uint8_t *)&key_entry;
 757         key_obj.size = TF_P4_EM_KEY_RECORD_SIZE;
 758
 759         rc = cfa_p4_devops.hcapi_cfa_key_hw_op(&op,
 760                                                &key_tbl,
 761                                                &key_obj,
 762                                                &key_loc);
 763
 764         if (rc == 0) {
 765                 table_type = TF_KEY0_TABLE;
 766         } else {
 767                 index = key1_index;
 768
 769                 key_tbl.base0 =
 770                         (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY1_TABLE];
 771                 key_obj.offset = index * TF_P4_EM_KEY_RECORD_SIZE;
 772
 773                 rc = cfa_p4_devops.hcapi_cfa_key_hw_op(&op,
 774                                                        &key_tbl,
 775                                                        &key_obj,
 776                                                        &key_loc);
 777                 if (rc != 0)
 778                         return rc;
 779
 780                 table_type = TF_KEY1_TABLE;
 781         }
 782
 783         TF_SET_GFID(gfid,
 784                     index,
 785                     table_type);
 786         TF_SET_FLOW_ID(parms->flow_id,
 787                        gfid,
 788                        TF_GFID_TABLE_EXTERNAL,
 789                        parms->dir);
 790         TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
 791                                      0,
 792                                      0,
 793                                      0,
 794                                      index,
 795                                      0,
 796                                      table_type);
 797
 798         return 0;
 799 }
 800
 801 /** delete EEM hash entry API
 802  *
 803  * returns:
 804  *   0
 805  *   -EINVAL      - parameter error
 806  *   TF_NO_SESSION    - bad session ID
 807  *   TF_ERR_TBL_SCOPE - invalid table scope
 808  *   TF_ERR_TBL_IF    - invalid table interface
 809  *
 810  * insert callback returns
 811  *   0
 812  *   TF_NO_EM_MATCH - entry not found
 813  */
 814 static int
 815 tf_delete_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 816                     struct tf_delete_em_entry_parms *parms)
 817 {
 818         enum hcapi_cfa_em_table_type hash_type;
 819         uint32_t index;
 820         struct hcapi_cfa_hwop op;
 821         struct hcapi_cfa_key_tbl key_tbl;
 822         struct hcapi_cfa_key_data key_obj;
 823         struct hcapi_cfa_key_loc key_loc;
 824         int rc;
 825
 826         TF_GET_HASH_TYPE_FROM_FLOW_HANDLE(parms->flow_handle, hash_type);
 827         TF_GET_INDEX_FROM_FLOW_HANDLE(parms->flow_handle, index);
 828
 829         op.opcode = HCAPI_CFA_HWOPS_DEL;
 830         key_tbl.base0 =
 831                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables
 832                         [(hash_type == 0 ? TF_KEY0_TABLE : TF_KEY1_TABLE)];
 833         key_tbl.page_size = TF_EM_PAGE_SIZE;
 834         key_obj.offset = index * TF_P4_EM_KEY_RECORD_SIZE;
 835         key_obj.data = NULL;
 836         key_obj.size = TF_P4_EM_KEY_RECORD_SIZE;
 837
 838         rc = cfa_p4_devops.hcapi_cfa_key_hw_op(&op,
 839                                                &key_tbl,
 840                                                &key_obj,
 841                                                &key_loc);
 842
 843         if (!rc)
 844                 return rc;
 845
 846         return 0;
 847 }
 848
 849 /** insert EM hash entry API
 850  *
 851  *    returns:
 852  *    0       - Success
 853  *    -EINVAL - Error
 854  */
 855 int
 856 tf_em_insert_ext_entry(struct tf *tfp,
 857                        struct tf_insert_em_entry_parms *parms)
 858 {
 859         int rc;
 860         struct tf_tbl_scope_cb *tbl_scope_cb;
 861         struct tf_session *tfs;
 862         struct tf_dev_info *dev;
 863
 864         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
 865         if (tbl_scope_cb == NULL) {
 866                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 867                 return -EINVAL;
 868         }
 869
 870         /* Retrieve the session information */
 871         rc = tf_session_get_session_internal(tfp, &tfs);
 872         if (rc)
 873                 return rc;
 874
 875         /* Retrieve the device information */
 876         rc = tf_session_get_device(tfs, &dev);
 877         if (rc)
 878                 return rc;
 879
 880         return tf_insert_eem_entry
 881                 (dev,
 882                  tbl_scope_cb,
 883                  parms);
 884 }
 885
 886 /** Delete EM hash entry API
 887  *
 888  *    returns:
 889  *    0       - Success
 890  *    -EINVAL - Error
 891  */
 892 int
 893 tf_em_delete_ext_entry(struct tf *tfp,
 894                        struct tf_delete_em_entry_parms *parms)
 895 {
 896         struct tf_tbl_scope_cb *tbl_scope_cb;
 897
 898         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
 899         if (tbl_scope_cb == NULL) {
 900                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 901                 return -EINVAL;
 902         }
 903
 904         return tf_delete_eem_entry(tbl_scope_cb, parms);
 905 }
 906
 907
 908 int
 909 tf_em_ext_common_bind(struct tf *tfp,
 910                       struct tf_em_cfg_parms *parms)
 911 {
 912         int rc;
 913         int i;
 914         struct tf_rm_create_db_parms db_cfg = { 0 };
 915         struct em_ext_db *ext_db;
 916         struct tfp_calloc_parms cparms;
 917
 918         TF_CHECK_PARMS2(tfp, parms);
 919
 920         cparms.nitems = 1;
 921         cparms.size = sizeof(struct em_ext_db);
 922         cparms.alignment = 0;
 923         if (tfp_calloc(&cparms) != 0) {
 924                 TFP_DRV_LOG(ERR, "em_ext_db alloc error %s\n",
 925                             strerror(ENOMEM));
 926                 return -ENOMEM;
 927         }
 928
 929         ext_db = cparms.mem_va;
 930         ll_init(&ext_db->tbl_scope_ll);
 931         for (i = 0; i < TF_DIR_MAX; i++)
 932                 ext_db->eem_db[i] = NULL;
 933         tf_session_set_em_ext_db(tfp, ext_db);
 934
 935         db_cfg.module = TF_MODULE_TYPE_EM;
 936         db_cfg.num_elements = parms->num_elements;
 937         db_cfg.cfg = parms->cfg;
 938
 939         for (i = 0; i < TF_DIR_MAX; i++) {
 940                 db_cfg.dir = i;
 941                 db_cfg.alloc_cnt = parms->resources->em_cnt[i].cnt;
 942
 943                 /* Check if we got any request to support EEM, if so
 944                  * we build an EM Ext DB holding Table Scopes.
 945                  */
 946                 if (db_cfg.alloc_cnt[TF_EM_TBL_TYPE_TBL_SCOPE] == 0)
 947                         continue;
 948
 949                 db_cfg.rm_db = (void *)&ext_db->eem_db[i];
 950                 rc = tf_rm_create_db(tfp, &db_cfg);
 951                 if (rc) {
 952                         TFP_DRV_LOG(ERR,
 953                                     "%s: EM Ext DB creation failed\n",
 954                                     tf_dir_2_str(i));
 955
 956                         return rc;
 957                 }
 958         }
 959
 960         mem_type = parms->mem_type;
 961
 962         return 0;
 963 }
 964
 965 int
 966 tf_em_ext_common_unbind(struct tf *tfp)
 967 {
 968         int rc;
 969         int i;
 970         struct tf_rm_free_db_parms fparms = { 0 };
 971         struct em_ext_db *ext_db = NULL;
 972         struct tf_session *tfs = NULL;
 973         struct tf_dev_info *dev;
 974         struct ll_entry *entry;
 975         struct tf_tbl_scope_cb *tbl_scope_cb = NULL;
 976         void *ext_ptr = NULL;
 977         struct tf_free_tbl_scope_parms tparms = { 0 };
 978
 979         TF_CHECK_PARMS1(tfp);
 980
 981         rc = tf_session_get_session_internal(tfp, &tfs);
 982         if (rc) {
 983                 TFP_DRV_LOG(ERR, "Failed to get tf_session, rc:%s\n",
 984                 strerror(-rc));
 985                 return rc;
 986         }
 987
 988         /* Retrieve the device information */
 989         rc = tf_session_get_device(tfs, &dev);
 990         if (rc) {
 991                 TFP_DRV_LOG(ERR,
 992                             "Failed to lookup device, rc:%s\n",
 993                             strerror(-rc));
 994                 return rc;
 995         }
 996
 997         rc = tf_session_get_em_ext_db(tfp, &ext_ptr);
 998         if (rc) {
 999                 TFP_DRV_LOG(ERR,
1000                             "Failed to get em_ext_db from session, rc:%s\n",
1001                             strerror(-rc));
1002                 return rc;
1003         }
1004         ext_db = (struct em_ext_db *)ext_ptr;
1005
1006         if (ext_db != NULL) {
1007                 entry = ext_db->tbl_scope_ll.head;
1008                 while (entry != NULL) {
1009                         tbl_scope_cb = (struct tf_tbl_scope_cb *)entry;
1010                         entry = entry->next;
1011                         tparms.tbl_scope_id =
1012                                 tbl_scope_cb->tbl_scope_id;
1013
1014                         if (dev->ops->tf_dev_free_tbl_scope) {
1015                                 dev->ops->tf_dev_free_tbl_scope(tfp,
1016                                                                 &tparms);
1017                         } else {
1018                                 /* should not reach here */
1019                                 ll_delete(&ext_db->tbl_scope_ll,
1020                                           &tbl_scope_cb->ll_entry);
1021                                 tfp_free(tbl_scope_cb);
1022                         }
1023                 }
1024
1025                 for (i = 0; i < TF_DIR_MAX; i++) {
1026                         if (ext_db->eem_db[i] == NULL)
1027                                 continue;
1028
1029                         fparms.dir = i;
1030                         fparms.rm_db = ext_db->eem_db[i];
1031                         rc = tf_rm_free_db(tfp, &fparms);
1032                         if (rc)
1033                                 return rc;
1034
1035                         ext_db->eem_db[i] = NULL;
1036                 }
1037
1038                 tfp_free(ext_db);
1039         }
1040
1041         tf_session_set_em_ext_db(tfp, NULL);
1042
1043         return 0;
1044 }
1045
1046 /**
1047  * Sets the specified external table type element.
1048  *
1049  * This API sets the specified element data
1050  *
1051  * [in] tfp
1052  *   Pointer to TF handle
1053  *
1054  * [in] parms
1055  *   Pointer to table set parameters
1056  *
1057  * Returns
1058  *   - (0) if successful.
1059  *   - (-EINVAL) on failure.
1060  */
1061 int tf_tbl_ext_common_set(struct tf *tfp,
1062                           struct tf_tbl_set_parms *parms)
1063 {
1064         int rc = 0;
1065         struct tf_tbl_scope_cb *tbl_scope_cb;
1066         uint32_t tbl_scope_id;
1067         struct hcapi_cfa_hwop op;
1068         struct hcapi_cfa_key_tbl key_tbl;
1069         struct hcapi_cfa_key_data key_obj;
1070         struct hcapi_cfa_key_loc key_loc;
1071
1072         TF_CHECK_PARMS2(tfp, parms);
1073
1074         if (parms->data == NULL) {
1075                 TFP_DRV_LOG(ERR,
1076                             "%s, invalid parms->data\n",
1077                             tf_dir_2_str(parms->dir));
1078                 return -EINVAL;
1079         }
1080
1081         tbl_scope_id = parms->tbl_scope_id;
1082
1083         if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1084                 TFP_DRV_LOG(ERR,
1085                             "%s, Table scope not allocated\n",
1086                             tf_dir_2_str(parms->dir));
1087                 return -EINVAL;
1088         }
1089
1090         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, tbl_scope_id);
1091         if (tbl_scope_cb == NULL) {
1092                 TFP_DRV_LOG(ERR,
1093                             "%s, table scope error\n",
1094                             tf_dir_2_str(parms->dir));
1095                 return -EINVAL;
1096         }
1097
1098         op.opcode = HCAPI_CFA_HWOPS_PUT;
1099         key_tbl.base0 =
1100                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_RECORD_TABLE];
1101         key_tbl.page_size = TF_EM_PAGE_SIZE;
1102         key_obj.offset = parms->idx;
1103         key_obj.data = parms->data;
1104         key_obj.size = parms->data_sz_in_bytes;
1105
1106         rc = cfa_p4_devops.hcapi_cfa_key_hw_op(&op,
1107                                                &key_tbl,
1108                                                &key_obj,
1109                                                &key_loc);
1110
1111         return rc;
1112 }
1113
1114 int
1115 tf_em_ext_common_alloc(struct tf *tfp,
1116                        struct tf_alloc_tbl_scope_parms *parms)
1117 {
1118         return tf_em_ext_alloc(tfp, parms);
1119 }
1120
1121 int
1122 tf_em_ext_common_free(struct tf *tfp,
1123                       struct tf_free_tbl_scope_parms *parms)
1124 {
1125         return tf_em_ext_free(tfp, parms);
1126 }
1127
1128 int tf_em_ext_map_tbl_scope(struct tf *tfp,
1129                             struct tf_map_tbl_scope_parms *parms)
1130 {
1131         int rc = 0;
1132         struct tf_session *tfs;
1133         struct tf_tbl_scope_cb *tbl_scope_cb;
1134         struct tf_global_cfg_parms gcfg_parms = { 0 };
1135         struct tfp_calloc_parms aparms;
1136         uint32_t *data, *mask;
1137         uint32_t sz_in_bytes = 8;
1138         struct tf_dev_info *dev;
1139
1140         /* Retrieve the session information */
1141         rc = tf_session_get_session_internal(tfp, &tfs);
1142         if (rc)
1143                 return rc;
1144
1145         /* Retrieve the device information */
1146         rc = tf_session_get_device(tfs, &dev);
1147         if (rc)
1148                 return rc;
1149
1150         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
1151         if (tbl_scope_cb == NULL) {
1152                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb tbl_scope_id(%d)\n",
1153                             parms->tbl_scope_id);
1154                 return -EINVAL;
1155         }
1156
1157         if (dev->ops->tf_dev_map_tbl_scope == NULL) {
1158                 rc = -EOPNOTSUPP;
1159                 TFP_DRV_LOG(ERR,
1160                             "Map table scope operation not supported, rc:%s\n",
1161                             strerror(-rc));
1162                 return rc;
1163         }
1164
1165         aparms.nitems = 2;
1166         aparms.size = sizeof(uint32_t);
1167         aparms.alignment = 0;
1168
1169         if (tfp_calloc(&aparms) != 0) {
1170                 TFP_DRV_LOG(ERR, "Map tbl scope alloc data error %s\n",
1171                             strerror(ENOMEM));
1172                 return -ENOMEM;
1173         }
1174         data = aparms.mem_va;
1175
1176         if (tfp_calloc(&aparms) != 0) {
1177                 TFP_DRV_LOG(ERR, "Map tbl scope alloc mask error %s\n",
1178                             strerror(ENOMEM));
1179                 rc = -ENOMEM;
1180                 goto clean;
1181         }
1182         mask = aparms.mem_va;
1183
1184         rc = dev->ops->tf_dev_map_parif(tfp, parms->parif_bitmask,
1185                                         tbl_scope_cb->pf,
1186                                         (uint8_t *)data, (uint8_t *)mask,
1187                                         sz_in_bytes);
1188
1189         if (rc) {
1190                 TFP_DRV_LOG(ERR,
1191                             "Map table scope config failure, rc:%s\n",
1192                             strerror(-rc));
1193                 goto cleaner;
1194         }
1195
1196         /* Note that TF_GLOBAL_CFG_INTERNAL_PARIF_2_PF is same as below enum */
1197         gcfg_parms.type = TF_GLOBAL_CFG_TYPE_MAX;
1198         gcfg_parms.offset = 0;
1199         gcfg_parms.config = (uint8_t *)data;
1200         gcfg_parms.config_mask = (uint8_t *)mask;
1201         gcfg_parms.config_sz_in_bytes = sizeof(uint64_t);
1202
1203
1204         rc = tf_msg_set_global_cfg(tfp, &gcfg_parms);
1205         if (rc) {
1206                 TFP_DRV_LOG(ERR,
1207                             "Map tbl scope, set failed, rc:%s\n",
1208                             strerror(-rc));
1209         }
1210 cleaner:
1211         tfp_free(mask);
1212 clean:
1213         tfp_free(data);
1214
1215         return rc;
1216 }