drivers/net/bnxt/tf_core/tf_em_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2021 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <math.h>
   8 #include <sys/param.h>
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12
  13 #include "tf_core.h"
  14 #include "tf_util.h"
  15 #include "tf_common.h"
  16 #include "tf_em.h"
  17 #include "tf_em_common.h"
  18 #include "tf_msg.h"
  19 #include "tfp.h"
  20 #include "tf_device.h"
  21 #include "tf_ext_flow_handle.h"
  22 #include "hcapi_cfa.h"
  23
  24 #include "bnxt.h"
  25
  26 /* Number of pointers per page_size */
  27 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  28
  29 /**
  30  * Init flag, set on bind and cleared on unbind
  31  */
  32 static uint8_t init;
  33
  34 /**
  35  * Host or system
  36  */
  37 static enum tf_mem_type mem_type;
  38
  39 /* API defined in tf_em.h */
  40 int
  41 tf_create_tbl_pool_external(enum tf_dir dir,
  42                             struct tf_tbl_scope_cb *tbl_scope_cb,
  43                             uint32_t num_entries,
  44                             uint32_t entry_sz_bytes)
  45 {
  46         struct tfp_calloc_parms parms;
  47         uint32_t i;
  48         int32_t j;
  49         int rc = 0;
  50         struct stack *pool = &tbl_scope_cb->ext_act_pool[dir];
  51
  52         parms.nitems = num_entries;
  53         parms.size = sizeof(uint32_t);
  54         parms.alignment = 0;
  55
  56         if (tfp_calloc(&parms) != 0) {
  57                 TFP_DRV_LOG(ERR, "%s: TBL: external pool failure %s\n",
  58                             tf_dir_2_str(dir), strerror(ENOMEM));
  59                 return -ENOMEM;
  60         }
  61
  62         /* Create empty stack
  63          */
  64         rc = stack_init(num_entries, parms.mem_va, pool);
  65
  66         if (rc != 0) {
  67                 TFP_DRV_LOG(ERR, "%s: TBL: stack init failure %s\n",
  68                             tf_dir_2_str(dir), strerror(-rc));
  69                 goto cleanup;
  70         }
  71
  72         /* Save the  malloced memory address so that it can
  73          * be freed when the table scope is freed.
  74          */
  75         tbl_scope_cb->ext_act_pool_mem[dir] = (uint32_t *)parms.mem_va;
  76
  77         /* Fill pool with indexes in reverse
  78          */
  79         j = (num_entries - 1) * entry_sz_bytes;
  80
  81         for (i = 0; i < num_entries; i++) {
  82                 rc = stack_push(pool, j);
  83                 if (rc != 0) {
  84                         TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
  85                                     tf_dir_2_str(dir), strerror(-rc));
  86                         goto cleanup;
  87                 }
  88
  89                 if (j < 0) {
  90                         TFP_DRV_LOG(ERR, "%d TBL: invalid offset (%d)\n",
  91                                     dir, j);
  92                         goto cleanup;
  93                 }
  94                 j -= entry_sz_bytes;
  95         }
  96
  97         if (!stack_is_full(pool)) {
  98                 rc = -EINVAL;
  99                 TFP_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
 100                             tf_dir_2_str(dir), strerror(-rc));
 101                 goto cleanup;
 102         }
 103         return 0;
 104 cleanup:
 105         tfp_free((void *)parms.mem_va);
 106         return rc;
 107 }
 108
 109 /**
 110  * Destroy External Tbl pool of memory indexes.
 111  *
 112  * [in] dir
 113  *   direction
 114  * [in] tbl_scope_cb
 115  *   pointer to the table scope
 116  */
 117 void
 118 tf_destroy_tbl_pool_external(enum tf_dir dir,
 119                              struct tf_tbl_scope_cb *tbl_scope_cb)
 120 {
 121         uint32_t *ext_act_pool_mem =
 122                 tbl_scope_cb->ext_act_pool_mem[dir];
 123
 124         tfp_free(ext_act_pool_mem);
 125 }
 126
 127 /**
 128  * Looks up table scope control block using tbl_scope_id from tf_session.
 129  *
 130  * [in] tfp
 131  *   Pointer to Truflow Handle
 132  * [in] tbl_scope_id
 133  *   table scope id
 134  *
 135  * Return:
 136  *  - Pointer to the tf_tbl_scope_cb, if found.
 137  *  - (NULL) on failure, not found.
 138  */
 139 struct tf_tbl_scope_cb *
 140 tf_em_ext_common_tbl_scope_find(struct tf *tfp,
 141                         uint32_t tbl_scope_id)
 142 {
 143         int rc;
 144         struct em_ext_db *ext_db;
 145         void *ext_ptr = NULL;
 146         struct tf_tbl_scope_cb *tbl_scope_cb = NULL;
 147         struct ll_entry *entry;
 148
 149         rc = tf_session_get_em_ext_db(tfp, &ext_ptr);
 150         if (rc)
 151                 return NULL;
 152
 153         ext_db = (struct em_ext_db *)ext_ptr;
 154
 155         for (entry = ext_db->tbl_scope_ll.head; entry != NULL;
 156                         entry = entry->next) {
 157                 tbl_scope_cb = (struct tf_tbl_scope_cb *)entry;
 158                 if (tbl_scope_cb->tbl_scope_id == tbl_scope_id)
 159                         return tbl_scope_cb;
 160         }
 161
 162         return NULL;
 163 }
 164
 165 /**
 166  * Allocate External Tbl entry from the scope pool.
 167  *
 168  * [in] tfp
 169  *   Pointer to Truflow Handle
 170  * [in] parms
 171  *   Allocation parameters
 172  *
 173  * Return:
 174  *  0       - Success, entry allocated - no search support
 175  *  -ENOMEM -EINVAL -EOPNOTSUPP
 176  *          - Failure, entry not allocated, out of resources
 177  */
 178 int
 179 tf_tbl_ext_alloc(struct tf *tfp,
 180                  struct tf_tbl_alloc_parms *parms)
 181 {
 182         int rc;
 183         uint32_t index;
 184         struct tf_tbl_scope_cb *tbl_scope_cb;
 185         struct stack *pool;
 186
 187         TF_CHECK_PARMS2(tfp, parms);
 188
 189         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
 190         if (tbl_scope_cb == NULL) {
 191                 TFP_DRV_LOG(ERR,
 192                             "%s, table scope not allocated\n",
 193                             tf_dir_2_str(parms->dir));
 194                 return -EINVAL;
 195         }
 196
 197         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
 198
 199         /* Allocate an element
 200          */
 201         rc = stack_pop(pool, &index);
 202
 203         if (rc != 0) {
 204                 TFP_DRV_LOG(ERR,
 205                    "%s, Allocation failed, type:%d\n",
 206                    tf_dir_2_str(parms->dir),
 207                    parms->type);
 208                 return rc;
 209         }
 210
 211         *parms->idx = index;
 212         return rc;
 213 }
 214
 215 /**
 216  * Free External Tbl entry to the scope pool.
 217  *
 218  * [in] tfp
 219  *   Pointer to Truflow Handle
 220  * [in] parms
 221  *   Allocation parameters
 222  *
 223  * Return:
 224  *  0       - Success, entry freed
 225  *
 226  * - Failure, entry not successfully freed for these reasons
 227  *  -ENOMEM
 228  *  -EOPNOTSUPP
 229  *  -EINVAL
 230  */
 231 int
 232 tf_tbl_ext_free(struct tf *tfp,
 233                 struct tf_tbl_free_parms *parms)
 234 {
 235         int rc = 0;
 236         uint32_t index;
 237         struct tf_tbl_scope_cb *tbl_scope_cb;
 238         struct stack *pool;
 239
 240         TF_CHECK_PARMS2(tfp, parms);
 241
 242         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
 243         if (tbl_scope_cb == NULL) {
 244                 TFP_DRV_LOG(ERR,
 245                             "%s, table scope error\n",
 246                             tf_dir_2_str(parms->dir));
 247                 return -EINVAL;
 248         }
 249         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
 250
 251         index = parms->idx;
 252
 253         rc = stack_push(pool, index);
 254
 255         if (rc != 0) {
 256                 TFP_DRV_LOG(ERR,
 257                    "%s, consistency error, stack full, type:%d, idx:%d\n",
 258                    tf_dir_2_str(parms->dir),
 259                    parms->type,
 260                    index);
 261         }
 262         return rc;
 263 }
 264
 265 uint32_t
 266 tf_em_get_key_mask(int num_entries)
 267 {
 268         uint32_t mask = num_entries - 1;
 269
 270         if (num_entries & TF_EM_MAX_MASK)
 271                 return 0;
 272
 273         if (num_entries > TF_EM_MAX_ENTRY)
 274                 return 0;
 275
 276         return mask;
 277 }
 278
 279 void
 280 tf_em_create_key_entry(struct cfa_p4_eem_entry_hdr *result,
 281                        uint8_t *in_key,
 282                        struct cfa_p4_eem_64b_entry *key_entry)
 283 {
 284         key_entry->hdr.word1 = result->word1;
 285         key_entry->hdr.pointer = result->pointer;
 286         memcpy(key_entry->key, in_key, TF_P4_HW_EM_KEY_MAX_SIZE + 4);
 287 }
 288
 289
 290 /**
 291  * Return the number of page table pages needed to
 292  * reference the given number of next level pages.
 293  *
 294  * [in] num_pages
 295  *   Number of EM pages
 296  *
 297  * [in] page_size
 298  *   Size of each EM page
 299  *
 300  * Returns:
 301  *   Number of EM page table pages
 302  */
 303 static uint32_t
 304 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 305                      uint32_t page_size)
 306 {
 307         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 308                        MAX_PAGE_PTRS(page_size);
 309 }
 310
 311 /**
 312  * Given the number of data pages, page_size and the maximum
 313  * number of page table levels (already determined), size
 314  * the number of page table pages required at each level.
 315  *
 316  * [in] max_lvl
 317  *   Max number of levels
 318  *
 319  * [in] num_data_pages
 320  *   Number of EM data pages
 321  *
 322  * [in] page_size
 323  *   Size of an EM page
 324  *
 325  * [out] *page_cnt
 326  *   EM page count
 327  */
 328 static void
 329 tf_em_size_page_tbls(int max_lvl,
 330                      uint64_t num_data_pages,
 331                      uint32_t page_size,
 332                      uint32_t *page_cnt)
 333 {
 334         if (max_lvl == TF_PT_LVL_0) {
 335                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 336         } else if (max_lvl == TF_PT_LVL_1) {
 337                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 338                 page_cnt[TF_PT_LVL_0] =
 339                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 340         } else if (max_lvl == TF_PT_LVL_2) {
 341                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 342                 page_cnt[TF_PT_LVL_1] =
 343                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 344                 page_cnt[TF_PT_LVL_0] =
 345                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 346         } else {
 347                 return;
 348         }
 349 }
 350
 351 /**
 352  * Given the page size, size of each data item (entry size),
 353  * and the total number of entries needed, determine the number
 354  * of page table levels and the number of data pages required.
 355  *
 356  * [in] page_size
 357  *   Page size
 358  *
 359  * [in] entry_size
 360  *   Entry size
 361  *
 362  * [in] num_entries
 363  *   Number of entries needed
 364  *
 365  * [out] num_data_pages
 366  *   Number of pages required
 367  *
 368  * Returns:
 369  *   Success  - Number of EM page levels required
 370  *   -ENOMEM  - Out of memory
 371  */
 372 static int
 373 tf_em_size_page_tbl_lvl(uint32_t page_size,
 374                         uint32_t entry_size,
 375                         uint32_t num_entries,
 376                         uint64_t *num_data_pages)
 377 {
 378         uint64_t lvl_data_size = page_size;
 379         int lvl = TF_PT_LVL_0;
 380         uint64_t data_size;
 381
 382         *num_data_pages = 0;
 383         data_size = (uint64_t)num_entries * entry_size;
 384
 385         while (lvl_data_size < data_size) {
 386                 lvl++;
 387
 388                 if (lvl == TF_PT_LVL_1)
 389                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 390                                 page_size;
 391                 else if (lvl == TF_PT_LVL_2)
 392                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 393                                 MAX_PAGE_PTRS(page_size) * page_size;
 394                 else
 395                         return -ENOMEM;
 396         }
 397
 398         *num_data_pages = roundup(data_size, page_size) / page_size;
 399
 400         return lvl;
 401 }
 402
 403 /**
 404  * Size the EM table based on capabilities
 405  *
 406  * [in] tbl
 407  *   EM table to size
 408  *
 409  * Returns:
 410  *   0        - Success
 411  *   - EINVAL - Parameter error
 412  *   - ENOMEM - Out of memory
 413  */
 414 int
 415 tf_em_size_table(struct hcapi_cfa_em_table *tbl,
 416                  uint32_t page_size)
 417 {
 418         uint64_t num_data_pages;
 419         uint32_t *page_cnt;
 420         int max_lvl;
 421         uint32_t num_entries;
 422         uint32_t cnt = TF_EM_MIN_ENTRIES;
 423
 424         /* Ignore entry if both size and number are zero */
 425         if (!tbl->entry_size && !tbl->num_entries)
 426                 return 0;
 427
 428         /* If only one is set then error */
 429         if (!tbl->entry_size || !tbl->num_entries)
 430                 return -EINVAL;
 431
 432         /* Determine number of page table levels and the number
 433          * of data pages needed to process the given eem table.
 434          */
 435         if (tbl->type == TF_RECORD_TABLE) {
 436                 /*
 437                  * For action records just a memory size is provided. Work
 438                  * backwards to resolve to number of entries
 439                  */
 440                 num_entries = tbl->num_entries / tbl->entry_size;
 441                 if (num_entries < TF_EM_MIN_ENTRIES) {
 442                         num_entries = TF_EM_MIN_ENTRIES;
 443                 } else {
 444                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 445                                 cnt *= 2;
 446                         num_entries = cnt;
 447                 }
 448         } else {
 449                 num_entries = tbl->num_entries;
 450         }
 451
 452         max_lvl = tf_em_size_page_tbl_lvl(page_size,
 453                                           tbl->entry_size,
 454                                           tbl->num_entries,
 455                                           &num_data_pages);
 456         if (max_lvl < 0) {
 457                 TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 458                 TFP_DRV_LOG(WARNING,
 459                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 460                             tbl->type, (uint64_t)num_entries * tbl->entry_size,
 461                             page_size);
 462                 return -ENOMEM;
 463         }
 464
 465         tbl->num_lvl = max_lvl + 1;
 466         tbl->num_data_pages = num_data_pages;
 467
 468         /* Determine the number of pages needed at each level */
 469         page_cnt = tbl->page_cnt;
 470         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 471         tf_em_size_page_tbls(max_lvl, num_data_pages, page_size,
 472                                 page_cnt);
 473
 474         TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 475         TFP_DRV_LOG(INFO,
 476                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 \
 477                     " l0: %u l1: %u l2: %u\n",
 478                     max_lvl + 1,
 479                     (uint64_t)num_data_pages * page_size,
 480                     num_data_pages,
 481                     page_cnt[TF_PT_LVL_0],
 482                     page_cnt[TF_PT_LVL_1],
 483                     page_cnt[TF_PT_LVL_2]);
 484
 485         return 0;
 486 }
 487
 488 /**
 489  * Validates EM number of entries requested
 490  *
 491  * [in] tbl_scope_cb
 492  *   Pointer to table scope control block to be populated
 493  *
 494  * [in] parms
 495  *   Pointer to input parameters
 496  *
 497  * Returns:
 498  *   0       - Success
 499  *   -EINVAL - Parameter error
 500  */
 501 int
 502 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 503                            struct tf_alloc_tbl_scope_parms *parms)
 504 {
 505         uint32_t cnt;
 506
 507         if (parms->rx_mem_size_in_mb != 0) {
 508                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 509                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 510                                      + 1);
 511                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 512                                         TF_MEGABYTE) / (key_b + action_b);
 513
 514                 if (num_entries < TF_EM_MIN_ENTRIES) {
 515                         TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 516                                     "%uMB\n",
 517                                     parms->rx_mem_size_in_mb);
 518                         return -EINVAL;
 519                 }
 520
 521                 cnt = TF_EM_MIN_ENTRIES;
 522                 while (num_entries > cnt &&
 523                        cnt <= TF_EM_MAX_ENTRIES)
 524                         cnt *= 2;
 525
 526                 if (cnt > TF_EM_MAX_ENTRIES) {
 527                         TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 528                                     "%u\n",
 529                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 530                         return -EINVAL;
 531                 }
 532
 533                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 534         } else {
 535                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 536                     TF_EM_MIN_ENTRIES ||
 537                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 538                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 539                         TFP_DRV_LOG(ERR,
 540                                     "EEM: Invalid number of Rx flows "
 541                                     "requested:%u max:%u\n",
 542                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 543                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 544                         return -EINVAL;
 545                 }
 546
 547                 /* must be a power-of-2 supported value
 548                  * in the range 32K - 128M
 549                  */
 550                 cnt = TF_EM_MIN_ENTRIES;
 551                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 552                        cnt <= TF_EM_MAX_ENTRIES)
 553                         cnt *= 2;
 554
 555                 if (cnt > TF_EM_MAX_ENTRIES) {
 556                         TFP_DRV_LOG(ERR,
 557                                     "EEM: Invalid number of Rx requested: %u\n",
 558                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 559                         return -EINVAL;
 560                 }
 561         }
 562
 563         if (parms->tx_mem_size_in_mb != 0) {
 564                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 565                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 566                                      + 1);
 567                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 568                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 569                         (key_b + action_b);
 570
 571                 if (num_entries < TF_EM_MIN_ENTRIES) {
 572                         TFP_DRV_LOG(ERR,
 573                                     "EEM: Insufficient memory requested:%uMB\n",
 574                                     parms->rx_mem_size_in_mb);
 575                         return -EINVAL;
 576                 }
 577
 578                 cnt = TF_EM_MIN_ENTRIES;
 579                 while (num_entries > cnt &&
 580                        cnt <= TF_EM_MAX_ENTRIES)
 581                         cnt *= 2;
 582
 583                 if (cnt > TF_EM_MAX_ENTRIES) {
 584                         TFP_DRV_LOG(ERR,
 585                                     "EEM: Invalid number of Tx requested: %u\n",
 586                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 587                         return -EINVAL;
 588                 }
 589
 590                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 591         } else {
 592                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 593                     TF_EM_MIN_ENTRIES ||
 594                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 595                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 596                         TFP_DRV_LOG(ERR,
 597                                     "EEM: Invalid number of Tx flows "
 598                                     "requested:%u max:%u\n",
 599                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 600                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 601                         return -EINVAL;
 602                 }
 603
 604                 cnt = TF_EM_MIN_ENTRIES;
 605                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 606                        cnt <= TF_EM_MAX_ENTRIES)
 607                         cnt *= 2;
 608
 609                 if (cnt > TF_EM_MAX_ENTRIES) {
 610                         TFP_DRV_LOG(ERR,
 611                                     "EEM: Invalid number of Tx requested: %u\n",
 612                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 613                         return -EINVAL;
 614                 }
 615         }
 616
 617         if (parms->rx_num_flows_in_k != 0 &&
 618             parms->rx_max_key_sz_in_bits / 8 == 0) {
 619                 TFP_DRV_LOG(ERR,
 620                             "EEM: Rx key size required: %u\n",
 621                             (parms->rx_max_key_sz_in_bits));
 622                 return -EINVAL;
 623         }
 624
 625         if (parms->tx_num_flows_in_k != 0 &&
 626             parms->tx_max_key_sz_in_bits / 8 == 0) {
 627                 TFP_DRV_LOG(ERR,
 628                             "EEM: Tx key size required: %u\n",
 629                             (parms->tx_max_key_sz_in_bits));
 630                 return -EINVAL;
 631         }
 632         /* Rx */
 633         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 634                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 635         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 636                 parms->rx_max_key_sz_in_bits / 8;
 637
 638         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 639                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 640         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 641                 parms->rx_max_key_sz_in_bits / 8;
 642
 643         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 644                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 645         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 646                 parms->rx_max_action_entry_sz_in_bits / 8;
 647
 648         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries =
 649                 0;
 650
 651         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_ACTION_TABLE].num_entries =
 652                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 653         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_ACTION_TABLE].entry_size =
 654                 parms->rx_max_action_entry_sz_in_bits / 8;
 655
 656         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EM_LKUP_TABLE].num_entries =
 657                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 658         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EM_LKUP_TABLE].entry_size =
 659                 parms->rx_max_key_sz_in_bits / 8;
 660
 661         /* Tx */
 662         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 663                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 664         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 665                 parms->tx_max_key_sz_in_bits / 8;
 666
 667         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 668                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 669         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 670                 parms->tx_max_key_sz_in_bits / 8;
 671
 672         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 673                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 674         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 675                 parms->tx_max_action_entry_sz_in_bits / 8;
 676
 677         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries =
 678                 0;
 679
 680         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_ACTION_TABLE].num_entries =
 681                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 682         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_ACTION_TABLE].entry_size =
 683                 parms->tx_max_action_entry_sz_in_bits / 8;
 684
 685         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EM_LKUP_TABLE].num_entries =
 686                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 687         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EM_LKUP_TABLE].entry_size =
 688                 parms->tx_max_key_sz_in_bits / 8;
 689
 690         return 0;
 691 }
 692
 693 /** insert EEM entry API
 694  *
 695  * returns:
 696  *  0
 697  *  TF_ERR          - unable to get lock
 698  *
 699  * insert callback returns:
 700  *   0
 701  *   TF_ERR_EM_DUP  - key is already in table
 702  */
 703 static int
 704 tf_insert_eem_entry(struct tf_dev_info *dev,
 705                     struct tf_tbl_scope_cb *tbl_scope_cb,
 706                     struct tf_insert_em_entry_parms *parms)
 707 {
 708         uint32_t mask;
 709         uint32_t key0_hash;
 710         uint32_t key1_hash;
 711         uint32_t key0_index;
 712         uint32_t key1_index;
 713         struct cfa_p4_eem_64b_entry key_entry;
 714         uint32_t index;
 715         enum hcapi_cfa_em_table_type table_type;
 716         uint32_t gfid;
 717         struct hcapi_cfa_hwop op;
 718         struct hcapi_cfa_key_tbl key_tbl;
 719         struct hcapi_cfa_key_data key_obj;
 720         struct hcapi_cfa_key_loc key_loc;
 721         uint64_t big_hash;
 722         int rc;
 723
 724         /* Get mask to use on hash */
 725         mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
 726
 727         if (!mask)
 728                 return -EINVAL;
 729
 730 #ifdef TF_EEM_DEBUG
 731         dump_raw((uint8_t *)parms->key, TF_P4_HW_EM_KEY_MAX_SIZE + 4, "In Key");
 732 #endif
 733
 734         if (dev->ops->tf_dev_cfa_key_hash == NULL)
 735                 return -EINVAL;
 736
 737         big_hash = dev->ops->tf_dev_cfa_key_hash((uint64_t *)parms->key,
 738                                          (TF_P4_HW_EM_KEY_MAX_SIZE + 4) * 8);
 739         key0_hash = (uint32_t)(big_hash >> 32);
 740         key1_hash = (uint32_t)(big_hash & 0xFFFFFFFF);
 741
 742         key0_index = key0_hash & mask;
 743         key1_index = key1_hash & mask;
 744
 745 #ifdef TF_EEM_DEBUG
 746         TFP_DRV_LOG(DEBUG, "Key0 hash:0x%08x\n", key0_hash);
 747         TFP_DRV_LOG(DEBUG, "Key1 hash:0x%08x\n", key1_hash);
 748 #endif
 749         /*
 750          * Use the "result" arg to populate all of the key entry then
 751          * store the byte swapped "raw" entry in a local copy ready
 752          * for insertion in to the table.
 753          */
 754         tf_em_create_key_entry((struct cfa_p4_eem_entry_hdr *)parms->em_record,
 755                                 ((uint8_t *)parms->key),
 756                                 &key_entry);
 757
 758         /*
 759          * Try to add to Key0 table, if that does not work then
 760          * try the key1 table.
 761          */
 762         index = key0_index;
 763         op.opcode = HCAPI_CFA_HWOPS_ADD;
 764         key_tbl.base0 =
 765                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE];
 766         key_tbl.page_size = TF_EM_PAGE_SIZE;
 767         key_obj.offset = index * TF_P4_EM_KEY_RECORD_SIZE;
 768         key_obj.data = (uint8_t *)&key_entry;
 769         key_obj.size = TF_P4_EM_KEY_RECORD_SIZE;
 770
 771         rc = cfa_p4_devops.hcapi_cfa_key_hw_op(&op,
 772                                                &key_tbl,
 773                                                &key_obj,
 774                                                &key_loc);
 775
 776         if (rc == 0) {
 777                 table_type = TF_KEY0_TABLE;
 778         } else {
 779                 index = key1_index;
 780
 781                 key_tbl.base0 =
 782                         (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY1_TABLE];
 783                 key_obj.offset = index * TF_P4_EM_KEY_RECORD_SIZE;
 784
 785                 rc = cfa_p4_devops.hcapi_cfa_key_hw_op(&op,
 786                                                        &key_tbl,
 787                                                        &key_obj,
 788                                                        &key_loc);
 789                 if (rc != 0)
 790                         return rc;
 791
 792                 table_type = TF_KEY1_TABLE;
 793         }
 794
 795         TF_SET_GFID(gfid,
 796                     index,
 797                     table_type);
 798         TF_SET_FLOW_ID(parms->flow_id,
 799                        gfid,
 800                        TF_GFID_TABLE_EXTERNAL,
 801                        parms->dir);
 802         TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
 803                                      0,
 804                                      0,
 805                                      0,
 806                                      index,
 807                                      0,
 808                                      table_type);
 809
 810         return 0;
 811 }
 812
 813 /** delete EEM hash entry API
 814  *
 815  * returns:
 816  *   0
 817  *   -EINVAL      - parameter error
 818  *   TF_NO_SESSION    - bad session ID
 819  *   TF_ERR_TBL_SCOPE - invalid table scope
 820  *   TF_ERR_TBL_IF    - invalid table interface
 821  *
 822  * insert callback returns
 823  *   0
 824  *   TF_NO_EM_MATCH - entry not found
 825  */
 826 static int
 827 tf_delete_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 828                     struct tf_delete_em_entry_parms *parms)
 829 {
 830         enum hcapi_cfa_em_table_type hash_type;
 831         uint32_t index;
 832         struct hcapi_cfa_hwop op;
 833         struct hcapi_cfa_key_tbl key_tbl;
 834         struct hcapi_cfa_key_data key_obj;
 835         struct hcapi_cfa_key_loc key_loc;
 836         int rc;
 837
 838         TF_GET_HASH_TYPE_FROM_FLOW_HANDLE(parms->flow_handle, hash_type);
 839         TF_GET_INDEX_FROM_FLOW_HANDLE(parms->flow_handle, index);
 840
 841         op.opcode = HCAPI_CFA_HWOPS_DEL;
 842         key_tbl.base0 =
 843                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables
 844                         [(hash_type == 0 ? TF_KEY0_TABLE : TF_KEY1_TABLE)];
 845         key_tbl.page_size = TF_EM_PAGE_SIZE;
 846         key_obj.offset = index * TF_P4_EM_KEY_RECORD_SIZE;
 847         key_obj.data = NULL;
 848         key_obj.size = TF_P4_EM_KEY_RECORD_SIZE;
 849
 850         rc = cfa_p4_devops.hcapi_cfa_key_hw_op(&op,
 851                                                &key_tbl,
 852                                                &key_obj,
 853                                                &key_loc);
 854
 855         if (!rc)
 856                 return rc;
 857
 858         return 0;
 859 }
 860
 861 /** insert EM hash entry API
 862  *
 863  *    returns:
 864  *    0       - Success
 865  *    -EINVAL - Error
 866  */
 867 int
 868 tf_em_insert_ext_entry(struct tf *tfp,
 869                        struct tf_insert_em_entry_parms *parms)
 870 {
 871         int rc;
 872         struct tf_tbl_scope_cb *tbl_scope_cb;
 873         struct tf_session *tfs;
 874         struct tf_dev_info *dev;
 875
 876         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
 877         if (tbl_scope_cb == NULL) {
 878                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 879                 return -EINVAL;
 880         }
 881
 882         /* Retrieve the session information */
 883         rc = tf_session_get_session_internal(tfp, &tfs);
 884         if (rc)
 885                 return rc;
 886
 887         /* Retrieve the device information */
 888         rc = tf_session_get_device(tfs, &dev);
 889         if (rc)
 890                 return rc;
 891
 892         return tf_insert_eem_entry
 893                 (dev,
 894                  tbl_scope_cb,
 895                  parms);
 896 }
 897
 898 /** Delete EM hash entry API
 899  *
 900  *    returns:
 901  *    0       - Success
 902  *    -EINVAL - Error
 903  */
 904 int
 905 tf_em_delete_ext_entry(struct tf *tfp,
 906                        struct tf_delete_em_entry_parms *parms)
 907 {
 908         struct tf_tbl_scope_cb *tbl_scope_cb;
 909
 910         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
 911         if (tbl_scope_cb == NULL) {
 912                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb\n");
 913                 return -EINVAL;
 914         }
 915
 916         return tf_delete_eem_entry(tbl_scope_cb, parms);
 917 }
 918
 919
 920 int
 921 tf_em_ext_common_bind(struct tf *tfp,
 922                       struct tf_em_cfg_parms *parms)
 923 {
 924         int rc;
 925         int i;
 926         struct tf_rm_create_db_parms db_cfg = { 0 };
 927         uint8_t db_exists = 0;
 928         struct em_ext_db *ext_db;
 929         struct tfp_calloc_parms cparms;
 930
 931         TF_CHECK_PARMS2(tfp, parms);
 932
 933         if (init) {
 934                 TFP_DRV_LOG(ERR,
 935                             "EM Ext DB already initialized\n");
 936                 return -EINVAL;
 937         }
 938
 939         cparms.nitems = 1;
 940         cparms.size = sizeof(struct em_ext_db);
 941         cparms.alignment = 0;
 942         if (tfp_calloc(&cparms) != 0) {
 943                 TFP_DRV_LOG(ERR, "em_ext_db alloc error %s\n",
 944                             strerror(ENOMEM));
 945                 return -ENOMEM;
 946         }
 947
 948         ext_db = cparms.mem_va;
 949         ll_init(&ext_db->tbl_scope_ll);
 950         for (i = 0; i < TF_DIR_MAX; i++)
 951                 ext_db->eem_db[i] = NULL;
 952         tf_session_set_em_ext_db(tfp, ext_db);
 953
 954         db_cfg.module = TF_MODULE_TYPE_EM;
 955         db_cfg.num_elements = parms->num_elements;
 956         db_cfg.cfg = parms->cfg;
 957
 958         for (i = 0; i < TF_DIR_MAX; i++) {
 959                 db_cfg.dir = i;
 960                 db_cfg.alloc_cnt = parms->resources->em_cnt[i].cnt;
 961
 962                 /* Check if we got any request to support EEM, if so
 963                  * we build an EM Ext DB holding Table Scopes.
 964                  */
 965                 if (db_cfg.alloc_cnt[TF_EM_TBL_TYPE_TBL_SCOPE] == 0)
 966                         continue;
 967
 968                 db_cfg.rm_db = (void *)&ext_db->eem_db[i];
 969                 rc = tf_rm_create_db(tfp, &db_cfg);
 970                 if (rc) {
 971                         TFP_DRV_LOG(ERR,
 972                                     "%s: EM Ext DB creation failed\n",
 973                                     tf_dir_2_str(i));
 974
 975                         return rc;
 976                 }
 977                 db_exists = 1;
 978         }
 979
 980         if (db_exists)
 981                 init = 1;
 982
 983         mem_type = parms->mem_type;
 984
 985         return 0;
 986 }
 987
 988 int
 989 tf_em_ext_common_unbind(struct tf *tfp)
 990 {
 991         int rc;
 992         int i;
 993         struct tf_rm_free_db_parms fparms = { 0 };
 994         struct em_ext_db *ext_db = NULL;
 995         struct tf_session *tfs = NULL;
 996         struct tf_dev_info *dev;
 997         struct ll_entry *entry;
 998         struct tf_tbl_scope_cb *tbl_scope_cb = NULL;
 999         void *ext_ptr = NULL;
1000         struct tf_free_tbl_scope_parms tparms = { 0 };
1001
1002         TF_CHECK_PARMS1(tfp);
1003
1004         /* Bail if nothing has been initialized */
1005         if (!init) {
1006                 TFP_DRV_LOG(INFO,
1007                             "No EM Ext DBs created\n");
1008                 return 0;
1009         }
1010
1011         rc = tf_session_get_session_internal(tfp, &tfs);
1012         if (rc) {
1013                 TFP_DRV_LOG(ERR, "Failed to get tf_session, rc:%s\n",
1014                 strerror(-rc));
1015                 return rc;
1016         }
1017
1018         /* Retrieve the device information */
1019         rc = tf_session_get_device(tfs, &dev);
1020         if (rc) {
1021                 TFP_DRV_LOG(ERR,
1022                             "Failed to lookup device, rc:%s\n",
1023                             strerror(-rc));
1024                 return rc;
1025         }
1026
1027         rc = tf_session_get_em_ext_db(tfp, &ext_ptr);
1028         if (rc) {
1029                 TFP_DRV_LOG(ERR,
1030                             "Failed to get em_ext_db from session, rc:%s\n",
1031                             strerror(-rc));
1032                 return rc;
1033         }
1034         ext_db = (struct em_ext_db *)ext_ptr;
1035
1036         entry = ext_db->tbl_scope_ll.head;
1037         while (entry != NULL) {
1038                 tbl_scope_cb = (struct tf_tbl_scope_cb *)entry;
1039                 entry = entry->next;
1040                 tparms.tbl_scope_id = tbl_scope_cb->tbl_scope_id;
1041
1042                 if (dev->ops->tf_dev_free_tbl_scope) {
1043                         dev->ops->tf_dev_free_tbl_scope(tfp, &tparms);
1044                 } else {
1045                         /* should not reach here */
1046                         ll_delete(&ext_db->tbl_scope_ll, &tbl_scope_cb->ll_entry);
1047                         tfp_free(tbl_scope_cb);
1048                 }
1049         }
1050
1051         for (i = 0; i < TF_DIR_MAX; i++) {
1052                 if (ext_db->eem_db[i] == NULL)
1053                         continue;
1054
1055                 fparms.dir = i;
1056                 fparms.rm_db = ext_db->eem_db[i];
1057                 rc = tf_rm_free_db(tfp, &fparms);
1058                 if (rc)
1059                         return rc;
1060
1061                 ext_db->eem_db[i] = NULL;
1062         }
1063
1064         tfp_free(ext_db);
1065         tf_session_set_em_ext_db(tfp, NULL);
1066
1067         init = 0;
1068
1069         return 0;
1070 }
1071
1072 /**
1073  * Sets the specified external table type element.
1074  *
1075  * This API sets the specified element data
1076  *
1077  * [in] tfp
1078  *   Pointer to TF handle
1079  *
1080  * [in] parms
1081  *   Pointer to table set parameters
1082  *
1083  * Returns
1084  *   - (0) if successful.
1085  *   - (-EINVAL) on failure.
1086  */
1087 int tf_tbl_ext_common_set(struct tf *tfp,
1088                           struct tf_tbl_set_parms *parms)
1089 {
1090         int rc = 0;
1091         struct tf_tbl_scope_cb *tbl_scope_cb;
1092         uint32_t tbl_scope_id;
1093         struct hcapi_cfa_hwop op;
1094         struct hcapi_cfa_key_tbl key_tbl;
1095         struct hcapi_cfa_key_data key_obj;
1096         struct hcapi_cfa_key_loc key_loc;
1097
1098         TF_CHECK_PARMS2(tfp, parms);
1099
1100         if (parms->data == NULL) {
1101                 TFP_DRV_LOG(ERR,
1102                             "%s, invalid parms->data\n",
1103                             tf_dir_2_str(parms->dir));
1104                 return -EINVAL;
1105         }
1106
1107         tbl_scope_id = parms->tbl_scope_id;
1108
1109         if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1110                 TFP_DRV_LOG(ERR,
1111                             "%s, Table scope not allocated\n",
1112                             tf_dir_2_str(parms->dir));
1113                 return -EINVAL;
1114         }
1115
1116         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, tbl_scope_id);
1117         if (tbl_scope_cb == NULL) {
1118                 TFP_DRV_LOG(ERR,
1119                             "%s, table scope error\n",
1120                             tf_dir_2_str(parms->dir));
1121                 return -EINVAL;
1122         }
1123
1124         op.opcode = HCAPI_CFA_HWOPS_PUT;
1125         key_tbl.base0 =
1126                 (uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_RECORD_TABLE];
1127         key_tbl.page_size = TF_EM_PAGE_SIZE;
1128         key_obj.offset = parms->idx;
1129         key_obj.data = parms->data;
1130         key_obj.size = parms->data_sz_in_bytes;
1131
1132         rc = cfa_p4_devops.hcapi_cfa_key_hw_op(&op,
1133                                                &key_tbl,
1134                                                &key_obj,
1135                                                &key_loc);
1136
1137         return rc;
1138 }
1139
1140 int
1141 tf_em_ext_common_alloc(struct tf *tfp,
1142                        struct tf_alloc_tbl_scope_parms *parms)
1143 {
1144         return tf_em_ext_alloc(tfp, parms);
1145 }
1146
1147 int
1148 tf_em_ext_common_free(struct tf *tfp,
1149                       struct tf_free_tbl_scope_parms *parms)
1150 {
1151         return tf_em_ext_free(tfp, parms);
1152 }
1153
1154 int tf_em_ext_map_tbl_scope(struct tf *tfp,
1155                             struct tf_map_tbl_scope_parms *parms)
1156 {
1157         int rc = 0;
1158         struct tf_session *tfs;
1159         struct tf_tbl_scope_cb *tbl_scope_cb;
1160         struct tf_global_cfg_parms gcfg_parms = { 0 };
1161         struct tfp_calloc_parms aparms;
1162         uint32_t *data, *mask;
1163         uint32_t sz_in_bytes = 8;
1164         struct tf_dev_info *dev;
1165
1166         /* Retrieve the session information */
1167         rc = tf_session_get_session_internal(tfp, &tfs);
1168         if (rc)
1169                 return rc;
1170
1171         /* Retrieve the device information */
1172         rc = tf_session_get_device(tfs, &dev);
1173         if (rc)
1174                 return rc;
1175
1176         tbl_scope_cb = tf_em_ext_common_tbl_scope_find(tfp, parms->tbl_scope_id);
1177         if (tbl_scope_cb == NULL) {
1178                 TFP_DRV_LOG(ERR, "Invalid tbl_scope_cb tbl_scope_id(%d)\n",
1179                             parms->tbl_scope_id);
1180                 return -EINVAL;
1181         }
1182
1183         if (dev->ops->tf_dev_map_tbl_scope == NULL) {
1184                 rc = -EOPNOTSUPP;
1185                 TFP_DRV_LOG(ERR,
1186                             "Map table scope operation not supported, rc:%s\n",
1187                             strerror(-rc));
1188                 return rc;
1189         }
1190
1191         aparms.nitems = 2;
1192         aparms.size = sizeof(uint32_t);
1193         aparms.alignment = 0;
1194
1195         if (tfp_calloc(&aparms) != 0) {
1196                 TFP_DRV_LOG(ERR, "Map tbl scope alloc data error %s\n",
1197                             strerror(ENOMEM));
1198                 return -ENOMEM;
1199         }
1200         data = aparms.mem_va;
1201
1202         if (tfp_calloc(&aparms) != 0) {
1203                 TFP_DRV_LOG(ERR, "Map tbl scope alloc mask error %s\n",
1204                             strerror(ENOMEM));
1205                 rc = -ENOMEM;
1206                 goto clean;
1207         }
1208         mask = aparms.mem_va;
1209
1210         rc = dev->ops->tf_dev_map_parif(tfp, parms->parif_bitmask,
1211                                         tbl_scope_cb->pf,
1212                                         (uint8_t *)data, (uint8_t *)mask,
1213                                         sz_in_bytes);
1214
1215         if (rc) {
1216                 TFP_DRV_LOG(ERR,
1217                             "Map table scope config failure, rc:%s\n",
1218                             strerror(-rc));
1219                 goto cleaner;
1220         }
1221
1222         /* Note that TF_GLOBAL_CFG_INTERNAL_PARIF_2_PF is same as below enum */
1223         gcfg_parms.type = TF_GLOBAL_CFG_TYPE_MAX;
1224         gcfg_parms.offset = 0;
1225         gcfg_parms.config = (uint8_t *)data;
1226         gcfg_parms.config_mask = (uint8_t *)mask;
1227         gcfg_parms.config_sz_in_bytes = sizeof(uint64_t);
1228
1229
1230         rc = tf_msg_set_global_cfg(tfp, &gcfg_parms);
1231         if (rc) {
1232                 TFP_DRV_LOG(ERR,
1233                             "Map tbl scope, set failed, rc:%s\n",
1234                             strerror(-rc));
1235         }
1236 cleaner:
1237         tfp_free(mask);
1238 clean:
1239         tfp_free(data);
1240
1241         return rc;
1242 }