drivers/net/bnxt/tf_core/tf_tbl.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 /* Truflow Table APIs and supporting code */
   7
   8 #include <stdio.h>
   9 #include <string.h>
  10 #include <stdbool.h>
  11 #include <math.h>
  12 #include <sys/param.h>
  13 #include <rte_common.h>
  14 #include <rte_errno.h>
  15 #include "hsi_struct_def_dpdk.h"
  16
  17 #include "tf_core.h"
  18 #include "tf_util.h"
  19 #include "tf_em.h"
  20 #include "tf_msg.h"
  21 #include "tfp.h"
  22 #include "hwrm_tf.h"
  23 #include "bnxt.h"
  24 #include "tf_resources.h"
  25 #include "tf_rm.h"
  26 #include "tf_common.h"
  27
  28 #define PTU_PTE_VALID          0x1UL
  29 #define PTU_PTE_LAST           0x2UL
  30 #define PTU_PTE_NEXT_TO_LAST   0x4UL
  31
  32 /* Number of pointers per page_size */
  33 #define MAX_PAGE_PTRS(page_size)  ((page_size) / sizeof(void *))
  34
  35 #define TF_EM_PG_SZ_4K        (1 << 12)
  36 #define TF_EM_PG_SZ_8K        (1 << 13)
  37 #define TF_EM_PG_SZ_64K       (1 << 16)
  38 #define TF_EM_PG_SZ_256K      (1 << 18)
  39 #define TF_EM_PG_SZ_1M        (1 << 20)
  40 #define TF_EM_PG_SZ_2M        (1 << 21)
  41 #define TF_EM_PG_SZ_4M        (1 << 22)
  42 #define TF_EM_PG_SZ_1G        (1 << 30)
  43
  44 #define TF_EM_CTX_ID_INVALID   0xFFFF
  45
  46 #define TF_EM_MIN_ENTRIES     (1 << 15) /* 32K */
  47 #define TF_EM_MAX_ENTRIES     (1 << 27) /* 128M */
  48
  49 /**
  50  * Function to free a page table
  51  *
  52  * [in] tp
  53  *   Pointer to the page table to free
  54  */
  55 static void
  56 tf_em_free_pg_tbl(struct tf_em_page_tbl *tp)
  57 {
  58         uint32_t i;
  59
  60         for (i = 0; i < tp->pg_count; i++) {
  61                 if (!tp->pg_va_tbl[i]) {
  62                         PMD_DRV_LOG(WARNING,
  63                                     "No map for page %d table %016" PRIu64 "\n",
  64                                     i,
  65                                     (uint64_t)(uintptr_t)tp);
  66                         continue;
  67                 }
  68
  69                 tfp_free(tp->pg_va_tbl[i]);
  70                 tp->pg_va_tbl[i] = NULL;
  71         }
  72
  73         tp->pg_count = 0;
  74         tfp_free(tp->pg_va_tbl);
  75         tp->pg_va_tbl = NULL;
  76         tfp_free(tp->pg_pa_tbl);
  77         tp->pg_pa_tbl = NULL;
  78 }
  79
  80 /**
  81  * Function to free an EM table
  82  *
  83  * [in] tbl
  84  *   Pointer to the EM table to free
  85  */
  86 static void
  87 tf_em_free_page_table(struct tf_em_table *tbl)
  88 {
  89         struct tf_em_page_tbl *tp;
  90         int i;
  91
  92         for (i = 0; i < tbl->num_lvl; i++) {
  93                 tp = &tbl->pg_tbl[i];
  94
  95                 PMD_DRV_LOG(INFO,
  96                            "EEM: Freeing page table: size %u lvl %d cnt %u\n",
  97                            TF_EM_PAGE_SIZE,
  98                             i,
  99                             tp->pg_count);
 100
 101                 tf_em_free_pg_tbl(tp);
 102         }
 103
 104         tbl->l0_addr = NULL;
 105         tbl->l0_dma_addr = 0;
 106         tbl->num_lvl = 0;
 107         tbl->num_data_pages = 0;
 108 }
 109
 110 /**
 111  * Allocation of page tables
 112  *
 113  * [in] tfp
 114  *   Pointer to a TruFlow handle
 115  *
 116  * [in] pg_count
 117  *   Page count to allocate
 118  *
 119  * [in] pg_size
 120  *   Size of each page
 121  *
 122  * Returns:
 123  *   0       - Success
 124  *   -ENOMEM - Out of memory
 125  */
 126 static int
 127 tf_em_alloc_pg_tbl(struct tf_em_page_tbl *tp,
 128                    uint32_t pg_count,
 129                    uint32_t pg_size)
 130 {
 131         uint32_t i;
 132         struct tfp_calloc_parms parms;
 133
 134         parms.nitems = pg_count;
 135         parms.size = sizeof(void *);
 136         parms.alignment = 0;
 137
 138         if (tfp_calloc(&parms) != 0)
 139                 return -ENOMEM;
 140
 141         tp->pg_va_tbl = parms.mem_va;
 142
 143         if (tfp_calloc(&parms) != 0) {
 144                 tfp_free(tp->pg_va_tbl);
 145                 return -ENOMEM;
 146         }
 147
 148         tp->pg_pa_tbl = parms.mem_va;
 149
 150         tp->pg_count = 0;
 151         tp->pg_size = pg_size;
 152
 153         for (i = 0; i < pg_count; i++) {
 154                 parms.nitems = 1;
 155                 parms.size = pg_size;
 156                 parms.alignment = TF_EM_PAGE_ALIGNMENT;
 157
 158                 if (tfp_calloc(&parms) != 0)
 159                         goto cleanup;
 160
 161                 tp->pg_pa_tbl[i] = (uintptr_t)parms.mem_pa;
 162                 tp->pg_va_tbl[i] = parms.mem_va;
 163
 164                 memset(tp->pg_va_tbl[i], 0, pg_size);
 165                 tp->pg_count++;
 166         }
 167
 168         return 0;
 169
 170 cleanup:
 171         tf_em_free_pg_tbl(tp);
 172         return -ENOMEM;
 173 }
 174
 175 /**
 176  * Allocates EM page tables
 177  *
 178  * [in] tbl
 179  *   Table to allocate pages for
 180  *
 181  * Returns:
 182  *   0       - Success
 183  *   -ENOMEM - Out of memory
 184  */
 185 static int
 186 tf_em_alloc_page_table(struct tf_em_table *tbl)
 187 {
 188         struct tf_em_page_tbl *tp;
 189         int rc = 0;
 190         int i;
 191         uint32_t j;
 192
 193         for (i = 0; i < tbl->num_lvl; i++) {
 194                 tp = &tbl->pg_tbl[i];
 195
 196                 rc = tf_em_alloc_pg_tbl(tp,
 197                                         tbl->page_cnt[i],
 198                                         TF_EM_PAGE_SIZE);
 199                 if (rc) {
 200                         PMD_DRV_LOG(WARNING,
 201                                 "Failed to allocate page table: lvl: %d\n",
 202                                 i);
 203                         goto cleanup;
 204                 }
 205
 206                 for (j = 0; j < tp->pg_count; j++) {
 207                         PMD_DRV_LOG(INFO,
 208                                 "EEM: Allocated page table: size %u lvl %d cnt"
 209                                 " %u VA:%p PA:%p\n",
 210                                 TF_EM_PAGE_SIZE,
 211                                 i,
 212                                 tp->pg_count,
 213                                 (uint32_t *)tp->pg_va_tbl[j],
 214                                 (uint32_t *)(uintptr_t)tp->pg_pa_tbl[j]);
 215                 }
 216         }
 217         return rc;
 218
 219 cleanup:
 220         tf_em_free_page_table(tbl);
 221         return rc;
 222 }
 223
 224 /**
 225  * Links EM page tables
 226  *
 227  * [in] tp
 228  *   Pointer to page table
 229  *
 230  * [in] tp_next
 231  *   Pointer to the next page table
 232  *
 233  * [in] set_pte_last
 234  *   Flag controlling if the page table is last
 235  */
 236 static void
 237 tf_em_link_page_table(struct tf_em_page_tbl *tp,
 238                       struct tf_em_page_tbl *tp_next,
 239                       bool set_pte_last)
 240 {
 241         uint64_t *pg_pa = tp_next->pg_pa_tbl;
 242         uint64_t *pg_va;
 243         uint64_t valid;
 244         uint32_t k = 0;
 245         uint32_t i;
 246         uint32_t j;
 247
 248         for (i = 0; i < tp->pg_count; i++) {
 249                 pg_va = tp->pg_va_tbl[i];
 250
 251                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
 252                         if (k == tp_next->pg_count - 2 && set_pte_last)
 253                                 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
 254                         else if (k == tp_next->pg_count - 1 && set_pte_last)
 255                                 valid = PTU_PTE_LAST | PTU_PTE_VALID;
 256                         else
 257                                 valid = PTU_PTE_VALID;
 258
 259                         pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
 260                         if (++k >= tp_next->pg_count)
 261                                 return;
 262                 }
 263         }
 264 }
 265
 266 /**
 267  * Setup a EM page table
 268  *
 269  * [in] tbl
 270  *   Pointer to EM page table
 271  */
 272 static void
 273 tf_em_setup_page_table(struct tf_em_table *tbl)
 274 {
 275         struct tf_em_page_tbl *tp_next;
 276         struct tf_em_page_tbl *tp;
 277         bool set_pte_last = 0;
 278         int i;
 279
 280         for (i = 0; i < tbl->num_lvl - 1; i++) {
 281                 tp = &tbl->pg_tbl[i];
 282                 tp_next = &tbl->pg_tbl[i + 1];
 283                 if (i == tbl->num_lvl - 2)
 284                         set_pte_last = 1;
 285                 tf_em_link_page_table(tp, tp_next, set_pte_last);
 286         }
 287
 288         tbl->l0_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_va_tbl[0];
 289         tbl->l0_dma_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_pa_tbl[0];
 290 }
 291
 292 /**
 293  * Given the page size, size of each data item (entry size),
 294  * and the total number of entries needed, determine the number
 295  * of page table levels and the number of data pages required.
 296  *
 297  * [in] page_size
 298  *   Page size
 299  *
 300  * [in] entry_size
 301  *   Entry size
 302  *
 303  * [in] num_entries
 304  *   Number of entries needed
 305  *
 306  * [out] num_data_pages
 307  *   Number of pages required
 308  *
 309  * Returns:
 310  *   Success  - Number of EM page levels required
 311  *   -ENOMEM  - Out of memory
 312  */
 313 static int
 314 tf_em_size_page_tbl_lvl(uint32_t page_size,
 315                         uint32_t entry_size,
 316                         uint32_t num_entries,
 317                         uint64_t *num_data_pages)
 318 {
 319         uint64_t lvl_data_size = page_size;
 320         int lvl = TF_PT_LVL_0;
 321         uint64_t data_size;
 322
 323         *num_data_pages = 0;
 324         data_size = (uint64_t)num_entries * entry_size;
 325
 326         while (lvl_data_size < data_size) {
 327                 lvl++;
 328
 329                 if (lvl == TF_PT_LVL_1)
 330                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 331                                 page_size;
 332                 else if (lvl == TF_PT_LVL_2)
 333                         lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
 334                                 MAX_PAGE_PTRS(page_size) * page_size;
 335                 else
 336                         return -ENOMEM;
 337         }
 338
 339         *num_data_pages = roundup(data_size, page_size) / page_size;
 340
 341         return lvl;
 342 }
 343
 344 /**
 345  * Return the number of page table pages needed to
 346  * reference the given number of next level pages.
 347  *
 348  * [in] num_pages
 349  *   Number of EM pages
 350  *
 351  * [in] page_size
 352  *   Size of each EM page
 353  *
 354  * Returns:
 355  *   Number of EM page table pages
 356  */
 357 static uint32_t
 358 tf_em_page_tbl_pgcnt(uint32_t num_pages,
 359                      uint32_t page_size)
 360 {
 361         return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
 362                        MAX_PAGE_PTRS(page_size);
 363         return 0;
 364 }
 365
 366 /**
 367  * Given the number of data pages, page_size and the maximum
 368  * number of page table levels (already determined), size
 369  * the number of page table pages required at each level.
 370  *
 371  * [in] max_lvl
 372  *   Max number of levels
 373  *
 374  * [in] num_data_pages
 375  *   Number of EM data pages
 376  *
 377  * [in] page_size
 378  *   Size of an EM page
 379  *
 380  * [out] *page_cnt
 381  *   EM page count
 382  */
 383 static void
 384 tf_em_size_page_tbls(int max_lvl,
 385                      uint64_t num_data_pages,
 386                      uint32_t page_size,
 387                      uint32_t *page_cnt)
 388 {
 389         if (max_lvl == TF_PT_LVL_0) {
 390                 page_cnt[TF_PT_LVL_0] = num_data_pages;
 391         } else if (max_lvl == TF_PT_LVL_1) {
 392                 page_cnt[TF_PT_LVL_1] = num_data_pages;
 393                 page_cnt[TF_PT_LVL_0] =
 394                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 395         } else if (max_lvl == TF_PT_LVL_2) {
 396                 page_cnt[TF_PT_LVL_2] = num_data_pages;
 397                 page_cnt[TF_PT_LVL_1] =
 398                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
 399                 page_cnt[TF_PT_LVL_0] =
 400                 tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
 401         } else {
 402                 return;
 403         }
 404 }
 405
 406 /**
 407  * Size the EM table based on capabilities
 408  *
 409  * [in] tbl
 410  *   EM table to size
 411  *
 412  * Returns:
 413  *   0        - Success
 414  *   - EINVAL - Parameter error
 415  *   - ENOMEM - Out of memory
 416  */
 417 static int
 418 tf_em_size_table(struct tf_em_table *tbl)
 419 {
 420         uint64_t num_data_pages;
 421         uint32_t *page_cnt;
 422         int max_lvl;
 423         uint32_t num_entries;
 424         uint32_t cnt = TF_EM_MIN_ENTRIES;
 425
 426         /* Ignore entry if both size and number are zero */
 427         if (!tbl->entry_size && !tbl->num_entries)
 428                 return 0;
 429
 430         /* If only one is set then error */
 431         if (!tbl->entry_size || !tbl->num_entries)
 432                 return -EINVAL;
 433
 434         /* Determine number of page table levels and the number
 435          * of data pages needed to process the given eem table.
 436          */
 437         if (tbl->type == TF_RECORD_TABLE) {
 438                 /*
 439                  * For action records just a memory size is provided. Work
 440                  * backwards to resolve to number of entries
 441                  */
 442                 num_entries = tbl->num_entries / tbl->entry_size;
 443                 if (num_entries < TF_EM_MIN_ENTRIES) {
 444                         num_entries = TF_EM_MIN_ENTRIES;
 445                 } else {
 446                         while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
 447                                 cnt *= 2;
 448                         num_entries = cnt;
 449                 }
 450         } else {
 451                 num_entries = tbl->num_entries;
 452         }
 453
 454         max_lvl = tf_em_size_page_tbl_lvl(TF_EM_PAGE_SIZE,
 455                                           tbl->entry_size,
 456                                           tbl->num_entries,
 457                                           &num_data_pages);
 458         if (max_lvl < 0) {
 459                 PMD_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
 460                 PMD_DRV_LOG(WARNING,
 461                             "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
 462                             tbl->type,
 463                             (uint64_t)num_entries * tbl->entry_size,
 464                             TF_EM_PAGE_SIZE);
 465                 return -ENOMEM;
 466         }
 467
 468         tbl->num_lvl = max_lvl + 1;
 469         tbl->num_data_pages = num_data_pages;
 470
 471         /* Determine the number of pages needed at each level */
 472         page_cnt = tbl->page_cnt;
 473         memset(page_cnt, 0, sizeof(tbl->page_cnt));
 474         tf_em_size_page_tbls(max_lvl, num_data_pages, TF_EM_PAGE_SIZE,
 475                                 page_cnt);
 476
 477         PMD_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
 478         PMD_DRV_LOG(INFO,
 479                     "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 " l0: %u l1: %u l2: %u\n",
 480                     max_lvl + 1,
 481                     (uint64_t)num_data_pages * TF_EM_PAGE_SIZE,
 482                     num_data_pages,
 483                     page_cnt[TF_PT_LVL_0],
 484                     page_cnt[TF_PT_LVL_1],
 485                     page_cnt[TF_PT_LVL_2]);
 486
 487         return 0;
 488 }
 489
 490 /**
 491  * Unregisters EM Ctx in Firmware
 492  *
 493  * [in] tfp
 494  *   Pointer to a TruFlow handle
 495  *
 496  * [in] tbl_scope_cb
 497  *   Pointer to a table scope control block
 498  *
 499  * [in] dir
 500  *   Receive or transmit direction
 501  */
 502 static void
 503 tf_em_ctx_unreg(struct tf *tfp,
 504                 struct tf_tbl_scope_cb *tbl_scope_cb,
 505                 int dir)
 506 {
 507         struct tf_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 508         struct tf_em_table *tbl;
 509         int i;
 510
 511         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 512                 tbl = &ctxp->em_tables[i];
 513
 514                 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
 515                         tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
 516                         tf_em_free_page_table(tbl);
 517                 }
 518         }
 519 }
 520
 521 /**
 522  * Registers EM Ctx in Firmware
 523  *
 524  * [in] tfp
 525  *   Pointer to a TruFlow handle
 526  *
 527  * [in] tbl_scope_cb
 528  *   Pointer to a table scope control block
 529  *
 530  * [in] dir
 531  *   Receive or transmit direction
 532  *
 533  * Returns:
 534  *   0       - Success
 535  *   -ENOMEM - Out of Memory
 536  */
 537 static int
 538 tf_em_ctx_reg(struct tf *tfp,
 539               struct tf_tbl_scope_cb *tbl_scope_cb,
 540               int dir)
 541 {
 542         struct tf_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
 543         struct tf_em_table *tbl;
 544         int rc = 0;
 545         int i;
 546
 547         for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
 548                 tbl = &ctxp->em_tables[i];
 549
 550                 if (tbl->num_entries && tbl->entry_size) {
 551                         rc = tf_em_size_table(tbl);
 552
 553                         if (rc)
 554                                 goto cleanup;
 555
 556                         rc = tf_em_alloc_page_table(tbl);
 557                         if (rc)
 558                                 goto cleanup;
 559
 560                         tf_em_setup_page_table(tbl);
 561                         rc = tf_msg_em_mem_rgtr(tfp,
 562                                                 tbl->num_lvl - 1,
 563                                                 TF_EM_PAGE_SIZE_ENUM,
 564                                                 tbl->l0_dma_addr,
 565                                                 &tbl->ctx_id);
 566                         if (rc)
 567                                 goto cleanup;
 568                 }
 569         }
 570         return rc;
 571
 572 cleanup:
 573         tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
 574         return rc;
 575 }
 576
 577 /**
 578  * Validates EM number of entries requested
 579  *
 580  * [in] tbl_scope_cb
 581  *   Pointer to table scope control block to be populated
 582  *
 583  * [in] parms
 584  *   Pointer to input parameters
 585  *
 586  * Returns:
 587  *   0       - Success
 588  *   -EINVAL - Parameter error
 589  */
 590 static int
 591 tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
 592                            struct tf_alloc_tbl_scope_parms *parms)
 593 {
 594         uint32_t cnt;
 595
 596         if (parms->rx_mem_size_in_mb != 0) {
 597                 uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
 598                 uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
 599                                      + 1);
 600                 uint32_t num_entries = (parms->rx_mem_size_in_mb *
 601                                         TF_MEGABYTE) / (key_b + action_b);
 602
 603                 if (num_entries < TF_EM_MIN_ENTRIES) {
 604                         PMD_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
 605                                     "%uMB\n",
 606                                     parms->rx_mem_size_in_mb);
 607                         return -EINVAL;
 608                 }
 609
 610                 cnt = TF_EM_MIN_ENTRIES;
 611                 while (num_entries > cnt &&
 612                        cnt <= TF_EM_MAX_ENTRIES)
 613                         cnt *= 2;
 614
 615                 if (cnt > TF_EM_MAX_ENTRIES) {
 616                         PMD_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
 617                                     "%u\n",
 618                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 619                         return -EINVAL;
 620                 }
 621
 622                 parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
 623         } else {
 624                 if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
 625                     TF_EM_MIN_ENTRIES ||
 626                     (parms->rx_num_flows_in_k * TF_KILOBYTE) >
 627                     tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
 628                         PMD_DRV_LOG(ERR,
 629                                     "EEM: Invalid number of Rx flows "
 630                                     "requested:%u max:%u\n",
 631                                     parms->rx_num_flows_in_k * TF_KILOBYTE,
 632                         tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
 633                         return -EINVAL;
 634                 }
 635
 636                 /* must be a power-of-2 supported value
 637                  * in the range 32K - 128M
 638                  */
 639                 cnt = TF_EM_MIN_ENTRIES;
 640                 while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 641                        cnt <= TF_EM_MAX_ENTRIES)
 642                         cnt *= 2;
 643
 644                 if (cnt > TF_EM_MAX_ENTRIES) {
 645                         PMD_DRV_LOG(ERR,
 646                                     "EEM: Invalid number of Rx requested: %u\n",
 647                                     (parms->rx_num_flows_in_k * TF_KILOBYTE));
 648                         return -EINVAL;
 649                 }
 650         }
 651
 652         if (parms->tx_mem_size_in_mb != 0) {
 653                 uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
 654                 uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
 655                                      + 1);
 656                 uint32_t num_entries = (parms->tx_mem_size_in_mb *
 657                                         (TF_KILOBYTE * TF_KILOBYTE)) /
 658                         (key_b + action_b);
 659
 660                 if (num_entries < TF_EM_MIN_ENTRIES) {
 661                         PMD_DRV_LOG(ERR,
 662                                     "EEM: Insufficient memory requested:%uMB\n",
 663                                     parms->rx_mem_size_in_mb);
 664                         return -EINVAL;
 665                 }
 666
 667                 cnt = TF_EM_MIN_ENTRIES;
 668                 while (num_entries > cnt &&
 669                        cnt <= TF_EM_MAX_ENTRIES)
 670                         cnt *= 2;
 671
 672                 if (cnt > TF_EM_MAX_ENTRIES) {
 673                         PMD_DRV_LOG(ERR,
 674                                     "EEM: Invalid number of Tx requested: %u\n",
 675                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 676                         return -EINVAL;
 677                 }
 678
 679                 parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
 680         } else {
 681                 if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
 682                     TF_EM_MIN_ENTRIES ||
 683                     (parms->tx_num_flows_in_k * TF_KILOBYTE) >
 684                     tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
 685                         PMD_DRV_LOG(ERR,
 686                                     "EEM: Invalid number of Tx flows "
 687                                     "requested:%u max:%u\n",
 688                                     (parms->tx_num_flows_in_k * TF_KILOBYTE),
 689                         tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
 690                         return -EINVAL;
 691                 }
 692
 693                 cnt = TF_EM_MIN_ENTRIES;
 694                 while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
 695                        cnt <= TF_EM_MAX_ENTRIES)
 696                         cnt *= 2;
 697
 698                 if (cnt > TF_EM_MAX_ENTRIES) {
 699                         PMD_DRV_LOG(ERR,
 700                                     "EEM: Invalid number of Tx requested: %u\n",
 701                        (parms->tx_num_flows_in_k * TF_KILOBYTE));
 702                         return -EINVAL;
 703                 }
 704         }
 705
 706         if (parms->rx_num_flows_in_k != 0 &&
 707             (parms->rx_max_key_sz_in_bits / 8 == 0)) {
 708                 PMD_DRV_LOG(ERR,
 709                             "EEM: Rx key size required: %u\n",
 710                             (parms->rx_max_key_sz_in_bits));
 711                 return -EINVAL;
 712         }
 713
 714         if (parms->tx_num_flows_in_k != 0 &&
 715             (parms->tx_max_key_sz_in_bits / 8 == 0)) {
 716                 PMD_DRV_LOG(ERR,
 717                             "EEM: Tx key size required: %u\n",
 718                             (parms->tx_max_key_sz_in_bits));
 719                 return -EINVAL;
 720         }
 721         /* Rx */
 722         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
 723                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 724         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
 725                 parms->rx_max_key_sz_in_bits / 8;
 726
 727         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
 728                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 729         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
 730                 parms->rx_max_key_sz_in_bits / 8;
 731
 732         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
 733                 parms->rx_num_flows_in_k * TF_KILOBYTE;
 734         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
 735                 parms->rx_max_action_entry_sz_in_bits / 8;
 736
 737         tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries =
 738                 0;
 739
 740         /* Tx */
 741         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
 742                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 743         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
 744                 parms->tx_max_key_sz_in_bits / 8;
 745
 746         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
 747                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 748         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
 749                 parms->tx_max_key_sz_in_bits / 8;
 750
 751         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
 752                 parms->tx_num_flows_in_k * TF_KILOBYTE;
 753         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
 754                 parms->tx_max_action_entry_sz_in_bits / 8;
 755
 756         tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries =
 757                 0;
 758
 759         return 0;
 760 }
 761
 762 /**
 763  * Internal function to set a Table Entry. Supports all internal Table Types
 764  *
 765  * [in] tfp
 766  *   Pointer to TruFlow handle
 767  *
 768  * [in] parms
 769  *   Pointer to input parameters
 770  *
 771  * Returns:
 772  *   0       - Success
 773  *   -EINVAL - Parameter error
 774  */
 775 static int
 776 tf_set_tbl_entry_internal(struct tf *tfp,
 777                           struct tf_set_tbl_entry_parms *parms)
 778 {
 779         int rc;
 780         int id;
 781         uint32_t index;
 782         struct bitalloc *session_pool;
 783         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 784
 785         /* Lookup the pool using the table type of the element */
 786         rc = tf_rm_lookup_tbl_type_pool(tfs,
 787                                         parms->dir,
 788                                         parms->type,
 789                                         &session_pool);
 790         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 791         if (rc)
 792                 return rc;
 793
 794         index = parms->idx;
 795
 796         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
 797             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
 798             parms->type != TF_TBL_TYPE_MIRROR_CONFIG &&
 799             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
 800                 PMD_DRV_LOG(ERR,
 801                             "dir:%d, Type not supported, type:%d\n",
 802                             parms->dir,
 803                             parms->type);
 804                 return -EOPNOTSUPP;
 805         }
 806
 807         /* Adjust the returned index/offset as there is no guarantee
 808          * that the start is 0 at time of RM allocation
 809          */
 810         tf_rm_convert_index(tfs,
 811                             parms->dir,
 812                             parms->type,
 813                             TF_RM_CONVERT_RM_BASE,
 814                             parms->idx,
 815                             &index);
 816
 817         /* Verify that the entry has been previously allocated */
 818         id = ba_inuse(session_pool, index);
 819         if (id != 1) {
 820                 PMD_DRV_LOG(ERR,
 821                    "dir:%d, Invalid or not allocated index, type:%d, idx:%d\n",
 822                    parms->dir,
 823                    parms->type,
 824                    index);
 825                 return -EINVAL;
 826         }
 827
 828         /* Set the entry */
 829         rc = tf_msg_set_tbl_entry(tfp,
 830                                   parms->dir,
 831                                   parms->type,
 832                                   parms->data_sz_in_bytes,
 833                                   parms->data,
 834                                   parms->idx);
 835         if (rc) {
 836                 PMD_DRV_LOG(ERR,
 837                             "dir:%d, Set failed, type:%d, rc:%d\n",
 838                             parms->dir,
 839                             parms->type,
 840                             rc);
 841         }
 842
 843         return rc;
 844 }
 845
 846 /**
 847  * Internal function to get a Table Entry. Supports all Table Types
 848  * except the TF_TBL_TYPE_EXT as that is handled as a table scope.
 849  *
 850  * [in] tfp
 851  *   Pointer to TruFlow handle
 852  *
 853  * [in] parms
 854  *   Pointer to input parameters
 855  *
 856  * Returns:
 857  *   0       - Success
 858  *   -EINVAL - Parameter error
 859  */
 860 static int
 861 tf_get_tbl_entry_internal(struct tf *tfp,
 862                           struct tf_get_tbl_entry_parms *parms)
 863 {
 864         int rc;
 865         int id;
 866         uint32_t index;
 867         struct bitalloc *session_pool;
 868         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 869
 870         /* Lookup the pool using the table type of the element */
 871         rc = tf_rm_lookup_tbl_type_pool(tfs,
 872                                         parms->dir,
 873                                         parms->type,
 874                                         &session_pool);
 875         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 876         if (rc)
 877                 return rc;
 878
 879         index = parms->idx;
 880
 881         /* Adjust the returned index/offset as there is no guarantee
 882          * that the start is 0 at time of RM allocation
 883          */
 884         tf_rm_convert_index(tfs,
 885                             parms->dir,
 886                             parms->type,
 887                             TF_RM_CONVERT_RM_BASE,
 888                             parms->idx,
 889                             &index);
 890
 891         /* Verify that the entry has been previously allocated */
 892         id = ba_inuse(session_pool, index);
 893         if (id != 1) {
 894                 PMD_DRV_LOG(ERR,
 895                    "dir:%d, Invalid or not allocated index, type:%d, idx:%d\n",
 896                    parms->dir,
 897                    parms->type,
 898                    index);
 899                 return -EINVAL;
 900         }
 901
 902         /* Get the entry */
 903         rc = tf_msg_get_tbl_entry(tfp,
 904                                   parms->dir,
 905                                   parms->type,
 906                                   parms->data_sz_in_bytes,
 907                                   parms->data,
 908                                   parms->idx);
 909         if (rc) {
 910                 PMD_DRV_LOG(ERR,
 911                             "dir:%d, Get failed, type:%d, rc:%d\n",
 912                             parms->dir,
 913                             parms->type,
 914                             rc);
 915         }
 916
 917         return rc;
 918 }
 919
 920 /**
 921  * Internal function to get a Table Entry. Supports all Table Types
 922  * except the TF_TBL_TYPE_EXT as that is handled as a table scope.
 923  *
 924  * [in] tfp
 925  *   Pointer to TruFlow handle
 926  *
 927  * [in] parms
 928  *   Pointer to input parameters
 929  *
 930  * Returns:
 931  *   0       - Success
 932  *   -EINVAL - Parameter error
 933  */
 934 static int
 935 tf_get_bulk_tbl_entry_internal(struct tf *tfp,
 936                           struct tf_get_bulk_tbl_entry_parms *parms)
 937 {
 938         int rc;
 939         int id;
 940         uint32_t index;
 941         struct bitalloc *session_pool;
 942         struct tf_session *tfs = (struct tf_session *)(tfp->session->core_data);
 943
 944         /* Lookup the pool using the table type of the element */
 945         rc = tf_rm_lookup_tbl_type_pool(tfs,
 946                                         parms->dir,
 947                                         parms->type,
 948                                         &session_pool);
 949         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
 950         if (rc)
 951                 return rc;
 952
 953         index = parms->starting_idx;
 954
 955         /*
 956          * Adjust the returned index/offset as there is no guarantee
 957          * that the start is 0 at time of RM allocation
 958          */
 959         tf_rm_convert_index(tfs,
 960                             parms->dir,
 961                             parms->type,
 962                             TF_RM_CONVERT_RM_BASE,
 963                             parms->starting_idx,
 964                             &index);
 965
 966         /* Verify that the entry has been previously allocated */
 967         id = ba_inuse(session_pool, index);
 968         if (id != 1) {
 969                 TFP_DRV_LOG(ERR,
 970                    "%s, Invalid or not allocated index, type:%d, starting_idx:%d\n",
 971                    tf_dir_2_str(parms->dir),
 972                    parms->type,
 973                    index);
 974                 return -EINVAL;
 975         }
 976
 977         /* Get the entry */
 978         rc = tf_msg_get_bulk_tbl_entry(tfp, parms);
 979         if (rc) {
 980                 TFP_DRV_LOG(ERR,
 981                             "%s, Bulk get failed, type:%d, rc:%s\n",
 982                             tf_dir_2_str(parms->dir),
 983                             parms->type,
 984                             strerror(-rc));
 985         }
 986
 987         return rc;
 988 }
 989
 990 #if (TF_SHADOW == 1)
 991 /**
 992  * Allocate Tbl entry from the Shadow DB. Shadow DB is searched for
 993  * the requested entry. If found the ref count is incremente and
 994  * returned.
 995  *
 996  * [in] tfs
 997  *   Pointer to session
 998  * [in] parms
 999  *   Allocation parameters
1000  *
1001  * Return:
1002  *  0       - Success, entry found and ref count incremented
1003  *  -ENOENT - Failure, entry not found
1004  */
1005 static int
1006 tf_alloc_tbl_entry_shadow(struct tf_session *tfs __rte_unused,
1007                           struct tf_alloc_tbl_entry_parms *parms __rte_unused)
1008 {
1009         PMD_DRV_LOG(ERR,
1010                     "dir:%d, Entry Alloc with search not supported\n",
1011                     parms->dir);
1012
1013
1014         return -EOPNOTSUPP;
1015 }
1016
1017 /**
1018  * Free Tbl entry from the Shadow DB. Shadow DB is searched for
1019  * the requested entry. If found the ref count is decremente and
1020  * new ref_count returned.
1021  *
1022  * [in] tfs
1023  *   Pointer to session
1024  * [in] parms
1025  *   Allocation parameters
1026  *
1027  * Return:
1028  *  0       - Success, entry found and ref count decremented
1029  *  -ENOENT - Failure, entry not found
1030  */
1031 static int
1032 tf_free_tbl_entry_shadow(struct tf_session *tfs,
1033                          struct tf_free_tbl_entry_parms *parms)
1034 {
1035         PMD_DRV_LOG(ERR,
1036                     "dir:%d, Entry Free with search not supported\n",
1037                     parms->dir);
1038
1039         return -EOPNOTSUPP;
1040 }
1041 #endif /* TF_SHADOW */
1042
1043 /**
1044  * Create External Tbl pool of memory indexes.
1045  *
1046  * [in] dir
1047  *   direction
1048  * [in] tbl_scope_cb
1049  *   pointer to the table scope
1050  * [in] num_entries
1051  *   number of entries to write
1052  * [in] entry_sz_bytes
1053  *   size of each entry
1054  *
1055  * Return:
1056  *  0       - Success, entry allocated - no search support
1057  *  -ENOMEM -EINVAL -EOPNOTSUPP
1058  *          - Failure, entry not allocated, out of resources
1059  */
1060 static int
1061 tf_create_tbl_pool_external(enum tf_dir dir,
1062                             struct tf_tbl_scope_cb *tbl_scope_cb,
1063                             uint32_t num_entries,
1064                             uint32_t entry_sz_bytes)
1065 {
1066         struct tfp_calloc_parms parms;
1067         uint32_t i;
1068         int32_t j;
1069         int rc = 0;
1070         struct stack *pool = &tbl_scope_cb->ext_act_pool[dir];
1071
1072         parms.nitems = num_entries;
1073         parms.size = sizeof(uint32_t);
1074         parms.alignment = 0;
1075
1076         if (tfp_calloc(&parms) != 0) {
1077                 PMD_DRV_LOG(ERR, "%d: TBL: external pool failure %s\n",
1078                             dir, strerror(-ENOMEM));
1079                 return -ENOMEM;
1080         }
1081
1082         /* Create empty stack
1083          */
1084         rc = stack_init(num_entries, parms.mem_va, pool);
1085
1086         if (rc != 0) {
1087                 PMD_DRV_LOG(ERR, "%d: TBL: stack init failure %s\n",
1088                             dir, strerror(-rc));
1089                 goto cleanup;
1090         }
1091
1092         /* Save the  malloced memory address so that it can
1093          * be freed when the table scope is freed.
1094          */
1095         tbl_scope_cb->ext_act_pool_mem[dir] = (uint32_t *)parms.mem_va;
1096
1097         /* Fill pool with indexes in reverse
1098          */
1099         j = (num_entries - 1) * entry_sz_bytes;
1100
1101         for (i = 0; i < num_entries; i++) {
1102                 rc = stack_push(pool, j);
1103                 if (rc != 0) {
1104                         PMD_DRV_LOG(ERR, "%s TBL: stack failure %s\n",
1105                                     tf_dir_2_str(dir), strerror(-rc));
1106                         goto cleanup;
1107                 }
1108
1109                 if (j < 0) {
1110                         PMD_DRV_LOG(ERR, "%d TBL: invalid offset (%d)\n",
1111                                     dir, j);
1112                         goto cleanup;
1113                 }
1114                 j -= entry_sz_bytes;
1115         }
1116
1117         if (!stack_is_full(pool)) {
1118                 rc = -EINVAL;
1119                 PMD_DRV_LOG(ERR, "%d TBL: stack failure %s\n",
1120                             dir, strerror(-rc));
1121                 goto cleanup;
1122         }
1123         return 0;
1124 cleanup:
1125         tfp_free((void *)parms.mem_va);
1126         return rc;
1127 }
1128
1129 /**
1130  * Destroy External Tbl pool of memory indexes.
1131  *
1132  * [in] dir
1133  *   direction
1134  * [in] tbl_scope_cb
1135  *   pointer to the table scope
1136  *
1137  */
1138 static void
1139 tf_destroy_tbl_pool_external(enum tf_dir dir,
1140                              struct tf_tbl_scope_cb *tbl_scope_cb)
1141 {
1142         uint32_t *ext_act_pool_mem =
1143                 tbl_scope_cb->ext_act_pool_mem[dir];
1144
1145         tfp_free(ext_act_pool_mem);
1146 }
1147
1148 /**
1149  * Allocate External Tbl entry from the Session Pool.
1150  *
1151  * [in] tfp
1152  *   Pointer to Truflow Handle
1153  * [in] parms
1154  *   Allocation parameters
1155  *
1156  * Return:
1157  *  0       - Success, entry allocated - no search support
1158  *  -ENOMEM -EINVAL -EOPNOTSUPP
1159  *          - Failure, entry not allocated, out of resources
1160  */
1161 static int
1162 tf_alloc_tbl_entry_pool_external(struct tf *tfp,
1163                                  struct tf_alloc_tbl_entry_parms *parms)
1164 {
1165         int rc;
1166         uint32_t index;
1167         struct tf_session *tfs;
1168         struct tf_tbl_scope_cb *tbl_scope_cb;
1169         struct stack *pool;
1170
1171         /* Check parameters */
1172         if (tfp == NULL || parms == NULL) {
1173                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1174                 return -EINVAL;
1175         }
1176
1177         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1178                 PMD_DRV_LOG(ERR,
1179                             "dir:%d, Session info invalid\n",
1180                             parms->dir);
1181                 return -EINVAL;
1182         }
1183
1184         tfs = (struct tf_session *)(tfp->session->core_data);
1185
1186         /* Get the pool info from the table scope
1187          */
1188         tbl_scope_cb = tbl_scope_cb_find(tfs, parms->tbl_scope_id);
1189
1190         if (tbl_scope_cb == NULL) {
1191                 PMD_DRV_LOG(ERR,
1192                                         "%s, table scope not allocated\n",
1193                                         tf_dir_2_str(parms->dir));
1194                 return -EINVAL;
1195         }
1196         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
1197
1198         /* Allocate an element
1199          */
1200         rc = stack_pop(pool, &index);
1201
1202         if (rc != 0) {
1203                 PMD_DRV_LOG(ERR,
1204                    "dir:%d, Allocation failed, type:%d\n",
1205                    parms->dir,
1206                    parms->type);
1207                 return rc;
1208         }
1209         parms->idx = index;
1210         return rc;
1211 }
1212
1213 /**
1214  * Allocate Internal Tbl entry from the Session Pool.
1215  *
1216  * [in] tfp
1217  *   Pointer to Truflow Handle
1218  * [in] parms
1219  *   Allocation parameters
1220  *
1221  * Return:
1222  *  0       - Success, entry found and ref count decremented
1223  *  -ENOMEM - Failure, entry not allocated, out of resources
1224  */
1225 static int
1226 tf_alloc_tbl_entry_pool_internal(struct tf *tfp,
1227                                  struct tf_alloc_tbl_entry_parms *parms)
1228 {
1229         int rc;
1230         int id;
1231         int free_cnt;
1232         uint32_t index;
1233         struct bitalloc *session_pool;
1234         struct tf_session *tfs;
1235
1236         /* Check parameters */
1237         if (tfp == NULL || parms == NULL) {
1238                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1239                 return -EINVAL;
1240         }
1241
1242         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1243                 PMD_DRV_LOG(ERR,
1244                             "dir:%d, Session info invalid\n",
1245                             parms->dir);
1246                 return -EINVAL;
1247         }
1248
1249         tfs = (struct tf_session *)(tfp->session->core_data);
1250
1251         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1252             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1253             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1254             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1255             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1256             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B &&
1257             parms->type != TF_TBL_TYPE_MIRROR_CONFIG &&
1258             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
1259                 PMD_DRV_LOG(ERR,
1260                             "dir:%d, Type not supported, type:%d\n",
1261                             parms->dir,
1262                             parms->type);
1263                 return -EOPNOTSUPP;
1264         }
1265
1266         /* Lookup the pool using the table type of the element */
1267         rc = tf_rm_lookup_tbl_type_pool(tfs,
1268                                         parms->dir,
1269                                         parms->type,
1270                                         &session_pool);
1271         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1272         if (rc)
1273                 return rc;
1274
1275         id = ba_alloc(session_pool);
1276         if (id == -1) {
1277                 free_cnt = ba_free_count(session_pool);
1278
1279                 PMD_DRV_LOG(ERR,
1280                    "dir:%d, Allocation failed, type:%d, free:%d\n",
1281                    parms->dir,
1282                    parms->type,
1283                    free_cnt);
1284                 return -ENOMEM;
1285         }
1286
1287         /* Adjust the returned index/offset as there is no guarantee
1288          * that the start is 0 at time of RM allocation
1289          */
1290         tf_rm_convert_index(tfs,
1291                             parms->dir,
1292                             parms->type,
1293                             TF_RM_CONVERT_ADD_BASE,
1294                             id,
1295                             &index);
1296         parms->idx = index;
1297         return rc;
1298 }
1299
1300 /**
1301  * Free External Tbl entry to the session pool.
1302  *
1303  * [in] tfp
1304  *   Pointer to Truflow Handle
1305  * [in] parms
1306  *   Allocation parameters
1307  *
1308  * Return:
1309  *  0       - Success, entry freed
1310  *
1311  * - Failure, entry not successfully freed for these reasons
1312  *  -ENOMEM
1313  *  -EOPNOTSUPP
1314  *  -EINVAL
1315  */
1316 static int
1317 tf_free_tbl_entry_pool_external(struct tf *tfp,
1318                                 struct tf_free_tbl_entry_parms *parms)
1319 {
1320         int rc = 0;
1321         struct tf_session *tfs;
1322         uint32_t index;
1323         struct tf_tbl_scope_cb *tbl_scope_cb;
1324         struct stack *pool;
1325
1326         /* Check parameters */
1327         if (tfp == NULL || parms == NULL) {
1328                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1329                 return -EINVAL;
1330         }
1331
1332         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1333                 PMD_DRV_LOG(ERR,
1334                             "dir:%d, Session info invalid\n",
1335                             parms->dir);
1336                 return -EINVAL;
1337         }
1338
1339         tfs = (struct tf_session *)(tfp->session->core_data);
1340
1341         /* Get the pool info from the table scope
1342          */
1343         tbl_scope_cb = tbl_scope_cb_find(tfs, parms->tbl_scope_id);
1344
1345         if (tbl_scope_cb == NULL) {
1346                 PMD_DRV_LOG(ERR,
1347                             "dir:%d, Session info invalid\n",
1348                             parms->dir);
1349                 return -EINVAL;
1350         }
1351         pool = &tbl_scope_cb->ext_act_pool[parms->dir];
1352
1353         index = parms->idx;
1354
1355         rc = stack_push(pool, index);
1356
1357         if (rc != 0) {
1358                 PMD_DRV_LOG(ERR,
1359                    "dir:%d, consistency error, stack full, type:%d, idx:%d\n",
1360                    parms->dir,
1361                    parms->type,
1362                    index);
1363         }
1364         return rc;
1365 }
1366
1367 /**
1368  * Free Internal Tbl entry from the Session Pool.
1369  *
1370  * [in] tfp
1371  *   Pointer to Truflow Handle
1372  * [in] parms
1373  *   Allocation parameters
1374  *
1375  * Return:
1376  *  0       - Success, entry found and ref count decremented
1377  *  -ENOMEM - Failure, entry not allocated, out of resources
1378  */
1379 static int
1380 tf_free_tbl_entry_pool_internal(struct tf *tfp,
1381                        struct tf_free_tbl_entry_parms *parms)
1382 {
1383         int rc = 0;
1384         int id;
1385         struct bitalloc *session_pool;
1386         struct tf_session *tfs;
1387         uint32_t index;
1388
1389         /* Check parameters */
1390         if (tfp == NULL || parms == NULL) {
1391                 PMD_DRV_LOG(ERR, "Invalid parameters\n");
1392                 return -EINVAL;
1393         }
1394
1395         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1396                 PMD_DRV_LOG(ERR,
1397                             "dir:%d, Session info invalid\n",
1398                             parms->dir);
1399                 return -EINVAL;
1400         }
1401
1402         tfs = (struct tf_session *)(tfp->session->core_data);
1403
1404         if (parms->type != TF_TBL_TYPE_FULL_ACT_RECORD &&
1405             parms->type != TF_TBL_TYPE_ACT_SP_SMAC &&
1406             parms->type != TF_TBL_TYPE_ACT_SP_SMAC_IPV4 &&
1407             parms->type != TF_TBL_TYPE_ACT_ENCAP_8B &&
1408             parms->type != TF_TBL_TYPE_ACT_ENCAP_16B &&
1409             parms->type != TF_TBL_TYPE_ACT_ENCAP_64B &&
1410             parms->type != TF_TBL_TYPE_ACT_STATS_64) {
1411                 PMD_DRV_LOG(ERR,
1412                             "dir:%d, Type not supported, type:%d\n",
1413                             parms->dir,
1414                             parms->type);
1415                 return -EOPNOTSUPP;
1416         }
1417
1418         /* Lookup the pool using the table type of the element */
1419         rc = tf_rm_lookup_tbl_type_pool(tfs,
1420                                         parms->dir,
1421                                         parms->type,
1422                                         &session_pool);
1423         /* Error logging handled by tf_rm_lookup_tbl_type_pool */
1424         if (rc)
1425                 return rc;
1426
1427         index = parms->idx;
1428
1429         /* Adjust the returned index/offset as there is no guarantee
1430          * that the start is 0 at time of RM allocation
1431          */
1432         tf_rm_convert_index(tfs,
1433                             parms->dir,
1434                             parms->type,
1435                             TF_RM_CONVERT_RM_BASE,
1436                             parms->idx,
1437                             &index);
1438
1439         /* Check if element was indeed allocated */
1440         id = ba_inuse_free(session_pool, index);
1441         if (id == -1) {
1442                 PMD_DRV_LOG(ERR,
1443                    "dir:%d, Element not previously alloc'ed, type:%d, idx:%d\n",
1444                    parms->dir,
1445                    parms->type,
1446                    index);
1447                 return -ENOMEM;
1448         }
1449
1450         return rc;
1451 }
1452
1453 /* API defined in tf_em.h */
1454 struct tf_tbl_scope_cb *
1455 tbl_scope_cb_find(struct tf_session *session,
1456                   uint32_t tbl_scope_id)
1457 {
1458         int i;
1459
1460         /* Check that id is valid */
1461         i = ba_inuse(session->tbl_scope_pool_rx, tbl_scope_id);
1462         if (i < 0)
1463                 return NULL;
1464
1465         for (i = 0; i < TF_NUM_TBL_SCOPE; i++) {
1466                 if (session->tbl_scopes[i].tbl_scope_id == tbl_scope_id)
1467                         return &session->tbl_scopes[i];
1468         }
1469
1470         return NULL;
1471 }
1472
1473 /* API defined in tf_core.h */
1474 int
1475 tf_free_eem_tbl_scope_cb(struct tf *tfp,
1476                          struct tf_free_tbl_scope_parms *parms)
1477 {
1478         int rc = 0;
1479         enum tf_dir  dir;
1480         struct tf_tbl_scope_cb *tbl_scope_cb;
1481         struct tf_session *session;
1482
1483         session = (struct tf_session *)(tfp->session->core_data);
1484
1485         tbl_scope_cb = tbl_scope_cb_find(session,
1486                                          parms->tbl_scope_id);
1487
1488         if (tbl_scope_cb == NULL)
1489                 return -EINVAL;
1490
1491         /* Free Table control block */
1492         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1493
1494         /* free table scope locks */
1495         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1496                 /* Free associated external pools
1497                  */
1498                 tf_destroy_tbl_pool_external(dir,
1499                                              tbl_scope_cb);
1500                 tf_msg_em_op(tfp,
1501                              dir,
1502                              HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
1503
1504                 /* free table scope and all associated resources */
1505                 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
1506         }
1507
1508         return rc;
1509 }
1510
1511 /* API defined in tf_em.h */
1512 int
1513 tf_alloc_eem_tbl_scope(struct tf *tfp,
1514                        struct tf_alloc_tbl_scope_parms *parms)
1515 {
1516         int rc;
1517         enum tf_dir dir;
1518         struct tf_tbl_scope_cb *tbl_scope_cb;
1519         struct tf_em_table *em_tables;
1520         int index;
1521         struct tf_session *session;
1522         struct tf_free_tbl_scope_parms free_parms;
1523
1524         /* check parameters */
1525         if (parms == NULL || tfp->session == NULL) {
1526                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1527                 return -EINVAL;
1528         }
1529
1530         session = (struct tf_session *)tfp->session->core_data;
1531
1532         /* Get Table Scope control block from the session pool */
1533         index = ba_alloc(session->tbl_scope_pool_rx);
1534         if (index == -1) {
1535                 PMD_DRV_LOG(ERR, "EEM: Unable to allocate table scope "
1536                             "Control Block\n");
1537                 return -ENOMEM;
1538         }
1539
1540         tbl_scope_cb = &session->tbl_scopes[index];
1541         tbl_scope_cb->index = index;
1542         tbl_scope_cb->tbl_scope_id = index;
1543         parms->tbl_scope_id = index;
1544
1545         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1546                 rc = tf_msg_em_qcaps(tfp,
1547                                      dir,
1548                                      &tbl_scope_cb->em_caps[dir]);
1549                 if (rc) {
1550                         PMD_DRV_LOG(ERR,
1551                                 "EEM: Unable to query for EEM capability\n");
1552                         goto cleanup;
1553                 }
1554         }
1555
1556         /*
1557          * Validate and setup table sizes
1558          */
1559         if (tf_em_validate_num_entries(tbl_scope_cb, parms))
1560                 goto cleanup;
1561
1562         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1563                 /*
1564                  * Allocate tables and signal configuration to FW
1565                  */
1566                 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
1567                 if (rc) {
1568                         PMD_DRV_LOG(ERR,
1569                                     "EEM: Unable to register for EEM ctx\n");
1570                         goto cleanup;
1571                 }
1572
1573                 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
1574                 rc = tf_msg_em_cfg(tfp,
1575                                    em_tables[TF_KEY0_TABLE].num_entries,
1576                                    em_tables[TF_KEY0_TABLE].ctx_id,
1577                                    em_tables[TF_KEY1_TABLE].ctx_id,
1578                                    em_tables[TF_RECORD_TABLE].ctx_id,
1579                                    em_tables[TF_EFC_TABLE].ctx_id,
1580                                    parms->hw_flow_cache_flush_timer,
1581                                    dir);
1582                 if (rc) {
1583                         PMD_DRV_LOG(ERR,
1584                                 "TBL: Unable to configure EEM in firmware\n");
1585                         goto cleanup_full;
1586                 }
1587
1588                 rc = tf_msg_em_op(tfp,
1589                                   dir,
1590                                   HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
1591
1592                 if (rc) {
1593                         PMD_DRV_LOG(ERR,
1594                                     "EEM: Unable to enable EEM in firmware\n");
1595                         goto cleanup_full;
1596                 }
1597
1598                 /* Allocate the pool of offsets of the external memory.
1599                  * Initially, this is a single fixed size pool for all external
1600                  * actions related to a single table scope.
1601                  */
1602                 rc = tf_create_tbl_pool_external(dir,
1603                                     tbl_scope_cb,
1604                                     em_tables[TF_RECORD_TABLE].num_entries,
1605                                     em_tables[TF_RECORD_TABLE].entry_size);
1606                 if (rc) {
1607                         PMD_DRV_LOG(ERR,
1608                                     "%d TBL: Unable to allocate idx pools %s\n",
1609                                     dir,
1610                                     strerror(-rc));
1611                         goto cleanup_full;
1612                 }
1613         }
1614
1615         return 0;
1616
1617 cleanup_full:
1618         free_parms.tbl_scope_id = index;
1619         tf_free_eem_tbl_scope_cb(tfp, &free_parms);
1620         return -EINVAL;
1621
1622 cleanup:
1623         /* Free Table control block */
1624         ba_free(session->tbl_scope_pool_rx, tbl_scope_cb->index);
1625         return -EINVAL;
1626 }
1627
1628 /* API defined in tf_core.h */
1629 int
1630 tf_set_tbl_entry(struct tf *tfp,
1631                  struct tf_set_tbl_entry_parms *parms)
1632 {
1633         int rc = 0;
1634         struct tf_tbl_scope_cb *tbl_scope_cb;
1635         struct tf_session *session;
1636
1637         if (tfp == NULL || parms == NULL || parms->data == NULL)
1638                 return -EINVAL;
1639
1640         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1641                 PMD_DRV_LOG(ERR,
1642                             "dir:%d, Session info invalid\n",
1643                             parms->dir);
1644                 return -EINVAL;
1645         }
1646
1647         if (parms->type == TF_TBL_TYPE_EXT) {
1648                 void *base_addr;
1649                 uint32_t offset = parms->idx;
1650                 uint32_t tbl_scope_id;
1651
1652                 session = (struct tf_session *)(tfp->session->core_data);
1653
1654                 tbl_scope_id = parms->tbl_scope_id;
1655
1656                 if (tbl_scope_id == TF_TBL_SCOPE_INVALID)  {
1657                         PMD_DRV_LOG(ERR,
1658                                     "dir:%d, Table scope not allocated\n",
1659                                     parms->dir);
1660                         return -EINVAL;
1661                 }
1662
1663                 /* Get the table scope control block associated with the
1664                  * external pool
1665                  */
1666                 tbl_scope_cb = tbl_scope_cb_find(session, tbl_scope_id);
1667
1668                 if (tbl_scope_cb == NULL)
1669                         return -EINVAL;
1670
1671                 /* External table, implicitly the Action table */
1672                 base_addr = tf_em_get_table_page(tbl_scope_cb,
1673                                                  parms->dir,
1674                                                  offset,
1675                                                  TF_RECORD_TABLE);
1676                 if (base_addr == NULL) {
1677                         PMD_DRV_LOG(ERR,
1678                                     "dir:%d, Base address lookup failed\n",
1679                                     parms->dir);
1680                         return -EINVAL;
1681                 }
1682
1683                 offset %= TF_EM_PAGE_SIZE;
1684                 rte_memcpy((char *)base_addr + offset,
1685                            parms->data,
1686                            parms->data_sz_in_bytes);
1687         } else {
1688                 /* Internal table type processing */
1689                 rc = tf_set_tbl_entry_internal(tfp, parms);
1690                 if (rc) {
1691                         PMD_DRV_LOG(ERR,
1692                                     "dir:%d, Set failed, type:%d, rc:%d\n",
1693                                     parms->dir,
1694                                     parms->type,
1695                                     rc);
1696                 }
1697         }
1698
1699         return rc;
1700 }
1701
1702 /* API defined in tf_core.h */
1703 int
1704 tf_get_tbl_entry(struct tf *tfp,
1705                  struct tf_get_tbl_entry_parms *parms)
1706 {
1707         int rc = 0;
1708
1709         if (tfp == NULL || parms == NULL)
1710                 return -EINVAL;
1711
1712         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1713                 PMD_DRV_LOG(ERR,
1714                             "dir:%d, Session info invalid\n",
1715                             parms->dir);
1716                 return -EINVAL;
1717         }
1718
1719         if (parms->type == TF_TBL_TYPE_EXT) {
1720                 PMD_DRV_LOG(ERR,
1721                             "dir:%d, External table type not supported\n",
1722                             parms->dir);
1723
1724                 rc = -EOPNOTSUPP;
1725         } else {
1726                 /* Internal table type processing */
1727                 rc = tf_get_tbl_entry_internal(tfp, parms);
1728                 if (rc)
1729                         PMD_DRV_LOG(ERR,
1730                                     "dir:%d, Get failed, type:%d, rc:%d\n",
1731                                     parms->dir,
1732                                     parms->type,
1733                                     rc);
1734         }
1735
1736         return rc;
1737 }
1738
1739 /* API defined in tf_core.h */
1740 int
1741 tf_get_bulk_tbl_entry(struct tf *tfp,
1742                  struct tf_get_bulk_tbl_entry_parms *parms)
1743 {
1744         int rc = 0;
1745
1746         TF_CHECK_PARMS_SESSION(tfp, parms);
1747
1748         if (parms->type == TF_TBL_TYPE_EXT) {
1749                 /* Not supported, yet */
1750                 TFP_DRV_LOG(ERR,
1751                             "%s, External table type not supported\n",
1752                             tf_dir_2_str(parms->dir));
1753
1754                 rc = -EOPNOTSUPP;
1755         } else {
1756                 /* Internal table type processing */
1757                 rc = tf_get_bulk_tbl_entry_internal(tfp, parms);
1758                 if (rc)
1759                         TFP_DRV_LOG(ERR,
1760                                     "%s, Bulk get failed, type:%d, rc:%s\n",
1761                                     tf_dir_2_str(parms->dir),
1762                                     parms->type,
1763                                     strerror(-rc));
1764         }
1765
1766         return rc;
1767 }
1768
1769 /* API defined in tf_core.h */
1770 int
1771 tf_alloc_tbl_scope(struct tf *tfp,
1772                    struct tf_alloc_tbl_scope_parms *parms)
1773 {
1774         int rc;
1775
1776         /* check parameters */
1777         if (parms == NULL || tfp == NULL) {
1778                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1779                 return -EINVAL;
1780         }
1781
1782         rc = tf_alloc_eem_tbl_scope(tfp, parms);
1783
1784         return rc;
1785 }
1786
1787 /* API defined in tf_core.h */
1788 int
1789 tf_free_tbl_scope(struct tf *tfp,
1790                   struct tf_free_tbl_scope_parms *parms)
1791 {
1792         int rc;
1793
1794         /* check parameters */
1795         if (parms == NULL || tfp == NULL) {
1796                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1797                 return -EINVAL;
1798         }
1799
1800         /* free table scope and all associated resources */
1801         rc = tf_free_eem_tbl_scope_cb(tfp, parms);
1802
1803         return rc;
1804 }
1805
1806 /* API defined in tf_core.h */
1807 int
1808 tf_alloc_tbl_entry(struct tf *tfp,
1809                    struct tf_alloc_tbl_entry_parms *parms)
1810 {
1811         int rc;
1812 #if (TF_SHADOW == 1)
1813         struct tf_session *tfs;
1814 #endif /* TF_SHADOW */
1815
1816         /* Check parameters */
1817         if (parms == NULL || tfp == NULL) {
1818                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1819                 return -EINVAL;
1820         }
1821         /*
1822          * No shadow copy support for external tables, allocate and return
1823          */
1824         if (parms->type == TF_TBL_TYPE_EXT) {
1825                 rc = tf_alloc_tbl_entry_pool_external(tfp, parms);
1826                 return rc;
1827         }
1828
1829 #if (TF_SHADOW == 1)
1830         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1831                 PMD_DRV_LOG(ERR,
1832                             "dir:%d, Session info invalid\n",
1833                             parms->dir);
1834                 return -EINVAL;
1835         }
1836
1837         tfs = (struct tf_session *)(tfp->session->core_data);
1838
1839         /* Search the Shadow DB for requested element. If not found go
1840          * allocate one from the Session Pool
1841          */
1842         if (parms->search_enable && tfs->shadow_copy) {
1843                 rc = tf_alloc_tbl_entry_shadow(tfs, parms);
1844                 /* Entry found and parms populated with return data */
1845                 if (rc == 0)
1846                         return rc;
1847         }
1848 #endif /* TF_SHADOW */
1849
1850         rc = tf_alloc_tbl_entry_pool_internal(tfp, parms);
1851         if (rc)
1852                 PMD_DRV_LOG(ERR, "dir%d, Alloc failed, rc:%d\n",
1853                             parms->dir,
1854                             rc);
1855
1856         return rc;
1857 }
1858
1859 /* API defined in tf_core.h */
1860 int
1861 tf_free_tbl_entry(struct tf *tfp,
1862                   struct tf_free_tbl_entry_parms *parms)
1863 {
1864         int rc;
1865 #if (TF_SHADOW == 1)
1866         struct tf_session *tfs;
1867 #endif /* TF_SHADOW */
1868
1869         /* Check parameters */
1870         if (parms == NULL || tfp == NULL) {
1871                 PMD_DRV_LOG(ERR, "TBL: Invalid parameters\n");
1872                 return -EINVAL;
1873         }
1874         /*
1875          * No shadow of external tables so just free the entry
1876          */
1877         if (parms->type == TF_TBL_TYPE_EXT) {
1878                 rc = tf_free_tbl_entry_pool_external(tfp, parms);
1879                 return rc;
1880         }
1881
1882 #if (TF_SHADOW == 1)
1883         if (tfp->session == NULL || tfp->session->core_data == NULL) {
1884                 PMD_DRV_LOG(ERR,
1885                             "dir:%d, Session info invalid\n",
1886                             parms->dir);
1887                 return -EINVAL;
1888         }
1889
1890         tfs = (struct tf_session *)(tfp->session->core_data);
1891
1892         /* Search the Shadow DB for requested element. If not found go
1893          * allocate one from the Session Pool
1894          */
1895         if (parms->search_enable && tfs->shadow_copy) {
1896                 rc = tf_free_tbl_entry_shadow(tfs, parms);
1897                 /* Entry free'ed and parms populated with return data */
1898                 if (rc == 0)
1899                         return rc;
1900         }
1901 #endif /* TF_SHADOW */
1902
1903         rc = tf_free_tbl_entry_pool_internal(tfp, parms);
1904
1905         if (rc)
1906                 PMD_DRV_LOG(ERR, "dir:%d, Alloc failed, rc:%d\n",
1907                             parms->dir,
1908                             rc);
1909         return rc;
1910 }
1911
1912
1913 static void
1914 tf_dump_link_page_table(struct tf_em_page_tbl *tp,
1915                         struct tf_em_page_tbl *tp_next)
1916 {
1917         uint64_t *pg_va;
1918         uint32_t i;
1919         uint32_t j;
1920         uint32_t k = 0;
1921
1922         printf("pg_count:%d pg_size:0x%x\n",
1923                tp->pg_count,
1924                tp->pg_size);
1925         for (i = 0; i < tp->pg_count; i++) {
1926                 pg_va = tp->pg_va_tbl[i];
1927                 printf("\t%p\n", (void *)pg_va);
1928                 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
1929                         printf("\t\t%p\n", (void *)(uintptr_t)pg_va[j]);
1930                         if (((pg_va[j] & 0x7) ==
1931                              tfp_cpu_to_le_64(PTU_PTE_LAST |
1932                                               PTU_PTE_VALID)))
1933                                 return;
1934
1935                         if (!(pg_va[j] & tfp_cpu_to_le_64(PTU_PTE_VALID))) {
1936                                 printf("** Invalid entry **\n");
1937                                 return;
1938                         }
1939
1940                         if (++k >= tp_next->pg_count) {
1941                                 printf("** Shouldn't get here **\n");
1942                                 return;
1943                         }
1944                 }
1945         }
1946 }
1947
1948 void tf_dump_dma(struct tf *tfp, uint32_t tbl_scope_id);
1949
1950 void tf_dump_dma(struct tf *tfp, uint32_t tbl_scope_id)
1951 {
1952         struct tf_session      *session;
1953         struct tf_tbl_scope_cb *tbl_scope_cb;
1954         struct tf_em_page_tbl *tp;
1955         struct tf_em_page_tbl *tp_next;
1956         struct tf_em_table *tbl;
1957         int i;
1958         int j;
1959         int dir;
1960
1961         printf("called %s\n", __func__);
1962
1963         /* find session struct */
1964         session = (struct tf_session *)tfp->session->core_data;
1965
1966         /* find control block for table scope */
1967         tbl_scope_cb = tbl_scope_cb_find(session,
1968                                          tbl_scope_id);
1969         if (tbl_scope_cb == NULL)
1970                 TFP_DRV_LOG(ERR, "No table scope\n");
1971
1972         for (dir = 0; dir < TF_DIR_MAX; dir++) {
1973                 printf("Direction %s:\n", (dir == TF_DIR_RX ? "Rx" : "Tx"));
1974
1975                 for (j = TF_KEY0_TABLE; j < TF_MAX_TABLE; j++) {
1976                         tbl = &tbl_scope_cb->em_ctx_info[dir].em_tables[j];
1977                         printf
1978         ("Table: j:%d type:%d num_entries:%d entry_size:0x%x num_lvl:%d ",
1979                                j,
1980                                tbl->type,
1981                                tbl->num_entries,
1982                                tbl->entry_size,
1983                                tbl->num_lvl);
1984                         if (tbl->pg_tbl[0].pg_va_tbl &&
1985                             tbl->pg_tbl[0].pg_pa_tbl)
1986                                 printf("%p %p\n",
1987                                tbl->pg_tbl[0].pg_va_tbl[0],
1988                                (void *)(uintptr_t)tbl->pg_tbl[0].pg_pa_tbl[0]);
1989                         for (i = 0; i < tbl->num_lvl - 1; i++) {
1990                                 printf("Level:%d\n", i);
1991                                 tp = &tbl->pg_tbl[i];
1992                                 tp_next = &tbl->pg_tbl[i + 1];
1993                                 tf_dump_link_page_table(tp, tp_next);
1994                         }
1995                         printf("\n");
1996                 }
1997         }
1998 }