+ key_entry->hdr.pointer = result->pointer;
+ memcpy(key_entry->key, in_key, TF_P4_HW_EM_KEY_MAX_SIZE + 4);
+}
+
+
+/**
+ * Return the number of page table pages needed to
+ * reference the given number of next level pages.
+ *
+ * [in] num_pages
+ * Number of EM pages
+ *
+ * [in] page_size
+ * Size of each EM page
+ *
+ * Returns:
+ * Number of EM page table pages
+ */
+static uint32_t
+tf_em_page_tbl_pgcnt(uint32_t num_pages,
+ uint32_t page_size)
+{
+ return roundup(num_pages, MAX_PAGE_PTRS(page_size)) /
+ MAX_PAGE_PTRS(page_size);
+ return 0;
+}
+
+/**
+ * Given the number of data pages, page_size and the maximum
+ * number of page table levels (already determined), size
+ * the number of page table pages required at each level.
+ *
+ * [in] max_lvl
+ * Max number of levels
+ *
+ * [in] num_data_pages
+ * Number of EM data pages
+ *
+ * [in] page_size
+ * Size of an EM page
+ *
+ * [out] *page_cnt
+ * EM page count
+ */
+static void
+tf_em_size_page_tbls(int max_lvl,
+ uint64_t num_data_pages,
+ uint32_t page_size,
+ uint32_t *page_cnt)
+{
+ if (max_lvl == TF_PT_LVL_0) {
+ page_cnt[TF_PT_LVL_0] = num_data_pages;
+ } else if (max_lvl == TF_PT_LVL_1) {
+ page_cnt[TF_PT_LVL_1] = num_data_pages;
+ page_cnt[TF_PT_LVL_0] =
+ tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
+ } else if (max_lvl == TF_PT_LVL_2) {
+ page_cnt[TF_PT_LVL_2] = num_data_pages;
+ page_cnt[TF_PT_LVL_1] =
+ tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_2], page_size);
+ page_cnt[TF_PT_LVL_0] =
+ tf_em_page_tbl_pgcnt(page_cnt[TF_PT_LVL_1], page_size);
+ } else {
+ return;
+ }
+}
+
+/**
+ * Given the page size, size of each data item (entry size),
+ * and the total number of entries needed, determine the number
+ * of page table levels and the number of data pages required.
+ *
+ * [in] page_size
+ * Page size
+ *
+ * [in] entry_size
+ * Entry size
+ *
+ * [in] num_entries
+ * Number of entries needed
+ *
+ * [out] num_data_pages
+ * Number of pages required
+ *
+ * Returns:
+ * Success - Number of EM page levels required
+ * -ENOMEM - Out of memory
+ */
+static int
+tf_em_size_page_tbl_lvl(uint32_t page_size,
+ uint32_t entry_size,
+ uint32_t num_entries,
+ uint64_t *num_data_pages)
+{
+ uint64_t lvl_data_size = page_size;
+ int lvl = TF_PT_LVL_0;
+ uint64_t data_size;
+
+ *num_data_pages = 0;
+ data_size = (uint64_t)num_entries * entry_size;
+
+ while (lvl_data_size < data_size) {
+ lvl++;
+
+ if (lvl == TF_PT_LVL_1)
+ lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
+ page_size;
+ else if (lvl == TF_PT_LVL_2)
+ lvl_data_size = (uint64_t)MAX_PAGE_PTRS(page_size) *
+ MAX_PAGE_PTRS(page_size) * page_size;
+ else
+ return -ENOMEM;
+ }
+
+ *num_data_pages = roundup(data_size, page_size) / page_size;
+
+ return lvl;
+}
+
+/**
+ * Size the EM table based on capabilities
+ *
+ * [in] tbl
+ * EM table to size
+ *
+ * Returns:
+ * 0 - Success
+ * - EINVAL - Parameter error
+ * - ENOMEM - Out of memory
+ */
+int
+tf_em_size_table(struct hcapi_cfa_em_table *tbl,
+ uint32_t page_size)
+{
+ uint64_t num_data_pages;
+ uint32_t *page_cnt;
+ int max_lvl;
+ uint32_t num_entries;
+ uint32_t cnt = TF_EM_MIN_ENTRIES;
+
+ /* Ignore entry if both size and number are zero */
+ if (!tbl->entry_size && !tbl->num_entries)
+ return 0;
+
+ /* If only one is set then error */
+ if (!tbl->entry_size || !tbl->num_entries)
+ return -EINVAL;
+
+ /* Determine number of page table levels and the number
+ * of data pages needed to process the given eem table.
+ */
+ if (tbl->type == TF_RECORD_TABLE) {
+ /*
+ * For action records just a memory size is provided. Work
+ * backwards to resolve to number of entries
+ */
+ num_entries = tbl->num_entries / tbl->entry_size;
+ if (num_entries < TF_EM_MIN_ENTRIES) {
+ num_entries = TF_EM_MIN_ENTRIES;
+ } else {
+ while (num_entries > cnt && cnt <= TF_EM_MAX_ENTRIES)
+ cnt *= 2;
+ num_entries = cnt;
+ }
+ } else {
+ num_entries = tbl->num_entries;
+ }
+
+ max_lvl = tf_em_size_page_tbl_lvl(page_size,
+ tbl->entry_size,
+ tbl->num_entries,
+ &num_data_pages);
+ if (max_lvl < 0) {
+ TFP_DRV_LOG(WARNING, "EEM: Failed to size page table levels\n");
+ TFP_DRV_LOG(WARNING,
+ "table: %d data-sz: %016" PRIu64 " page-sz: %u\n",
+ tbl->type, (uint64_t)num_entries * tbl->entry_size,
+ page_size);
+ return -ENOMEM;
+ }
+
+ tbl->num_lvl = max_lvl + 1;
+ tbl->num_data_pages = num_data_pages;
+
+ /* Determine the number of pages needed at each level */
+ page_cnt = tbl->page_cnt;
+ memset(page_cnt, 0, sizeof(tbl->page_cnt));
+ tf_em_size_page_tbls(max_lvl, num_data_pages, page_size,
+ page_cnt);
+
+ TFP_DRV_LOG(INFO, "EEM: Sized page table: %d\n", tbl->type);
+ TFP_DRV_LOG(INFO,
+ "EEM: lvls: %d sz: %016" PRIu64 " pgs: %016" PRIu64 \
+ " l0: %u l1: %u l2: %u\n",
+ max_lvl + 1,
+ (uint64_t)num_data_pages * page_size,
+ num_data_pages,
+ page_cnt[TF_PT_LVL_0],
+ page_cnt[TF_PT_LVL_1],
+ page_cnt[TF_PT_LVL_2]);
+
+ return 0;
+}
+
+/**
+ * Validates EM number of entries requested
+ *
+ * [in] tbl_scope_cb
+ * Pointer to table scope control block to be populated
+ *
+ * [in] parms
+ * Pointer to input parameters
+ *
+ * Returns:
+ * 0 - Success
+ * -EINVAL - Parameter error
+ */
+int
+tf_em_validate_num_entries(struct tf_tbl_scope_cb *tbl_scope_cb,
+ struct tf_alloc_tbl_scope_parms *parms)
+{
+ uint32_t cnt;
+
+ if (parms->rx_mem_size_in_mb != 0) {
+ uint32_t key_b = 2 * ((parms->rx_max_key_sz_in_bits / 8) + 1);
+ uint32_t action_b = ((parms->rx_max_action_entry_sz_in_bits / 8)
+ + 1);
+ uint32_t num_entries = (parms->rx_mem_size_in_mb *
+ TF_MEGABYTE) / (key_b + action_b);
+
+ if (num_entries < TF_EM_MIN_ENTRIES) {
+ TFP_DRV_LOG(ERR, "EEM: Insufficient memory requested:"
+ "%uMB\n",
+ parms->rx_mem_size_in_mb);
+ return -EINVAL;
+ }
+
+ cnt = TF_EM_MIN_ENTRIES;
+ while (num_entries > cnt &&
+ cnt <= TF_EM_MAX_ENTRIES)
+ cnt *= 2;
+
+ if (cnt > TF_EM_MAX_ENTRIES) {
+ TFP_DRV_LOG(ERR, "EEM: Invalid number of Tx requested: "
+ "%u\n",
+ (parms->tx_num_flows_in_k * TF_KILOBYTE));
+ return -EINVAL;
+ }
+
+ parms->rx_num_flows_in_k = cnt / TF_KILOBYTE;
+ } else {
+ if ((parms->rx_num_flows_in_k * TF_KILOBYTE) <
+ TF_EM_MIN_ENTRIES ||
+ (parms->rx_num_flows_in_k * TF_KILOBYTE) >
+ tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported) {
+ TFP_DRV_LOG(ERR,
+ "EEM: Invalid number of Rx flows "
+ "requested:%u max:%u\n",
+ parms->rx_num_flows_in_k * TF_KILOBYTE,
+ tbl_scope_cb->em_caps[TF_DIR_RX].max_entries_supported);
+ return -EINVAL;
+ }
+
+ /* must be a power-of-2 supported value
+ * in the range 32K - 128M
+ */
+ cnt = TF_EM_MIN_ENTRIES;
+ while ((parms->rx_num_flows_in_k * TF_KILOBYTE) != cnt &&
+ cnt <= TF_EM_MAX_ENTRIES)
+ cnt *= 2;
+
+ if (cnt > TF_EM_MAX_ENTRIES) {
+ TFP_DRV_LOG(ERR,
+ "EEM: Invalid number of Rx requested: %u\n",
+ (parms->rx_num_flows_in_k * TF_KILOBYTE));
+ return -EINVAL;
+ }
+ }
+
+ if (parms->tx_mem_size_in_mb != 0) {
+ uint32_t key_b = 2 * (parms->tx_max_key_sz_in_bits / 8 + 1);
+ uint32_t action_b = ((parms->tx_max_action_entry_sz_in_bits / 8)
+ + 1);
+ uint32_t num_entries = (parms->tx_mem_size_in_mb *
+ (TF_KILOBYTE * TF_KILOBYTE)) /
+ (key_b + action_b);
+
+ if (num_entries < TF_EM_MIN_ENTRIES) {
+ TFP_DRV_LOG(ERR,
+ "EEM: Insufficient memory requested:%uMB\n",
+ parms->rx_mem_size_in_mb);
+ return -EINVAL;
+ }
+
+ cnt = TF_EM_MIN_ENTRIES;
+ while (num_entries > cnt &&
+ cnt <= TF_EM_MAX_ENTRIES)
+ cnt *= 2;
+
+ if (cnt > TF_EM_MAX_ENTRIES) {
+ TFP_DRV_LOG(ERR,
+ "EEM: Invalid number of Tx requested: %u\n",
+ (parms->tx_num_flows_in_k * TF_KILOBYTE));
+ return -EINVAL;
+ }
+
+ parms->tx_num_flows_in_k = cnt / TF_KILOBYTE;
+ } else {
+ if ((parms->tx_num_flows_in_k * TF_KILOBYTE) <
+ TF_EM_MIN_ENTRIES ||
+ (parms->tx_num_flows_in_k * TF_KILOBYTE) >
+ tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported) {
+ TFP_DRV_LOG(ERR,
+ "EEM: Invalid number of Tx flows "
+ "requested:%u max:%u\n",
+ (parms->tx_num_flows_in_k * TF_KILOBYTE),
+ tbl_scope_cb->em_caps[TF_DIR_TX].max_entries_supported);
+ return -EINVAL;
+ }
+
+ cnt = TF_EM_MIN_ENTRIES;
+ while ((parms->tx_num_flows_in_k * TF_KILOBYTE) != cnt &&
+ cnt <= TF_EM_MAX_ENTRIES)
+ cnt *= 2;
+
+ if (cnt > TF_EM_MAX_ENTRIES) {
+ TFP_DRV_LOG(ERR,
+ "EEM: Invalid number of Tx requested: %u\n",
+ (parms->tx_num_flows_in_k * TF_KILOBYTE));
+ return -EINVAL;
+ }
+ }
+
+ if (parms->rx_num_flows_in_k != 0 &&
+ parms->rx_max_key_sz_in_bits / 8 == 0) {
+ TFP_DRV_LOG(ERR,
+ "EEM: Rx key size required: %u\n",
+ (parms->rx_max_key_sz_in_bits));
+ return -EINVAL;
+ }
+
+ if (parms->tx_num_flows_in_k != 0 &&
+ parms->tx_max_key_sz_in_bits / 8 == 0) {
+ TFP_DRV_LOG(ERR,
+ "EEM: Tx key size required: %u\n",
+ (parms->tx_max_key_sz_in_bits));
+ return -EINVAL;
+ }
+ /* Rx */
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries =
+ parms->rx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].entry_size =
+ parms->rx_max_key_sz_in_bits / 8;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].num_entries =
+ parms->rx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY1_TABLE].entry_size =
+ parms->rx_max_key_sz_in_bits / 8;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].num_entries =
+ parms->rx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_RECORD_TABLE].entry_size =
+ parms->rx_max_action_entry_sz_in_bits / 8;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EFC_TABLE].num_entries =
+ 0;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_ACTION_TABLE].num_entries =
+ parms->rx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_ACTION_TABLE].entry_size =
+ parms->rx_max_action_entry_sz_in_bits / 8;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EM_LKUP_TABLE].num_entries =
+ parms->rx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_EM_LKUP_TABLE].entry_size =
+ parms->rx_max_key_sz_in_bits / 8;
+
+ /* Tx */
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].num_entries =
+ parms->tx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY0_TABLE].entry_size =
+ parms->tx_max_key_sz_in_bits / 8;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].num_entries =
+ parms->tx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_KEY1_TABLE].entry_size =
+ parms->tx_max_key_sz_in_bits / 8;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].num_entries =
+ parms->tx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_RECORD_TABLE].entry_size =
+ parms->tx_max_action_entry_sz_in_bits / 8;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EFC_TABLE].num_entries =
+ 0;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_ACTION_TABLE].num_entries =
+ parms->rx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_ACTION_TABLE].entry_size =
+ parms->tx_max_action_entry_sz_in_bits / 8;
+
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EM_LKUP_TABLE].num_entries =
+ parms->rx_num_flows_in_k * TF_KILOBYTE;
+ tbl_scope_cb->em_ctx_info[TF_DIR_TX].em_tables[TF_EM_LKUP_TABLE].entry_size =
+ parms->tx_max_key_sz_in_bits / 8;
+
+ return 0;
+}
+
+/** insert EEM entry API
+ *
+ * returns:
+ * 0
+ * TF_ERR - unable to get lock
+ *
+ * insert callback returns:
+ * 0
+ * TF_ERR_EM_DUP - key is already in table
+ */
+static int
+tf_insert_eem_entry(struct tf_dev_info *dev,
+ struct tf_tbl_scope_cb *tbl_scope_cb,
+ struct tf_insert_em_entry_parms *parms)
+{
+ uint32_t mask;
+ uint32_t key0_hash;
+ uint32_t key1_hash;
+ uint32_t key0_index;
+ uint32_t key1_index;
+ struct cfa_p4_eem_64b_entry key_entry;
+ uint32_t index;
+ enum hcapi_cfa_em_table_type table_type;
+ uint32_t gfid;
+ struct hcapi_cfa_hwop op;
+ struct hcapi_cfa_key_tbl key_tbl;
+ struct hcapi_cfa_key_data key_obj;
+ struct hcapi_cfa_key_loc key_loc;
+ uint64_t big_hash;
+ int rc;
+
+ /* Get mask to use on hash */
+ mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
+
+ if (!mask)
+ return -EINVAL;
+
+ if (dev->ops->tf_dev_cfa_key_hash == NULL)
+ return -EINVAL;
+
+ big_hash = dev->ops->tf_dev_cfa_key_hash((uint64_t *)parms->key,
+ (TF_P4_HW_EM_KEY_MAX_SIZE + 4) * 8);
+ key0_hash = (uint32_t)(big_hash >> 32);
+ key1_hash = (uint32_t)(big_hash & 0xFFFFFFFF);