1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019-2020 Broadcom
7 #include <rte_common.h>
16 #include "tf_ext_flow_handle.h"
20 /* Enable EEM table dump
24 static struct tf_eem_64b_entry zero_key_entry;
26 static uint32_t tf_em_get_key_mask(int num_entries)
28 uint32_t mask = num_entries - 1;
30 if (num_entries & 0x7FFF)
33 if (num_entries > (128 * 1024 * 1024))
39 /* CRC32i support for Key0 hash */
40 #define ucrc32(ch, crc) (crc32tbl[((crc) ^ (ch)) & 0xff] ^ ((crc) >> 8))
41 #define crc32(x, y) crc32i(~0, x, y)
43 static const uint32_t crc32tbl[] = { /* CRC polynomial 0xedb88320 */
44 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
45 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
46 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
47 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
48 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
49 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
50 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
51 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
52 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
53 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
54 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
55 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
56 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
57 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
58 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
59 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
60 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
61 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
62 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
63 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
64 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
65 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
66 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
67 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
68 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
69 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
70 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
71 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
72 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
73 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
74 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
75 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
76 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
77 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
78 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
79 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
80 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
81 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
82 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
83 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
84 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
85 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
86 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
87 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
88 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
89 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
90 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
91 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
92 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
93 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
94 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
95 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
96 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
97 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
98 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
99 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
100 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
101 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
102 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
103 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
104 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
105 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
106 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
107 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
110 static uint32_t crc32i(uint32_t crc, const uint8_t *buf, size_t len)
114 for (l = (len - 1); l >= 0; l--)
115 crc = ucrc32(buf[l], crc);
120 static uint32_t tf_em_lkup_get_crc32_hash(struct tf_session *session,
130 /* Do byte-wise XOR of the 52-byte HASH key first. */
134 for (i = TF_HW_EM_KEY_MAX_SIZE - 2; i >= 0; i--) {
135 index = index ^ *kptr;
140 val1 = session->lkup_em_seed_mem[dir][index * 2];
141 val2 = session->lkup_em_seed_mem[dir][index * 2 + 1];
143 temp[3] = (uint8_t)(val1 >> 24);
144 temp[2] = (uint8_t)(val1 >> 16);
145 temp[1] = (uint8_t)(val1 >> 8);
146 temp[0] = (uint8_t)(val1 & 0xff);
149 /* Start with seed */
151 val1 = crc32i(~val1, temp, 4);
154 (key - (TF_HW_EM_KEY_MAX_SIZE - 1)),
155 TF_HW_EM_KEY_MAX_SIZE);
159 val1 = crc32i(~val1, temp, 4);
164 static uint32_t tf_em_lkup_get_lookup3_hash(uint32_t lookup3_init_value,
169 val1 = hashword(((uint32_t *)in_key) + 1,
170 TF_HW_EM_KEY_MAX_SIZE / (sizeof(uint32_t)),
176 void *tf_em_get_table_page(struct tf_tbl_scope_cb *tbl_scope_cb,
179 enum tf_em_table_type table_type)
182 int page = offset / TF_EM_PAGE_SIZE;
184 struct tf_em_ctx_mem_info *ctx = &tbl_scope_cb->em_ctx_info[dir];
189 if (dir != TF_DIR_RX && dir != TF_DIR_TX)
192 if (table_type < KEY0_TABLE || table_type > EFC_TABLE)
196 * Use the level according to the num_level of page table
198 level = ctx->em_tables[table_type].num_lvl - 1;
200 addr = (void *)ctx->em_tables[table_type].pg_tbl[level].pg_va_tbl[page];
205 /** Read Key table entry
207 * Entry is read in to entry
209 static int tf_em_read_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
210 struct tf_eem_64b_entry *entry,
213 enum tf_em_table_type table_type,
217 uint32_t entry_offset = (index * entry_size) % TF_EM_PAGE_SIZE;
219 page = tf_em_get_table_page(tbl_scope_cb,
221 (index * entry_size),
227 memcpy((uint8_t *)entry, (uint8_t *)page + entry_offset, entry_size);
231 static int tf_em_write_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
232 struct tf_eem_64b_entry *entry,
235 enum tf_em_table_type table_type,
239 uint32_t entry_offset = (index * entry_size) % TF_EM_PAGE_SIZE;
241 page = tf_em_get_table_page(tbl_scope_cb,
243 (index * entry_size),
249 memcpy((uint8_t *)page + entry_offset, entry, entry_size);
254 static int tf_em_entry_exists(struct tf_tbl_scope_cb *tbl_scope_cb,
255 struct tf_eem_64b_entry *entry,
257 enum tf_em_table_type table_type,
261 struct tf_eem_64b_entry table_entry;
263 rc = tf_em_read_entry(tbl_scope_cb,
265 TF_EM_KEY_RECORD_SIZE,
273 if (table_entry.hdr.word1 & (1 << TF_LKUP_RECORD_VALID_SHIFT)) {
275 if (memcmp(&table_entry,
277 TF_EM_KEY_RECORD_SIZE) == 0)
289 static void tf_em_create_key_entry(struct tf_eem_entry_hdr *result,
291 struct tf_eem_64b_entry *key_entry)
293 key_entry->hdr.word1 = result->word1;
295 if (result->word1 & TF_LKUP_RECORD_ACT_REC_INT_MASK)
296 key_entry->hdr.pointer = result->pointer;
298 key_entry->hdr.pointer = result->pointer;
300 memcpy(key_entry->key, in_key, TF_HW_EM_KEY_MAX_SIZE + 4);
303 /* tf_em_select_inject_table
306 * 0 - Key does not exist in either table and can be inserted
307 * at "index" in table "table".
308 * EEXIST - Key does exist in table at "index" in table "table".
309 * TF_ERR - Something went horribly wrong.
311 static int tf_em_select_inject_table(struct tf_tbl_scope_cb *tbl_scope_cb,
313 struct tf_eem_64b_entry *entry,
317 enum tf_em_table_type *table)
325 key0_entry = tf_em_entry_exists(tbl_scope_cb,
334 key1_entry = tf_em_entry_exists(tbl_scope_cb,
340 if (key0_entry == -EEXIST) {
344 } else if (key1_entry == -EEXIST) {
348 } else if (key0_entry == 0) {
352 } else if (key1_entry == 0) {
361 /** insert EEM entry API
365 * TF_ERR - unable to get lock
367 * insert callback returns:
369 * TF_ERR_EM_DUP - key is already in table
371 int tf_insert_eem_entry(struct tf_session *session,
372 struct tf_tbl_scope_cb *tbl_scope_cb,
373 struct tf_insert_em_entry_parms *parms)
380 struct tf_eem_64b_entry key_entry;
382 enum tf_em_table_type table_type;
386 /* Get mask to use on hash */
387 mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[KEY0_TABLE].num_entries);
392 num_of_entry = TF_HW_EM_KEY_MAX_SIZE + 4;
394 key0_hash = tf_em_lkup_get_crc32_hash(session,
395 &parms->key[num_of_entry] - 1,
397 key0_index = key0_hash & mask;
400 tf_em_lkup_get_lookup3_hash(session->lkup_lkup3_init_cfg[parms->dir],
402 key1_index = key1_hash & mask;
405 * Use the "result" arg to populate all of the key entry then
406 * store the byte swapped "raw" entry in a local copy ready
407 * for insertion in to the table.
409 tf_em_create_key_entry((struct tf_eem_entry_hdr *)parms->em_record,
410 ((uint8_t *)parms->key),
414 * Find which table to use
416 if (tf_em_select_inject_table(tbl_scope_cb,
423 if (table_type == KEY0_TABLE) {
436 if (tf_em_write_entry(tbl_scope_cb,
438 TF_EM_KEY_RECORD_SIZE,
442 TF_SET_FLOW_ID(parms->flow_id,
444 TF_GFID_TABLE_EXTERNAL,
446 TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
461 * Insert EM internal entry API
466 int tf_insert_em_internal_entry(struct tf *tfp,
467 struct tf_insert_em_entry_parms *parms)
471 uint16_t rptr_index = 0;
472 uint8_t rptr_entry = 0;
473 uint8_t num_of_entries = 0;
474 struct tf_session *session =
475 (struct tf_session *)(tfp->session->core_data);
476 struct stack *pool = &session->em_pool[parms->dir];
479 rc = stack_pop(pool, &index);
483 "dir:%d, EM entry index allocation failed\n",
488 rptr_index = index * TF_SESSION_EM_ENTRY_SIZE;
489 rc = tf_msg_insert_em_internal_entry(tfp,
498 "Internal entry @ Index:%d rptr_index:0x%x rptr_entry:0x%x num_of_entries:%d\n",
499 index * TF_SESSION_EM_ENTRY_SIZE,
505 ((rptr_index << TF_EM_INTERNAL_INDEX_SHIFT) |
507 0); /* N/A for internal table */
509 TF_SET_FLOW_ID(parms->flow_id,
511 TF_GFID_TABLE_INTERNAL,
514 TF_SET_FIELDS_IN_FLOW_HANDLE(parms->flow_handle,
524 /** Delete EM internal entry API
530 int tf_delete_em_internal_entry(struct tf *tfp,
531 struct tf_delete_em_entry_parms *parms)
534 struct tf_session *session =
535 (struct tf_session *)(tfp->session->core_data);
536 struct stack *pool = &session->em_pool[parms->dir];
538 rc = tf_msg_delete_em_entry(tfp, parms);
540 /* Return resource to pool */
542 stack_push(pool, parms->index / TF_SESSION_EM_ENTRY_SIZE);
548 /** delete EEM hash entry API
552 * -EINVAL - parameter error
553 * TF_NO_SESSION - bad session ID
554 * TF_ERR_TBL_SCOPE - invalid table scope
555 * TF_ERR_TBL_IF - invalid table interface
557 * insert callback returns
559 * TF_NO_EM_MATCH - entry not found
561 int tf_delete_eem_entry(struct tf *tfp,
562 struct tf_delete_em_entry_parms *parms)
564 struct tf_session *session;
565 struct tf_tbl_scope_cb *tbl_scope_cb;
566 enum tf_em_table_type hash_type;
572 session = (struct tf_session *)tfp->session->core_data;
576 tbl_scope_cb = tbl_scope_cb_find(session,
577 parms->tbl_scope_id);
578 if (tbl_scope_cb == NULL)
581 if (parms->flow_handle == 0)
584 TF_GET_HASH_TYPE_FROM_FLOW_HANDLE(parms->flow_handle, hash_type);
585 TF_GET_INDEX_FROM_FLOW_HANDLE(parms->flow_handle, index);
587 if (tf_em_entry_exists(tbl_scope_cb,
591 parms->dir) == -EEXIST) {
592 tf_em_write_entry(tbl_scope_cb,
594 TF_EM_KEY_RECORD_SIZE,