- pkts_mask_match_many, &pkts_mask_out_slow, entries, 0);
- pkts_mask_out |= pkts_mask_out_slow;
- }
-
- *lookup_hit_mask = pkts_mask_out;
- RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out));
- return status;
-}
-
-static int rte_table_hash_lru_lookup_dosig(
- void *table,
- struct rte_mbuf **pkts,
- uint64_t pkts_mask,
- uint64_t *lookup_hit_mask,
- void **entries)
-{
- struct rte_table_hash *t = (struct rte_table_hash *) table;
- struct grinder *g = t->grinders;
- uint64_t pkt00_index, pkt01_index, pkt10_index, pkt11_index;
- uint64_t pkt20_index, pkt21_index, pkt30_index, pkt31_index;
- uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0;
- int status = 0;
-
- __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
- RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(t, n_pkts_in);
-
- /* Cannot run the pipeline with less than 7 packets */
- if (__builtin_popcountll(pkts_mask) < 7)
- return rte_table_hash_lru_lookup_unoptimized(table, pkts,
- pkts_mask, lookup_hit_mask, entries, 1);
-
- /* Pipeline stage 0 */
- lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
-
- /* Pipeline feed */
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 0 */
- lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index);
-
- /* Pipeline feed */
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 0 */
- lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index);
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index);
-
- /* Pipeline stage 2 */
- lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
-
- /*
- * Pipeline run
- *
- */
- for ( ; pkts_mask; ) {
- /* Pipeline feed */
- pkt30_index = pkt20_index;
- pkt31_index = pkt21_index;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 0 */
- lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask,
- pkt00_index, pkt01_index);
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index);
-
- /* Pipeline stage 2 */
- lookup2_stage2(t, g, pkt20_index, pkt21_index,
- pkts_mask_match_many);
-
- /* Pipeline stage 3 */
- lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index,
- pkts_mask_out, entries);
- }
-
- /* Pipeline feed */
- pkt30_index = pkt20_index;
- pkt31_index = pkt21_index;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
- pkt10_index = pkt00_index;
- pkt11_index = pkt01_index;
-
- /* Pipeline stage 1 */
- lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index);
-
- /* Pipeline stage 2 */
- lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
-
- /* Pipeline stage 3 */
- lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
- entries);
-
- /* Pipeline feed */
- pkt30_index = pkt20_index;
- pkt31_index = pkt21_index;
- pkt20_index = pkt10_index;
- pkt21_index = pkt11_index;
-
- /* Pipeline stage 2 */
- lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many);
-
- /* Pipeline stage 3 */
- lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
- entries);
-
- /* Pipeline feed */
- pkt30_index = pkt20_index;
- pkt31_index = pkt21_index;
-
- /* Pipeline stage 3 */
- lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out,
- entries);
-
- /* Slow path */
- pkts_mask_match_many &= ~pkts_mask_out;
- if (pkts_mask_match_many) {
- uint64_t pkts_mask_out_slow = 0;
-
- status = rte_table_hash_lru_lookup_unoptimized(table, pkts,
- pkts_mask_match_many, &pkts_mask_out_slow, entries, 1);