1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
5 #include "acl_run_sse.h"
7 /*sizeof(uint32_t) << match_log == sizeof(struct rte_acl_match_results)*/
8 static const uint32_t match_log = 5;
10 struct acl_flow_avx512 {
11 uint32_t num_packets; /* number of packets processed */
12 uint32_t total_packets; /* max number of packets to process */
13 uint32_t root_index; /* current root index */
14 const uint64_t *trans; /* transition table */
15 const uint32_t *data_index; /* input data indexes */
16 const uint8_t **idata; /* input data */
17 uint32_t *matches; /* match indexes */
21 acl_set_flow_avx512(struct acl_flow_avx512 *flow, const struct rte_acl_ctx *ctx,
22 uint32_t trie, const uint8_t *data[], uint32_t *matches,
23 uint32_t total_packets)
25 flow->num_packets = 0;
26 flow->total_packets = total_packets;
27 flow->root_index = ctx->trie[trie].root_index;
28 flow->trans = ctx->trans_table;
29 flow->data_index = ctx->trie[trie].data_index;
31 flow->matches = matches;
35 * Update flow and result masks based on the number of unprocessed flows.
37 static inline uint32_t
38 update_flow_mask(const struct acl_flow_avx512 *flow, uint32_t *fmsk,
41 uint32_t i, j, k, m, n;
46 k = __builtin_popcount(m);
47 n = flow->total_packets - flow->num_packets;
51 for (i = k - n; i != 0; i--) {
52 j = sizeof(m) * CHAR_BIT - 1 - __builtin_clz(m);
65 * Resolve matches for multiple categories (LE 8, use 128b instuctions/regs)
68 resolve_mcle8_avx512x1(uint32_t result[],
69 const struct rte_acl_match_results pr[], const uint32_t match[],
70 uint32_t nb_pkt, uint32_t nb_cat, uint32_t nb_trie)
73 const uint32_t *pm, *res;
74 uint32_t i, j, k, mi, mn;
81 for (k = 0; k != nb_pkt; k++, result += nb_cat) {
83 mi = match[k] << match_log;
85 for (j = 0; j != nb_cat; j += RTE_ACL_RESULTS_MULTIPLIER) {
87 cr = _mm_loadu_si128((const xmm_t *)(res + mi + j));
88 cp = _mm_loadu_si128((const xmm_t *)(pri + mi + j));
90 for (i = 1, pm = match + nb_pkt; i != nb_trie;
93 mn = j + (pm[k] << match_log);
95 nr = _mm_loadu_si128((const xmm_t *)(res + mn));
96 np = _mm_loadu_si128((const xmm_t *)(pri + mn));
98 msk = _mm_cmpgt_epi32_mask(cp, np);
99 cr = _mm_mask_mov_epi32(nr, msk, cr);
100 cp = _mm_mask_mov_epi32(np, msk, cp);
103 _mm_storeu_si128((xmm_t *)(result + j), cr);
108 #include "acl_run_avx512x8.h"
111 rte_acl_classify_avx512x16(const struct rte_acl_ctx *ctx, const uint8_t **data,
112 uint32_t *results, uint32_t num, uint32_t categories)
114 const uint32_t max_iter = MAX_SEARCHES_AVX16 * MAX_SEARCHES_AVX16;
116 /* split huge lookup (gt 256) into series of fixed size ones */
117 while (num > max_iter) {
118 search_avx512x8x2(ctx, data, results, max_iter, categories);
120 results += max_iter * categories;
124 /* select classify method based on number of remaining requests */
125 if (num >= MAX_SEARCHES_AVX16)
126 return search_avx512x8x2(ctx, data, results, num, categories);
127 if (num >= MAX_SEARCHES_SSE8)
128 return search_sse_8(ctx, data, results, num, categories);
129 if (num >= MAX_SEARCHES_SSE4)
130 return search_sse_4(ctx, data, results, num, categories);
132 return rte_acl_classify_scalar(ctx, data, results, num, categories);
135 #include "acl_run_avx512x16.h"
138 rte_acl_classify_avx512x32(const struct rte_acl_ctx *ctx, const uint8_t **data,
139 uint32_t *results, uint32_t num, uint32_t categories)
141 const uint32_t max_iter = MAX_SEARCHES_AVX16 * MAX_SEARCHES_AVX16;
143 /* split huge lookup (gt 256) into series of fixed size ones */
144 while (num > max_iter) {
145 search_avx512x16x2(ctx, data, results, max_iter, categories);
147 results += max_iter * categories;
151 /* select classify method based on number of remaining requests */
152 if (num >= 2 * MAX_SEARCHES_AVX16)
153 return search_avx512x16x2(ctx, data, results, num, categories);
154 if (num >= MAX_SEARCHES_AVX16)
155 return search_avx512x8x2(ctx, data, results, num, categories);
156 if (num >= MAX_SEARCHES_SSE8)
157 return search_sse_8(ctx, data, results, num, categories);
158 if (num >= MAX_SEARCHES_SSE4)
159 return search_sse_4(ctx, data, results, num, categories);
161 return rte_acl_classify_scalar(ctx, data, results, num, categories);