1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2020 Intel Corporation
8 #include <rte_cpuflags.h>
9 #include <rte_common.h>
10 #include <rte_net_crc.h>
16 /** CRC polynomials */
17 #define CRC32_ETH_POLYNOMIAL 0x04c11db7UL
18 #define CRC16_CCITT_POLYNOMIAL 0x1021U
20 #define CRC_LUT_SIZE 256
23 static uint32_t crc32_eth_lut[CRC_LUT_SIZE];
24 static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE];
27 rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len);
30 rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len);
33 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len);
36 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len);
39 (*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len);
41 static rte_net_crc_handler handlers_default[] = {
42 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_default_handler,
43 [RTE_NET_CRC32_ETH] = rte_crc32_eth_default_handler,
46 static const rte_net_crc_handler *handlers = handlers_default;
48 static const rte_net_crc_handler handlers_scalar[] = {
49 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler,
50 [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler,
52 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
53 static const rte_net_crc_handler handlers_avx512[] = {
54 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler,
55 [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler,
58 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
59 static const rte_net_crc_handler handlers_sse42[] = {
60 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler,
61 [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler,
64 #ifdef CC_ARM64_NEON_PMULL_SUPPORT
65 static const rte_net_crc_handler handlers_neon[] = {
66 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler,
67 [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler,
71 static uint16_t max_simd_bitwidth;
73 #define NET_LOG(level, fmt, args...) \
74 rte_log(RTE_LOG_ ## level, libnet_logtype, "%s(): " fmt "\n", \
77 RTE_LOG_REGISTER_DEFAULT(libnet_logtype, INFO);
82 * Reflect the bits about the middle
85 * value to be reflected
91 reflect_32bits(uint32_t val)
95 for (i = 0; i < 32; i++)
96 if ((val & (1U << i)) != 0)
97 res |= (uint32_t)(1U << (31 - i));
103 crc32_eth_init_lut(uint32_t poly,
108 for (i = 0; i < CRC_LUT_SIZE; i++) {
109 uint32_t crc = reflect_32bits(i);
111 for (j = 0; j < 8; j++) {
112 if (crc & 0x80000000L)
113 crc = (crc << 1) ^ poly;
117 lut[i] = reflect_32bits(crc);
121 static __rte_always_inline uint32_t
122 crc32_eth_calc_lut(const uint8_t *data,
128 crc = lut[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
134 rte_net_crc_scalar_init(void)
136 /* 32-bit crc init */
137 crc32_eth_init_lut(CRC32_ETH_POLYNOMIAL, crc32_eth_lut);
139 /* 16-bit CRC init */
140 crc32_eth_init_lut(CRC16_CCITT_POLYNOMIAL << 16, crc16_ccitt_lut);
143 static inline uint32_t
144 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len)
146 /* return 16-bit CRC value */
147 return (uint16_t)~crc32_eth_calc_lut(data,
153 static inline uint32_t
154 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len)
156 /* return 32-bit CRC value */
157 return ~crc32_eth_calc_lut(data,
163 /* AVX512/VPCLMULQDQ handling */
165 #define AVX512_VPCLMULQDQ_CPU_SUPPORTED ( \
166 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && \
167 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && \
168 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && \
169 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && \
170 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && \
171 rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ) \
174 static const rte_net_crc_handler *
175 avx512_vpclmulqdq_get_handlers(void)
177 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
178 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED &&
179 max_simd_bitwidth >= RTE_VECT_SIMD_512)
180 return handlers_avx512;
182 NET_LOG(INFO, "Requirements not met, can't use AVX512\n");
187 avx512_vpclmulqdq_init(void)
189 #ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
190 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED)
191 rte_net_crc_avx512_init();
195 /* SSE4.2/PCLMULQDQ handling */
197 #define SSE42_PCLMULQDQ_CPU_SUPPORTED \
198 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ)
200 static const rte_net_crc_handler *
201 sse42_pclmulqdq_get_handlers(void)
203 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
204 if (SSE42_PCLMULQDQ_CPU_SUPPORTED &&
205 max_simd_bitwidth >= RTE_VECT_SIMD_128)
206 return handlers_sse42;
208 NET_LOG(INFO, "Requirements not met, can't use SSE\n");
213 sse42_pclmulqdq_init(void)
215 #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
216 if (SSE42_PCLMULQDQ_CPU_SUPPORTED)
217 rte_net_crc_sse42_init();
221 /* NEON/PMULL handling */
223 #define NEON_PMULL_CPU_SUPPORTED \
224 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)
226 static const rte_net_crc_handler *
227 neon_pmull_get_handlers(void)
229 #ifdef CC_ARM64_NEON_PMULL_SUPPORT
230 if (NEON_PMULL_CPU_SUPPORTED &&
231 max_simd_bitwidth >= RTE_VECT_SIMD_128)
232 return handlers_neon;
234 NET_LOG(INFO, "Requirements not met, can't use NEON\n");
239 neon_pmull_init(void)
241 #ifdef CC_ARM64_NEON_PMULL_SUPPORT
242 if (NEON_PMULL_CPU_SUPPORTED)
243 rte_net_crc_neon_init();
247 /* Default handling */
250 rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len)
253 if (max_simd_bitwidth == 0)
254 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
256 handlers = avx512_vpclmulqdq_get_handlers();
257 if (handlers != NULL)
258 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
259 handlers = sse42_pclmulqdq_get_handlers();
260 if (handlers != NULL)
261 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
262 handlers = neon_pmull_get_handlers();
263 if (handlers != NULL)
264 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
265 handlers = handlers_scalar;
266 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
270 rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len)
273 if (max_simd_bitwidth == 0)
274 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
276 handlers = avx512_vpclmulqdq_get_handlers();
277 if (handlers != NULL)
278 return handlers[RTE_NET_CRC32_ETH](data, data_len);
279 handlers = sse42_pclmulqdq_get_handlers();
280 if (handlers != NULL)
281 return handlers[RTE_NET_CRC32_ETH](data, data_len);
282 handlers = neon_pmull_get_handlers();
283 if (handlers != NULL)
284 return handlers[RTE_NET_CRC32_ETH](data, data_len);
285 handlers = handlers_scalar;
286 return handlers[RTE_NET_CRC32_ETH](data, data_len);
292 rte_net_crc_set_alg(enum rte_net_crc_alg alg)
295 if (max_simd_bitwidth == 0)
296 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
299 case RTE_NET_CRC_AVX512:
300 handlers = avx512_vpclmulqdq_get_handlers();
301 if (handlers != NULL)
304 case RTE_NET_CRC_SSE42:
305 handlers = sse42_pclmulqdq_get_handlers();
306 break; /* for x86, always break here */
307 case RTE_NET_CRC_NEON:
308 handlers = neon_pmull_get_handlers();
310 case RTE_NET_CRC_SCALAR:
316 if (handlers == NULL)
317 handlers = handlers_scalar;
321 rte_net_crc_calc(const void *data,
323 enum rte_net_crc_type type)
326 rte_net_crc_handler f_handle;
328 f_handle = handlers[type];
329 ret = f_handle(data, data_len);
334 /* Call initialisation helpers for all crc algorithm handlers */
335 RTE_INIT(rte_net_crc_init)
337 rte_net_crc_scalar_init();
338 sse42_pclmulqdq_init();
339 avx512_vpclmulqdq_init();