X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_net%2Frte_net_crc.c;h=1fe58f7c7565b4509d4193e2376f33221a15920b;hb=5b38d8cd4663;hp=73ac3a9591fb27d696193d47317459636840bc19;hpb=369991d997e4abdee355e19ffbb41a4d246cafa2;p=dpdk.git diff --git a/lib/librte_net/rte_net_crc.c b/lib/librte_net/rte_net_crc.c index 73ac3a9591..1fe58f7c75 100644 --- a/lib/librte_net/rte_net_crc.c +++ b/lib/librte_net/rte_net_crc.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2017 Intel Corporation + * Copyright(c) 2017-2020 Intel Corporation */ #include @@ -9,23 +9,28 @@ #include #include #include +#include +#include +#include -#if defined(RTE_ARCH_X86_64) && defined(RTE_MACHINE_CPUFLAG_PCLMULQDQ) -#define X86_64_SSE42_PCLMULQDQ 1 -#elif defined(RTE_ARCH_ARM64) && defined(RTE_MACHINE_CPUFLAG_PMULL) -#define ARM64_NEON_PMULL 1 -#endif +#include "net_crc.h" -#ifdef X86_64_SSE42_PCLMULQDQ -#include -#elif defined ARM64_NEON_PMULL -#include -#endif +/** CRC polynomials */ +#define CRC32_ETH_POLYNOMIAL 0x04c11db7UL +#define CRC16_CCITT_POLYNOMIAL 0x1021U + +#define CRC_LUT_SIZE 256 /* crc tables */ static uint32_t crc32_eth_lut[CRC_LUT_SIZE]; static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE]; +static uint32_t +rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len); + +static uint32_t +rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len); + static uint32_t rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len); @@ -35,25 +40,46 @@ rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len); typedef uint32_t (*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len); -static rte_net_crc_handler *handlers; +static rte_net_crc_handler handlers_default[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_default_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_default_handler, +}; -static rte_net_crc_handler handlers_scalar[] = { +static const rte_net_crc_handler *handlers = handlers_default; + +static const rte_net_crc_handler handlers_scalar[] = { [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler, [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler, }; - -#ifdef X86_64_SSE42_PCLMULQDQ -static rte_net_crc_handler handlers_sse42[] = { +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT +static const rte_net_crc_handler handlers_avx512[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler, +}; +#endif +#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT +static const rte_net_crc_handler handlers_sse42[] = { [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler, [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler, }; -#elif defined ARM64_NEON_PMULL -static rte_net_crc_handler handlers_neon[] = { +#endif +#ifdef CC_ARM64_NEON_PMULL_SUPPORT +static const rte_net_crc_handler handlers_neon[] = { [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler, [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler, }; #endif +static uint16_t max_simd_bitwidth; + +#define NET_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, libnet_logtype, "%s(): " fmt "\n", \ + __func__, ## args) + +RTE_LOG_REGISTER(libnet_logtype, lib.net, INFO); + +/* Scalar handling */ + /** * Reflect the bits about the middle * @@ -69,8 +95,8 @@ reflect_32bits(uint32_t val) uint32_t i, res = 0; for (i = 0; i < 32; i++) - if ((val & (1 << i)) != 0) - res |= (uint32_t)(1 << (31 - i)); + if ((val & (1U << i)) != 0) + res |= (uint32_t)(1U << (31 - i)); return res; } @@ -136,29 +162,161 @@ rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len) crc32_eth_lut); } +/* AVX512/VPCLMULQDQ handling */ + +#define AVX512_VPCLMULQDQ_CPU_SUPPORTED ( \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ) \ +) + +static const rte_net_crc_handler * +avx512_vpclmulqdq_get_handlers(void) +{ +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT + if (AVX512_VPCLMULQDQ_CPU_SUPPORTED && + max_simd_bitwidth >= RTE_VECT_SIMD_512) + return handlers_avx512; +#endif + NET_LOG(INFO, "Requirements not met, can't use AVX512\n"); + return NULL; +} + +static void +avx512_vpclmulqdq_init(void) +{ +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT + if (AVX512_VPCLMULQDQ_CPU_SUPPORTED) + rte_net_crc_avx512_init(); +#endif +} + +/* SSE4.2/PCLMULQDQ handling */ + +#define SSE42_PCLMULQDQ_CPU_SUPPORTED \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) + +static const rte_net_crc_handler * +sse42_pclmulqdq_get_handlers(void) +{ +#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT + if (SSE42_PCLMULQDQ_CPU_SUPPORTED && + max_simd_bitwidth >= RTE_VECT_SIMD_128) + return handlers_sse42; +#endif + NET_LOG(INFO, "Requirements not met, can't use SSE\n"); + return NULL; +} + +static void +sse42_pclmulqdq_init(void) +{ +#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT + if (SSE42_PCLMULQDQ_CPU_SUPPORTED) + rte_net_crc_sse42_init(); +#endif +} + +/* NEON/PMULL handling */ + +#define NEON_PMULL_CPU_SUPPORTED \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL) + +static const rte_net_crc_handler * +neon_pmull_get_handlers(void) +{ +#ifdef CC_ARM64_NEON_PMULL_SUPPORT + if (NEON_PMULL_CPU_SUPPORTED && + max_simd_bitwidth >= RTE_VECT_SIMD_128) + return handlers_neon; +#endif + NET_LOG(INFO, "Requirements not met, can't use NEON\n"); + return NULL; +} + +static void +neon_pmull_init(void) +{ +#ifdef CC_ARM64_NEON_PMULL_SUPPORT + if (NEON_PMULL_CPU_SUPPORTED) + rte_net_crc_neon_init(); +#endif +} + +/* Default handling */ + +static uint32_t +rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len) +{ + handlers = NULL; + if (max_simd_bitwidth == 0) + max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); + + handlers = avx512_vpclmulqdq_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC16_CCITT](data, data_len); + handlers = sse42_pclmulqdq_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC16_CCITT](data, data_len); + handlers = neon_pmull_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC16_CCITT](data, data_len); + handlers = handlers_scalar; + return handlers[RTE_NET_CRC16_CCITT](data, data_len); +} + +static uint32_t +rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len) +{ + handlers = NULL; + if (max_simd_bitwidth == 0) + max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); + + handlers = avx512_vpclmulqdq_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC32_ETH](data, data_len); + handlers = sse42_pclmulqdq_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC32_ETH](data, data_len); + handlers = neon_pmull_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC32_ETH](data, data_len); + handlers = handlers_scalar; + return handlers[RTE_NET_CRC32_ETH](data, data_len); +} + +/* Public API */ + void rte_net_crc_set_alg(enum rte_net_crc_alg alg) { + handlers = NULL; + if (max_simd_bitwidth == 0) + max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); + switch (alg) { -#ifdef X86_64_SSE42_PCLMULQDQ - case RTE_NET_CRC_SSE42: - handlers = handlers_sse42; - break; -#elif defined ARM64_NEON_PMULL + case RTE_NET_CRC_AVX512: + handlers = avx512_vpclmulqdq_get_handlers(); + if (handlers != NULL) + break; /* fall-through */ + case RTE_NET_CRC_SSE42: + handlers = sse42_pclmulqdq_get_handlers(); + break; /* for x86, always break here */ case RTE_NET_CRC_NEON: - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) { - handlers = handlers_neon; - break; - } -#endif + handlers = neon_pmull_get_handlers(); /* fall-through */ case RTE_NET_CRC_SCALAR: /* fall-through */ default: - handlers = handlers_scalar; break; } + + if (handlers == NULL) + handlers = handlers_scalar; } uint32_t @@ -175,22 +333,11 @@ rte_net_crc_calc(const void *data, return ret; } -/* Select highest available crc algorithm as default one */ +/* Call initialisation helpers for all crc algorithm handlers */ RTE_INIT(rte_net_crc_init) { - enum rte_net_crc_alg alg = RTE_NET_CRC_SCALAR; - rte_net_crc_scalar_init(); - -#ifdef X86_64_SSE42_PCLMULQDQ - alg = RTE_NET_CRC_SSE42; - rte_net_crc_sse42_init(); -#elif defined ARM64_NEON_PMULL - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) { - alg = RTE_NET_CRC_NEON; - rte_net_crc_neon_init(); - } -#endif - - rte_net_crc_set_alg(alg); + sse42_pclmulqdq_init(); + avx512_vpclmulqdq_init(); + neon_pmull_init(); }