X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_net%2Frte_net_crc.c;h=1fe58f7c7565b4509d4193e2376f33221a15920b;hb=5b38d8cd4663;hp=0391c7209a8ee233504c12dca45c38b44739a0d0;hpb=c0583d98a9153549383d236e5cc5464bbbe8dd89;p=dpdk.git diff --git a/lib/librte_net/rte_net_crc.c b/lib/librte_net/rte_net_crc.c index 0391c7209a..1fe58f7c75 100644 --- a/lib/librte_net/rte_net_crc.c +++ b/lib/librte_net/rte_net_crc.c @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2017 Intel Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017-2020 Intel Corporation */ #include @@ -38,21 +9,28 @@ #include #include #include +#include +#include +#include -#if defined(RTE_ARCH_X86_64) \ - && defined(RTE_MACHINE_CPUFLAG_SSE4_2) \ - && defined(RTE_MACHINE_CPUFLAG_PCLMULQDQ) -#define X86_64_SSE42_PCLMULQDQ 1 -#endif +#include "net_crc.h" -#ifdef X86_64_SSE42_PCLMULQDQ -#include -#endif +/** CRC polynomials */ +#define CRC32_ETH_POLYNOMIAL 0x04c11db7UL +#define CRC16_CCITT_POLYNOMIAL 0x1021U + +#define CRC_LUT_SIZE 256 /* crc tables */ static uint32_t crc32_eth_lut[CRC_LUT_SIZE]; static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE]; +static uint32_t +rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len); + +static uint32_t +rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len); + static uint32_t rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len); @@ -62,19 +40,45 @@ rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len); typedef uint32_t (*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len); -static rte_net_crc_handler *handlers; +static rte_net_crc_handler handlers_default[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_default_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_default_handler, +}; + +static const rte_net_crc_handler *handlers = handlers_default; -static rte_net_crc_handler handlers_scalar[] = { +static const rte_net_crc_handler handlers_scalar[] = { [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler, [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler, }; - -#ifdef X86_64_SSE42_PCLMULQDQ -static rte_net_crc_handler handlers_sse42[] = { +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT +static const rte_net_crc_handler handlers_avx512[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler, +}; +#endif +#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT +static const rte_net_crc_handler handlers_sse42[] = { [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler, [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler, }; #endif +#ifdef CC_ARM64_NEON_PMULL_SUPPORT +static const rte_net_crc_handler handlers_neon[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler, +}; +#endif + +static uint16_t max_simd_bitwidth; + +#define NET_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, libnet_logtype, "%s(): " fmt "\n", \ + __func__, ## args) + +RTE_LOG_REGISTER(libnet_logtype, lib.net, INFO); + +/* Scalar handling */ /** * Reflect the bits about the middle @@ -91,8 +95,8 @@ reflect_32bits(uint32_t val) uint32_t i, res = 0; for (i = 0; i < 32; i++) - if ((val & (1 << i)) != 0) - res |= (uint32_t)(1 << (31 - i)); + if ((val & (1U << i)) != 0) + res |= (uint32_t)(1U << (31 - i)); return res; } @@ -158,22 +162,161 @@ rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len) crc32_eth_lut); } +/* AVX512/VPCLMULQDQ handling */ + +#define AVX512_VPCLMULQDQ_CPU_SUPPORTED ( \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ) \ +) + +static const rte_net_crc_handler * +avx512_vpclmulqdq_get_handlers(void) +{ +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT + if (AVX512_VPCLMULQDQ_CPU_SUPPORTED && + max_simd_bitwidth >= RTE_VECT_SIMD_512) + return handlers_avx512; +#endif + NET_LOG(INFO, "Requirements not met, can't use AVX512\n"); + return NULL; +} + +static void +avx512_vpclmulqdq_init(void) +{ +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT + if (AVX512_VPCLMULQDQ_CPU_SUPPORTED) + rte_net_crc_avx512_init(); +#endif +} + +/* SSE4.2/PCLMULQDQ handling */ + +#define SSE42_PCLMULQDQ_CPU_SUPPORTED \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) + +static const rte_net_crc_handler * +sse42_pclmulqdq_get_handlers(void) +{ +#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT + if (SSE42_PCLMULQDQ_CPU_SUPPORTED && + max_simd_bitwidth >= RTE_VECT_SIMD_128) + return handlers_sse42; +#endif + NET_LOG(INFO, "Requirements not met, can't use SSE\n"); + return NULL; +} + +static void +sse42_pclmulqdq_init(void) +{ +#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT + if (SSE42_PCLMULQDQ_CPU_SUPPORTED) + rte_net_crc_sse42_init(); +#endif +} + +/* NEON/PMULL handling */ + +#define NEON_PMULL_CPU_SUPPORTED \ + rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL) + +static const rte_net_crc_handler * +neon_pmull_get_handlers(void) +{ +#ifdef CC_ARM64_NEON_PMULL_SUPPORT + if (NEON_PMULL_CPU_SUPPORTED && + max_simd_bitwidth >= RTE_VECT_SIMD_128) + return handlers_neon; +#endif + NET_LOG(INFO, "Requirements not met, can't use NEON\n"); + return NULL; +} + +static void +neon_pmull_init(void) +{ +#ifdef CC_ARM64_NEON_PMULL_SUPPORT + if (NEON_PMULL_CPU_SUPPORTED) + rte_net_crc_neon_init(); +#endif +} + +/* Default handling */ + +static uint32_t +rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len) +{ + handlers = NULL; + if (max_simd_bitwidth == 0) + max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); + + handlers = avx512_vpclmulqdq_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC16_CCITT](data, data_len); + handlers = sse42_pclmulqdq_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC16_CCITT](data, data_len); + handlers = neon_pmull_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC16_CCITT](data, data_len); + handlers = handlers_scalar; + return handlers[RTE_NET_CRC16_CCITT](data, data_len); +} + +static uint32_t +rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len) +{ + handlers = NULL; + if (max_simd_bitwidth == 0) + max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); + + handlers = avx512_vpclmulqdq_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC32_ETH](data, data_len); + handlers = sse42_pclmulqdq_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC32_ETH](data, data_len); + handlers = neon_pmull_get_handlers(); + if (handlers != NULL) + return handlers[RTE_NET_CRC32_ETH](data, data_len); + handlers = handlers_scalar; + return handlers[RTE_NET_CRC32_ETH](data, data_len); +} + +/* Public API */ + void rte_net_crc_set_alg(enum rte_net_crc_alg alg) { + handlers = NULL; + if (max_simd_bitwidth == 0) + max_simd_bitwidth = rte_vect_get_max_simd_bitwidth(); + switch (alg) { + case RTE_NET_CRC_AVX512: + handlers = avx512_vpclmulqdq_get_handlers(); + if (handlers != NULL) + break; + /* fall-through */ case RTE_NET_CRC_SSE42: -#ifdef X86_64_SSE42_PCLMULQDQ - handlers = handlers_sse42; -#else - alg = RTE_NET_CRC_SCALAR; -#endif - break; + handlers = sse42_pclmulqdq_get_handlers(); + break; /* for x86, always break here */ + case RTE_NET_CRC_NEON: + handlers = neon_pmull_get_handlers(); + /* fall-through */ case RTE_NET_CRC_SCALAR: + /* fall-through */ default: - handlers = handlers_scalar; break; } + + if (handlers == NULL) + handlers = handlers_scalar; } uint32_t @@ -190,18 +333,11 @@ rte_net_crc_calc(const void *data, return ret; } -/* Select highest available crc algorithm as default one */ -static inline void __attribute__((constructor)) -rte_net_crc_init(void) +/* Call initialisation helpers for all crc algorithm handlers */ +RTE_INIT(rte_net_crc_init) { - enum rte_net_crc_alg alg = RTE_NET_CRC_SCALAR; - rte_net_crc_scalar_init(); - -#ifdef X86_64_SSE42_PCLMULQDQ - alg = RTE_NET_CRC_SSE42; - rte_net_crc_sse42_init(); -#endif - - rte_net_crc_set_alg(alg); + sse42_pclmulqdq_init(); + avx512_vpclmulqdq_init(); + neon_pmull_init(); }