From da8dcc27f644bf5d895adb4a1d7a5aa38c12a10f Mon Sep 17 00:00:00 2001 From: Jerin Jacob Date: Tue, 24 Nov 2015 00:15:36 +0530 Subject: [PATCH] hash: use armv8-a CRC32 instructions armv8-a has optional CRC32 extension, march=armv8-a+crc enables code generation for the ARMv8-A architecture together with the optional CRC32 extensions. added RTE_MACHINE_CPUFLAG_CRC32 to detect the availability of CRC32 extension in compile time. At run-time, The RTE_CPUFLAG_CRC32 can be used to find the availability. armv8-a+crc target support added in GCC 4.9, Used inline assembly and emulated __ARM_FEATURE_CRC32 to work with tool-chain < 4.9 Signed-off-by: Jerin Jacob --- app/test/test_hash.c | 7 ++ lib/librte_hash/Makefile | 3 + lib/librte_hash/rte_crc_arm64.h | 151 +++++++++++++++++++++++ lib/librte_hash/rte_hash_crc.h | 7 ++ mk/machine/armv8a/rte.vars.mk | 2 +- mk/machine/thunderx/rte.vars.mk | 2 +- mk/rte.cpuflags.mk | 4 + mk/toolchain/gcc/rte.toolchain-compat.mk | 6 +- 8 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 lib/librte_hash/rte_crc_arm64.h diff --git a/app/test/test_hash.c b/app/test/test_hash.c index 4f2509d384..2f3d884ea2 100644 --- a/app/test/test_hash.c +++ b/app/test/test_hash.c @@ -217,6 +217,13 @@ test_crc32_hash_alg_equiv(void) printf("Failed checking CRC32_SW against CRC32_SSE42_x64\n"); break; } + + /* Check against 8-byte-operand ARM64 CRC32 if available */ + rte_hash_crc_set_alg(CRC32_ARM64); + if (hash_val != rte_hash_crc(data64, data_len, init_val)) { + printf("Failed checking CRC32_SW against CRC32_ARM64\n"); + break; + } } /* Resetting to best available algorithm */ diff --git a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile index 7902c2b85a..bb1ea9905b 100644 --- a/lib/librte_hash/Makefile +++ b/lib/librte_hash/Makefile @@ -48,6 +48,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_hash_crc.h +ifeq ($(CONFIG_RTE_ARCH_ARM64),y) +SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_crc_arm64.h +endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h diff --git a/lib/librte_hash/rte_crc_arm64.h b/lib/librte_hash/rte_crc_arm64.h new file mode 100644 index 0000000000..02e26bcada --- /dev/null +++ b/lib/librte_hash/rte_crc_arm64.h @@ -0,0 +1,151 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015 Cavium networks. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_CRC_ARM64_H_ +#define _RTE_CRC_ARM64_H_ + +/** + * @file + * + * RTE CRC arm64 Hash + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +static inline uint32_t +crc32c_arm64_u32(uint32_t data, uint32_t init_val) +{ + asm(".arch armv8-a+crc"); + __asm__ volatile( + "crc32cw %w[crc], %w[crc], %w[value]" + : [crc] "+r" (init_val) + : [value] "r" (data)); + return init_val; +} + +static inline uint32_t +crc32c_arm64_u64(uint64_t data, uint32_t init_val) +{ + asm(".arch armv8-a+crc"); + __asm__ volatile( + "crc32cx %w[crc], %w[crc], %x[value]" + : [crc] "+r" (init_val) + : [value] "r" (data)); + return init_val; +} + +/** + * Allow or disallow use of arm64 SIMD instrinsics for CRC32 hash + * calculation. + * + * @param alg + * An OR of following flags: + * - (CRC32_SW) Don't use arm64 crc intrinsics + * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available + * + */ +static inline void +rte_hash_crc_set_alg(uint8_t alg) +{ + switch (alg) { + case CRC32_ARM64: + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32)) + alg = CRC32_SW; + case CRC32_SW: + crc32_alg = alg; + default: + break; + } +} + +/* Setting the best available algorithm */ +static inline void __attribute__((constructor)) +rte_hash_crc_init_alg(void) +{ + rte_hash_crc_set_alg(CRC32_ARM64); +} + +/** + * Use single crc32 instruction to perform a hash on a 4 byte value. + * Fall back to software crc32 implementation in case arm64 crc intrinsics is + * not supported + * + * @param data + * Data to perform hash on. + * @param init_val + * Value to initialise hash generator. + * @return + * 32bit calculated hash value. + */ +static inline uint32_t +rte_hash_crc_4byte(uint32_t data, uint32_t init_val) +{ + if (likely(crc32_alg & CRC32_ARM64)) + return crc32c_arm64_u32(data, init_val); + + return crc32c_1word(data, init_val); +} + +/** + * Use single crc32 instruction to perform a hash on a 8 byte value. + * Fall back to software crc32 implementation in case arm64 crc intrinsics is + * not supported + * + * @param data + * Data to perform hash on. + * @param init_val + * Value to initialise hash generator. + * @return + * 32bit calculated hash value. + */ +static inline uint32_t +rte_hash_crc_8byte(uint64_t data, uint32_t init_val) +{ + if (likely(crc32_alg == CRC32_ARM64)) + return crc32c_arm64_u64(data, init_val); + + return crc32c_2words(data, init_val); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_CRC_ARM64_H_ */ diff --git a/lib/librte_hash/rte_hash_crc.h b/lib/librte_hash/rte_hash_crc.h index 1f6f5bfc18..78a34b7642 100644 --- a/lib/librte_hash/rte_hash_crc.h +++ b/lib/librte_hash/rte_hash_crc.h @@ -407,9 +407,14 @@ crc32c_sse42_u64(uint64_t data, uint64_t init_val) #define CRC32_SSE42 (1U << 1) #define CRC32_x64 (1U << 2) #define CRC32_SSE42_x64 (CRC32_x64|CRC32_SSE42) +#define CRC32_ARM64 (1U << 3) static uint8_t crc32_alg = CRC32_SW; +#if defined(RTE_ARCH_ARM64) +#include "rte_crc_arm64.h" +#else + /** * Allow or disallow use of SSE4.2 instrinsics for CRC32 hash * calculation. @@ -498,6 +503,8 @@ rte_hash_crc_8byte(uint64_t data, uint32_t init_val) return crc32c_2words(data, init_val); } +#endif + /** * Calculate CRC32 hash on user-supplied byte array. * diff --git a/mk/machine/armv8a/rte.vars.mk b/mk/machine/armv8a/rte.vars.mk index bdf8c6b16b..8c018a4405 100644 --- a/mk/machine/armv8a/rte.vars.mk +++ b/mk/machine/armv8a/rte.vars.mk @@ -55,4 +55,4 @@ # CPU_LDFLAGS = # CPU_ASFLAGS = -MACHINE_CFLAGS += -march=armv8-a -DRTE_CACHE_LINE_SIZE=64 +MACHINE_CFLAGS += -march=armv8-a+crc -DRTE_CACHE_LINE_SIZE=64 diff --git a/mk/machine/thunderx/rte.vars.mk b/mk/machine/thunderx/rte.vars.mk index e49f9e1901..0bb6b3d009 100644 --- a/mk/machine/thunderx/rte.vars.mk +++ b/mk/machine/thunderx/rte.vars.mk @@ -55,4 +55,4 @@ CROSS ?= aarch64-thunderx-linux-gnu- # CPU_LDFLAGS = # CPU_ASFLAGS = -MACHINE_CFLAGS += -march=armv8-a -mcpu=thunderx -DRTE_CACHE_LINE_SIZE=128 +MACHINE_CFLAGS += -march=armv8-a+crc -mcpu=thunderx -DRTE_CACHE_LINE_SIZE=128 diff --git a/mk/rte.cpuflags.mk b/mk/rte.cpuflags.mk index bec7bddf4b..0a340a9d89 100644 --- a/mk/rte.cpuflags.mk +++ b/mk/rte.cpuflags.mk @@ -111,6 +111,10 @@ ifneq ($(filter $(AUTO_CPUFLAGS),__ARM_NEON_FP),) CPUFLAGS += NEON endif +ifneq ($(filter $(AUTO_CPUFLAGS),__ARM_FEATURE_CRC32),) +CPUFLAGS += CRC32 +endif + MACHINE_CFLAGS += $(addprefix -DRTE_MACHINE_CPUFLAG_,$(CPUFLAGS)) diff --git a/mk/toolchain/gcc/rte.toolchain-compat.mk b/mk/toolchain/gcc/rte.toolchain-compat.mk index 61bb5b7134..e144216732 100644 --- a/mk/toolchain/gcc/rte.toolchain-compat.mk +++ b/mk/toolchain/gcc/rte.toolchain-compat.mk @@ -54,7 +54,11 @@ else # GCC 4.5.x - added support for atom # GCC 4.6.x - added support for corei7, corei7-avx # GCC 4.7.x - added support for fsgsbase, rdrnd, f16c, core-avx-i, core-avx2 - +# GCC 4.9.x - added support for armv8-a+crc +# + ifeq ($(shell test $(GCC_VERSION) -le 49 && echo 1), 1) + MACHINE_CFLAGS := $(patsubst -march=armv8-a+crc,-march=armv8-a+crc -D__ARM_FEATURE_CRC32=1,$(MACHINE_CFLAGS)) + endif ifeq ($(shell test $(GCC_VERSION) -le 47 && echo 1), 1) MACHINE_CFLAGS := $(patsubst -march=core-avx-i,-march=corei7-avx,$(MACHINE_CFLAGS)) MACHINE_CFLAGS := $(patsubst -march=core-avx2,-march=core-avx2,$(MACHINE_CFLAGS)) -- 2.20.1