-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2017 Intel Corporation.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2020 Intel Corporation
*/
#include <stddef.h>
#include <rte_cpuflags.h>
#include <rte_common.h>
#include <rte_net_crc.h>
+#include <rte_eal.h>
+#include <rte_log.h>
+#include <rte_vect.h>
-#if defined(RTE_ARCH_X86_64) && defined(RTE_MACHINE_CPUFLAG_PCLMULQDQ)
-#define X86_64_SSE42_PCLMULQDQ 1
-#elif defined(RTE_ARCH_ARM64) && defined(RTE_MACHINE_CPUFLAG_PMULL)
-#define ARM64_NEON_PMULL 1
-#endif
+#include "net_crc.h"
-#ifdef X86_64_SSE42_PCLMULQDQ
-#include <net_crc_sse.h>
-#elif defined ARM64_NEON_PMULL
-#include <net_crc_neon.h>
-#endif
+/** CRC polynomials */
+#define CRC32_ETH_POLYNOMIAL 0x04c11db7UL
+#define CRC16_CCITT_POLYNOMIAL 0x1021U
+
+#define CRC_LUT_SIZE 256
/* crc tables */
static uint32_t crc32_eth_lut[CRC_LUT_SIZE];
static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE];
+static uint32_t
+rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len);
+
+static uint32_t
+rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len);
+
static uint32_t
rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len);
typedef uint32_t
(*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len);
-static rte_net_crc_handler *handlers;
+static rte_net_crc_handler handlers_default[] = {
+ [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_default_handler,
+ [RTE_NET_CRC32_ETH] = rte_crc32_eth_default_handler,
+};
+
+static const rte_net_crc_handler *handlers = handlers_default;
-static rte_net_crc_handler handlers_scalar[] = {
+static const rte_net_crc_handler handlers_scalar[] = {
[RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler,
[RTE_NET_CRC32_ETH] = rte_crc32_eth_handler,
};
-
-#ifdef X86_64_SSE42_PCLMULQDQ
-static rte_net_crc_handler handlers_sse42[] = {
+#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
+static const rte_net_crc_handler handlers_avx512[] = {
+ [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler,
+ [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler,
+};
+#endif
+#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
+static const rte_net_crc_handler handlers_sse42[] = {
[RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler,
[RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler,
};
-#elif defined ARM64_NEON_PMULL
-static rte_net_crc_handler handlers_neon[] = {
+#endif
+#ifdef CC_ARM64_NEON_PMULL_SUPPORT
+static const rte_net_crc_handler handlers_neon[] = {
[RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler,
[RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler,
};
#endif
+static uint16_t max_simd_bitwidth;
+
+#define NET_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, libnet_logtype, "%s(): " fmt "\n", \
+ __func__, ## args)
+
+RTE_LOG_REGISTER(libnet_logtype, lib.net, INFO);
+
+/* Scalar handling */
+
/**
* Reflect the bits about the middle
*
uint32_t i, res = 0;
for (i = 0; i < 32; i++)
- if ((val & (1 << i)) != 0)
- res |= (uint32_t)(1 << (31 - i));
+ if ((val & (1U << i)) != 0)
+ res |= (uint32_t)(1U << (31 - i));
return res;
}
crc32_eth_lut);
}
+/* AVX512/VPCLMULQDQ handling */
+
+#define AVX512_VPCLMULQDQ_CPU_SUPPORTED ( \
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && \
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && \
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && \
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && \
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && \
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ) \
+)
+
+static const rte_net_crc_handler *
+avx512_vpclmulqdq_get_handlers(void)
+{
+#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
+ if (AVX512_VPCLMULQDQ_CPU_SUPPORTED &&
+ max_simd_bitwidth >= RTE_VECT_SIMD_512)
+ return handlers_avx512;
+#endif
+ NET_LOG(INFO, "Requirements not met, can't use AVX512\n");
+ return NULL;
+}
+
+static void
+avx512_vpclmulqdq_init(void)
+{
+#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
+ if (AVX512_VPCLMULQDQ_CPU_SUPPORTED)
+ rte_net_crc_avx512_init();
+#endif
+}
+
+/* SSE4.2/PCLMULQDQ handling */
+
+#define SSE42_PCLMULQDQ_CPU_SUPPORTED \
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ)
+
+static const rte_net_crc_handler *
+sse42_pclmulqdq_get_handlers(void)
+{
+#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
+ if (SSE42_PCLMULQDQ_CPU_SUPPORTED &&
+ max_simd_bitwidth >= RTE_VECT_SIMD_128)
+ return handlers_sse42;
+#endif
+ NET_LOG(INFO, "Requirements not met, can't use SSE\n");
+ return NULL;
+}
+
+static void
+sse42_pclmulqdq_init(void)
+{
+#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
+ if (SSE42_PCLMULQDQ_CPU_SUPPORTED)
+ rte_net_crc_sse42_init();
+#endif
+}
+
+/* NEON/PMULL handling */
+
+#define NEON_PMULL_CPU_SUPPORTED \
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)
+
+static const rte_net_crc_handler *
+neon_pmull_get_handlers(void)
+{
+#ifdef CC_ARM64_NEON_PMULL_SUPPORT
+ if (NEON_PMULL_CPU_SUPPORTED &&
+ max_simd_bitwidth >= RTE_VECT_SIMD_128)
+ return handlers_neon;
+#endif
+ NET_LOG(INFO, "Requirements not met, can't use NEON\n");
+ return NULL;
+}
+
+static void
+neon_pmull_init(void)
+{
+#ifdef CC_ARM64_NEON_PMULL_SUPPORT
+ if (NEON_PMULL_CPU_SUPPORTED)
+ rte_net_crc_neon_init();
+#endif
+}
+
+/* Default handling */
+
+static uint32_t
+rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len)
+{
+ handlers = NULL;
+ if (max_simd_bitwidth == 0)
+ max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
+
+ handlers = avx512_vpclmulqdq_get_handlers();
+ if (handlers != NULL)
+ return handlers[RTE_NET_CRC16_CCITT](data, data_len);
+ handlers = sse42_pclmulqdq_get_handlers();
+ if (handlers != NULL)
+ return handlers[RTE_NET_CRC16_CCITT](data, data_len);
+ handlers = neon_pmull_get_handlers();
+ if (handlers != NULL)
+ return handlers[RTE_NET_CRC16_CCITT](data, data_len);
+ handlers = handlers_scalar;
+ return handlers[RTE_NET_CRC16_CCITT](data, data_len);
+}
+
+static uint32_t
+rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len)
+{
+ handlers = NULL;
+ if (max_simd_bitwidth == 0)
+ max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
+
+ handlers = avx512_vpclmulqdq_get_handlers();
+ if (handlers != NULL)
+ return handlers[RTE_NET_CRC32_ETH](data, data_len);
+ handlers = sse42_pclmulqdq_get_handlers();
+ if (handlers != NULL)
+ return handlers[RTE_NET_CRC32_ETH](data, data_len);
+ handlers = neon_pmull_get_handlers();
+ if (handlers != NULL)
+ return handlers[RTE_NET_CRC32_ETH](data, data_len);
+ handlers = handlers_scalar;
+ return handlers[RTE_NET_CRC32_ETH](data, data_len);
+}
+
+/* Public API */
+
void
rte_net_crc_set_alg(enum rte_net_crc_alg alg)
{
+ handlers = NULL;
+ if (max_simd_bitwidth == 0)
+ max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
+
switch (alg) {
-#ifdef X86_64_SSE42_PCLMULQDQ
- case RTE_NET_CRC_SSE42:
- handlers = handlers_sse42;
- break;
-#elif defined ARM64_NEON_PMULL
+ case RTE_NET_CRC_AVX512:
+ handlers = avx512_vpclmulqdq_get_handlers();
+ if (handlers != NULL)
+ break;
/* fall-through */
+ case RTE_NET_CRC_SSE42:
+ handlers = sse42_pclmulqdq_get_handlers();
+ break; /* for x86, always break here */
case RTE_NET_CRC_NEON:
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) {
- handlers = handlers_neon;
- break;
- }
-#endif
+ handlers = neon_pmull_get_handlers();
/* fall-through */
case RTE_NET_CRC_SCALAR:
/* fall-through */
default:
- handlers = handlers_scalar;
break;
}
+
+ if (handlers == NULL)
+ handlers = handlers_scalar;
}
uint32_t
return ret;
}
-/* Select highest available crc algorithm as default one */
-static inline void __attribute__((constructor))
-rte_net_crc_init(void)
+/* Call initialisation helpers for all crc algorithm handlers */
+RTE_INIT(rte_net_crc_init)
{
- enum rte_net_crc_alg alg = RTE_NET_CRC_SCALAR;
-
rte_net_crc_scalar_init();
-
-#ifdef X86_64_SSE42_PCLMULQDQ
- alg = RTE_NET_CRC_SSE42;
- rte_net_crc_sse42_init();
-#elif defined ARM64_NEON_PMULL
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) {
- alg = RTE_NET_CRC_NEON;
- rte_net_crc_neon_init();
- }
-#endif
-
- rte_net_crc_set_alg(alg);
+ sse42_pclmulqdq_init();
+ avx512_vpclmulqdq_init();
+ neon_pmull_init();
}
git.droids-corp.org / dpdk.git / blobdiff