net/bnxt: use first completion ring for fwd and async event
[dpdk.git] / lib / librte_net / rte_net_crc.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Intel Corporation
3  */
4
5 #include <stddef.h>
6 #include <string.h>
7 #include <stdint.h>
8
9 #include <rte_cpuflags.h>
10 #include <rte_common.h>
11 #include <rte_net_crc.h>
12
13 #if defined(RTE_ARCH_X86_64) && defined(RTE_MACHINE_CPUFLAG_PCLMULQDQ)
14 #define X86_64_SSE42_PCLMULQDQ     1
15 #elif defined(RTE_ARCH_ARM64) && defined(RTE_MACHINE_CPUFLAG_PMULL)
16 #define ARM64_NEON_PMULL           1
17 #endif
18
19 #ifdef X86_64_SSE42_PCLMULQDQ
20 #include <net_crc_sse.h>
21 #elif defined ARM64_NEON_PMULL
22 #include <net_crc_neon.h>
23 #endif
24
25 /* crc tables */
26 static uint32_t crc32_eth_lut[CRC_LUT_SIZE];
27 static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE];
28
29 static uint32_t
30 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len);
31
32 static uint32_t
33 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len);
34
35 typedef uint32_t
36 (*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len);
37
38 static rte_net_crc_handler *handlers;
39
40 static rte_net_crc_handler handlers_scalar[] = {
41         [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler,
42         [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler,
43 };
44
45 #ifdef X86_64_SSE42_PCLMULQDQ
46 static rte_net_crc_handler handlers_sse42[] = {
47         [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler,
48         [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler,
49 };
50 #elif defined ARM64_NEON_PMULL
51 static rte_net_crc_handler handlers_neon[] = {
52         [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler,
53         [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler,
54 };
55 #endif
56
57 /**
58  * Reflect the bits about the middle
59  *
60  * @param val
61  *   value to be reflected
62  *
63  * @return
64  *   reflected value
65  */
66 static uint32_t
67 reflect_32bits(uint32_t val)
68 {
69         uint32_t i, res = 0;
70
71         for (i = 0; i < 32; i++)
72                 if ((val & (1 << i)) != 0)
73                         res |= (uint32_t)(1 << (31 - i));
74
75         return res;
76 }
77
78 static void
79 crc32_eth_init_lut(uint32_t poly,
80         uint32_t *lut)
81 {
82         uint32_t i, j;
83
84         for (i = 0; i < CRC_LUT_SIZE; i++) {
85                 uint32_t crc = reflect_32bits(i);
86
87                 for (j = 0; j < 8; j++) {
88                         if (crc & 0x80000000L)
89                                 crc = (crc << 1) ^ poly;
90                         else
91                                 crc <<= 1;
92                 }
93                 lut[i] = reflect_32bits(crc);
94         }
95 }
96
97 static __rte_always_inline uint32_t
98 crc32_eth_calc_lut(const uint8_t *data,
99         uint32_t data_len,
100         uint32_t crc,
101         const uint32_t *lut)
102 {
103         while (data_len--)
104                 crc = lut[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
105
106         return crc;
107 }
108
109 static void
110 rte_net_crc_scalar_init(void)
111 {
112         /* 32-bit crc init */
113         crc32_eth_init_lut(CRC32_ETH_POLYNOMIAL, crc32_eth_lut);
114
115         /* 16-bit CRC init */
116         crc32_eth_init_lut(CRC16_CCITT_POLYNOMIAL << 16, crc16_ccitt_lut);
117 }
118
119 static inline uint32_t
120 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len)
121 {
122         /* return 16-bit CRC value */
123         return (uint16_t)~crc32_eth_calc_lut(data,
124                 data_len,
125                 0xffff,
126                 crc16_ccitt_lut);
127 }
128
129 static inline uint32_t
130 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len)
131 {
132         /* return 32-bit CRC value */
133         return ~crc32_eth_calc_lut(data,
134                 data_len,
135                 0xffffffffUL,
136                 crc32_eth_lut);
137 }
138
139 void
140 rte_net_crc_set_alg(enum rte_net_crc_alg alg)
141 {
142         switch (alg) {
143 #ifdef X86_64_SSE42_PCLMULQDQ
144         case RTE_NET_CRC_SSE42:
145                 handlers = handlers_sse42;
146                 break;
147 #elif defined ARM64_NEON_PMULL
148                 /* fall-through */
149         case RTE_NET_CRC_NEON:
150                 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) {
151                         handlers = handlers_neon;
152                         break;
153                 }
154 #endif
155                 /* fall-through */
156         case RTE_NET_CRC_SCALAR:
157                 /* fall-through */
158         default:
159                 handlers = handlers_scalar;
160                 break;
161         }
162 }
163
164 uint32_t
165 rte_net_crc_calc(const void *data,
166         uint32_t data_len,
167         enum rte_net_crc_type type)
168 {
169         uint32_t ret;
170         rte_net_crc_handler f_handle;
171
172         f_handle = handlers[type];
173         ret = f_handle(data, data_len);
174
175         return ret;
176 }
177
178 /* Select highest available crc algorithm as default one */
179 RTE_INIT(rte_net_crc_init)
180 {
181         enum rte_net_crc_alg alg = RTE_NET_CRC_SCALAR;
182
183         rte_net_crc_scalar_init();
184
185 #ifdef X86_64_SSE42_PCLMULQDQ
186         alg = RTE_NET_CRC_SSE42;
187         rte_net_crc_sse42_init();
188 #elif defined ARM64_NEON_PMULL
189         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) {
190                 alg = RTE_NET_CRC_NEON;
191                 rte_net_crc_neon_init();
192         }
193 #endif
194
195         rte_net_crc_set_alg(alg);
196 }