1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
5 #ifndef _RTE_THASH_X86_GFNI_H_
6 #define _RTE_THASH_X86_GFNI_H_
11 * Optimized Toeplitz hash functions implementation
12 * using Galois Fields New Instructions.
22 #define RTE_THASH_GFNI_DEFINED
24 #define RTE_THASH_FIRST_ITER_MSK 0x0f0f0f0f0f0e0c08
25 #define RTE_THASH_PERM_MSK 0x0f0f0f0f0f0f0f0f
26 #define RTE_THASH_FIRST_ITER_MSK_2 0xf0f0f0f0f0e0c080
27 #define RTE_THASH_PERM_MSK_2 0xf0f0f0f0f0f0f0f0
28 #define RTE_THASH_REWIND_MSK 0x0000000000113377
32 __rte_thash_xor_reduce(__m512i xor_acc, uint32_t *val_1, uint32_t *val_2)
34 __m256i tmp_256_1, tmp_256_2;
35 __m128i tmp128_1, tmp128_2;
36 uint64_t tmp_1, tmp_2;
38 tmp_256_1 = _mm512_castsi512_si256(xor_acc);
39 tmp_256_2 = _mm512_extracti32x8_epi32(xor_acc, 1);
40 tmp_256_1 = _mm256_xor_si256(tmp_256_1, tmp_256_2);
42 tmp128_1 = _mm256_castsi256_si128(tmp_256_1);
43 tmp128_2 = _mm256_extracti32x4_epi32(tmp_256_1, 1);
44 tmp128_1 = _mm_xor_si128(tmp128_1, tmp128_2);
46 tmp_1 = _mm_extract_epi64(tmp128_1, 0);
47 tmp_2 = _mm_extract_epi64(tmp128_1, 1);
50 *val_1 = (uint32_t)tmp_1;
51 *val_2 = (uint32_t)(tmp_1 >> 32);
56 __rte_thash_gfni(const uint64_t *mtrx, const uint8_t *tuple,
57 const uint8_t *secondary_tuple, int len)
59 __m512i permute_idx = _mm512_set_epi8(7, 6, 5, 4, 7, 6, 5, 4,
60 6, 5, 4, 3, 6, 5, 4, 3,
61 5, 4, 3, 2, 5, 4, 3, 2,
62 4, 3, 2, 1, 4, 3, 2, 1,
63 3, 2, 1, 0, 3, 2, 1, 0,
64 2, 1, 0, -1, 2, 1, 0, -1,
65 1, 0, -1, -2, 1, 0, -1, -2,
66 0, -1, -2, -3, 0, -1, -2, -3);
68 const __m512i rewind_idx = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,
69 0, 0, 0, 0, 0, 0, 0, 0,
70 0, 0, 0, 0, 0, 0, 0, 0,
71 0, 0, 0, 0, 0, 0, 0, 0,
72 0, 0, 0, 0, 0, 0, 0, 0,
73 0, 0, 0, 59, 0, 0, 0, 59,
74 0, 0, 59, 58, 0, 0, 59, 58,
75 0, 59, 58, 57, 0, 59, 58, 57);
76 const __mmask64 rewind_mask = RTE_THASH_REWIND_MSK;
77 const __m512i shift_8 = _mm512_set1_epi8(8);
78 __m512i xor_acc = _mm512_setzero_si512();
79 __m512i perm_bytes = _mm512_setzero_si512();
80 __m512i vals, matrixes, tuple_bytes, tuple_bytes_2;
81 __mmask64 load_mask, permute_mask, permute_mask_2;
82 int chunk_len = 0, i = 0;
84 const int prepend = 3;
86 for (; len > 0; len -= 64, tuple += 64) {
88 perm_bytes = _mm512_maskz_permutexvar_epi8(rewind_mask,
89 rewind_idx, perm_bytes);
91 permute_mask = RTE_THASH_FIRST_ITER_MSK;
92 load_mask = (len >= 64) ? UINT64_MAX : ((1ULL << len) - 1);
93 tuple_bytes = _mm512_maskz_loadu_epi8(load_mask, tuple);
94 if (secondary_tuple) {
95 permute_mask_2 = RTE_THASH_FIRST_ITER_MSK_2;
96 tuple_bytes_2 = _mm512_maskz_loadu_epi8(load_mask,
100 chunk_len = __builtin_popcountll(load_mask);
101 for (i = 0; i < ((chunk_len + prepend) / 8); i++, mtrx += 8) {
102 perm_bytes = _mm512_mask_permutexvar_epi8(perm_bytes,
103 permute_mask, permute_idx, tuple_bytes);
107 _mm512_mask_permutexvar_epi8(perm_bytes,
108 permute_mask_2, permute_idx,
111 matrixes = _mm512_maskz_loadu_epi64(UINT8_MAX, mtrx);
112 vals = _mm512_gf2p8affine_epi64_epi8(perm_bytes,
115 xor_acc = _mm512_xor_si512(xor_acc, vals);
116 permute_idx = _mm512_add_epi8(permute_idx, shift_8);
117 permute_mask = RTE_THASH_PERM_MSK;
119 permute_mask_2 = RTE_THASH_PERM_MSK_2;
123 int rest_len = (chunk_len + prepend) % 8;
125 mtrx_msk = (1 << (rest_len % 8)) - 1;
126 matrixes = _mm512_maskz_loadu_epi64(mtrx_msk, mtrx);
128 perm_bytes = _mm512_maskz_permutexvar_epi8(rewind_mask,
129 rewind_idx, perm_bytes);
131 perm_bytes = _mm512_mask_permutexvar_epi8(perm_bytes,
132 permute_mask, permute_idx, tuple_bytes);
136 _mm512_mask_permutexvar_epi8(
137 perm_bytes, permute_mask_2,
138 permute_idx, tuple_bytes_2);
141 vals = _mm512_gf2p8affine_epi64_epi8(perm_bytes, matrixes, 0);
142 xor_acc = _mm512_xor_si512(xor_acc, vals);
149 * Calculate Toeplitz hash.
152 * @b EXPERIMENTAL: this API may change without prior notice.
155 * Pointer to the matrices generated from the corresponding
156 * RSS hash key using rte_thash_complete_matrix().
157 * Note that @p len should not exceed the length of the rss_key minus 4.
159 * Pointer to the data to be hashed. Data must be in network byte order.
161 * Length of the data to be hashed.
163 * Calculated Toeplitz hash value.
166 static inline uint32_t
167 rte_thash_gfni(const uint64_t *m, const uint8_t *tuple, int len)
169 uint32_t val, val_zero;
171 __m512i xor_acc = __rte_thash_gfni(m, tuple, NULL, len);
172 __rte_thash_xor_reduce(xor_acc, &val, &val_zero);
178 * Bulk implementation for Toeplitz hash.
181 * @b EXPERIMENTAL: this API may change without prior notice.
184 * Pointer to the matrices generated from the corresponding
185 * RSS hash key using rte_thash_complete_matrix().
186 * Note that @p len should not exceed the length of the rss_key minus 4.
188 * Length of the largest data buffer to be hashed.
190 * Array of the pointers on data to be hashed.
191 * Data must be in network byte order.
193 * Array of uint32_t where to put calculated Toeplitz hash values
195 * Number of tuples to hash.
199 rte_thash_gfni_bulk(const uint64_t *mtrx, int len, uint8_t *tuple[],
200 uint32_t val[], uint32_t num)
206 for (i = 0; i != (num & ~1); i += 2) {
207 xor_acc = __rte_thash_gfni(mtrx, tuple[i], tuple[i + 1], len);
208 __rte_thash_xor_reduce(xor_acc, val + i, val + i + 1);
212 xor_acc = __rte_thash_gfni(mtrx, tuple[i], NULL, len);
213 __rte_thash_xor_reduce(xor_acc, val + i, &val_zero);
223 #endif /* _RTE_THASH_X86_GFNI_H_ */