lib/hash/rte_thash_x86_gfni.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2021 Intel Corporation
   3  */
   4
   5 #ifndef _RTE_THASH_X86_GFNI_H_
   6 #define _RTE_THASH_X86_GFNI_H_
   7
   8 /**
   9  * @file
  10  *
  11  * Optimized Toeplitz hash functions implementation
  12  * using Galois Fields New Instructions.
  13  */
  14
  15 #include <rte_vect.h>
  16
  17 #ifdef __cplusplus
  18 extern "C" {
  19 #endif
  20
  21 #if defined(__GFNI__) && defined(__AVX512F__)
  22 #define RTE_THASH_GFNI_DEFINED
  23
  24 #define RTE_THASH_FIRST_ITER_MSK        0x0f0f0f0f0f0e0c08
  25 #define RTE_THASH_PERM_MSK              0x0f0f0f0f0f0f0f0f
  26 #define RTE_THASH_FIRST_ITER_MSK_2      0xf0f0f0f0f0e0c080
  27 #define RTE_THASH_PERM_MSK_2            0xf0f0f0f0f0f0f0f0
  28 #define RTE_THASH_REWIND_MSK            0x0000000000113377
  29
  30 __rte_internal
  31 static inline void
  32 __rte_thash_xor_reduce(__m512i xor_acc, uint32_t *val_1, uint32_t *val_2)
  33 {
  34         __m256i tmp_256_1, tmp_256_2;
  35         __m128i tmp128_1, tmp128_2;
  36
  37         tmp_256_1 = _mm512_castsi512_si256(xor_acc);
  38         tmp_256_2 = _mm512_extracti32x8_epi32(xor_acc, 1);
  39         tmp_256_1 = _mm256_xor_si256(tmp_256_1, tmp_256_2);
  40
  41         tmp128_1 = _mm256_castsi256_si128(tmp_256_1);
  42         tmp128_2 = _mm256_extracti32x4_epi32(tmp_256_1, 1);
  43         tmp128_1 = _mm_xor_si128(tmp128_1, tmp128_2);
  44
  45 #ifdef RTE_ARCH_X86_64
  46         uint64_t tmp_1, tmp_2;
  47         tmp_1 = _mm_extract_epi64(tmp128_1, 0);
  48         tmp_2 = _mm_extract_epi64(tmp128_1, 1);
  49         tmp_1 ^= tmp_2;
  50
  51         *val_1 = (uint32_t)tmp_1;
  52         *val_2 = (uint32_t)(tmp_1 >> 32);
  53 #else
  54         uint32_t tmp_1, tmp_2;
  55         tmp_1 = _mm_extract_epi32(tmp128_1, 0);
  56         tmp_2 = _mm_extract_epi32(tmp128_1, 1);
  57         tmp_1 ^= _mm_extract_epi32(tmp128_1, 2);
  58         tmp_2 ^= _mm_extract_epi32(tmp128_1, 3);
  59
  60         *val_1 = tmp_1;
  61         *val_2 = tmp_2;
  62 #endif
  63 }
  64
  65 __rte_internal
  66 static inline __m512i
  67 __rte_thash_gfni(const uint64_t *mtrx, const uint8_t *tuple,
  68         const uint8_t *secondary_tuple, int len)
  69 {
  70         __m512i permute_idx = _mm512_set_epi32(0x07060504, 0x07060504,
  71                 0x06050403, 0x06050403,
  72                 0x05040302, 0x05040302,
  73                 0x04030201, 0x04030201,
  74                 0x03020100, 0x03020100,
  75                 0x020100FF, 0x020100FF,
  76                 0x0100FFFE, 0x0100FFFE,
  77                 0x00FFFEFD, 0x00FFFEFD);
  78         const __m512i rewind_idx = _mm512_set_epi32(0x00000000, 0x00000000,
  79                 0x00000000, 0x00000000,
  80                 0x00000000, 0x00000000,
  81                 0x00000000, 0x00000000,
  82                 0x00000000, 0x00000000,
  83                 0x0000003B, 0x0000003B,
  84                 0x00003B3A, 0x00003B3A,
  85                 0x003B3A39, 0x003B3A39);
  86         const __mmask64 rewind_mask = RTE_THASH_REWIND_MSK;
  87         const __m512i shift_8 = _mm512_set1_epi8(8);
  88         __m512i xor_acc = _mm512_setzero_si512();
  89         __m512i perm_bytes = _mm512_setzero_si512();
  90         __m512i vals, matrixes, tuple_bytes, tuple_bytes_2;
  91         __mmask64 load_mask, permute_mask, permute_mask_2;
  92         int chunk_len = 0, i = 0;
  93         uint8_t mtrx_msk;
  94         const int prepend = 3;
  95
  96         for (; len > 0; len -= 64, tuple += 64) {
  97                 if (i == 8)
  98                         perm_bytes = _mm512_maskz_permutexvar_epi8(rewind_mask,
  99                                 rewind_idx, perm_bytes);
 100
 101                 permute_mask = RTE_THASH_FIRST_ITER_MSK;
 102                 load_mask = (len >= 64) ? UINT64_MAX : ((1ULL << len) - 1);
 103                 tuple_bytes = _mm512_maskz_loadu_epi8(load_mask, tuple);
 104                 if (secondary_tuple) {
 105                         permute_mask_2 = RTE_THASH_FIRST_ITER_MSK_2;
 106                         tuple_bytes_2 = _mm512_maskz_loadu_epi8(load_mask,
 107                                 secondary_tuple);
 108                 }
 109
 110                 chunk_len = __builtin_popcountll(load_mask);
 111                 for (i = 0; i < ((chunk_len + prepend) / 8); i++, mtrx += 8) {
 112                         perm_bytes = _mm512_mask_permutexvar_epi8(perm_bytes,
 113                                 permute_mask, permute_idx, tuple_bytes);
 114
 115                         if (secondary_tuple)
 116                                 perm_bytes =
 117                                         _mm512_mask_permutexvar_epi8(perm_bytes,
 118                                         permute_mask_2, permute_idx,
 119                                         tuple_bytes_2);
 120
 121                         matrixes = _mm512_maskz_loadu_epi64(UINT8_MAX, mtrx);
 122                         vals = _mm512_gf2p8affine_epi64_epi8(perm_bytes,
 123                                 matrixes, 0);
 124
 125                         xor_acc = _mm512_xor_si512(xor_acc, vals);
 126                         permute_idx = _mm512_add_epi8(permute_idx, shift_8);
 127                         permute_mask = RTE_THASH_PERM_MSK;
 128                         if (secondary_tuple)
 129                                 permute_mask_2 = RTE_THASH_PERM_MSK_2;
 130                 }
 131         }
 132
 133         int rest_len = (chunk_len + prepend) % 8;
 134         if (rest_len != 0) {
 135                 mtrx_msk = (1 << (rest_len % 8)) - 1;
 136                 matrixes = _mm512_maskz_loadu_epi64(mtrx_msk, mtrx);
 137                 if (i == 8) {
 138                         perm_bytes = _mm512_maskz_permutexvar_epi8(rewind_mask,
 139                                 rewind_idx, perm_bytes);
 140                 } else {
 141                         perm_bytes = _mm512_mask_permutexvar_epi8(perm_bytes,
 142                                 permute_mask, permute_idx, tuple_bytes);
 143
 144                         if (secondary_tuple)
 145                                 perm_bytes =
 146                                         _mm512_mask_permutexvar_epi8(
 147                                         perm_bytes, permute_mask_2,
 148                                         permute_idx, tuple_bytes_2);
 149                 }
 150
 151                 vals = _mm512_gf2p8affine_epi64_epi8(perm_bytes, matrixes, 0);
 152                 xor_acc = _mm512_xor_si512(xor_acc, vals);
 153         }
 154
 155         return xor_acc;
 156 }
 157
 158 /**
 159  * Calculate Toeplitz hash.
 160  *
 161  * @warning
 162  * @b EXPERIMENTAL: this API may change without prior notice.
 163  *
 164  * @param m
 165  *  Pointer to the matrices generated from the corresponding
 166  *  RSS hash key using rte_thash_complete_matrix().
 167  *  Note that @p len should not exceed the length of the rss_key minus 4.
 168  * @param tuple
 169  *  Pointer to the data to be hashed. Data must be in network byte order.
 170  * @param len
 171  *  Length of the data to be hashed.
 172  * @return
 173  *  Calculated Toeplitz hash value.
 174  */
 175 __rte_experimental
 176 static inline uint32_t
 177 rte_thash_gfni(const uint64_t *m, const uint8_t *tuple, int len)
 178 {
 179         uint32_t val, val_zero;
 180
 181         __m512i xor_acc = __rte_thash_gfni(m, tuple, NULL, len);
 182         __rte_thash_xor_reduce(xor_acc, &val, &val_zero);
 183
 184         return val;
 185 }
 186
 187 /**
 188  * Bulk implementation for Toeplitz hash.
 189  *
 190  * @warning
 191  * @b EXPERIMENTAL: this API may change without prior notice.
 192  *
 193  * @param m
 194  *  Pointer to the matrices generated from the corresponding
 195  *  RSS hash key using rte_thash_complete_matrix().
 196  *  Note that @p len should not exceed the length of the rss_key minus 4.
 197  * @param len
 198  *  Length of the largest data buffer to be hashed.
 199  * @param tuple
 200  *  Array of the pointers on data to be hashed.
 201  *  Data must be in network byte order.
 202  * @param val
 203  *  Array of uint32_t where to put calculated Toeplitz hash values
 204  * @param num
 205  *  Number of tuples to hash.
 206  */
 207 __rte_experimental
 208 static inline void
 209 rte_thash_gfni_bulk(const uint64_t *mtrx, int len, uint8_t *tuple[],
 210         uint32_t val[], uint32_t num)
 211 {
 212         uint32_t i;
 213         uint32_t val_zero;
 214         __m512i xor_acc;
 215
 216         for (i = 0; i != (num & ~1); i += 2) {
 217                 xor_acc = __rte_thash_gfni(mtrx, tuple[i], tuple[i + 1], len);
 218                 __rte_thash_xor_reduce(xor_acc, val + i, val + i + 1);
 219         }
 220
 221         if (num & 1) {
 222                 xor_acc = __rte_thash_gfni(mtrx, tuple[i], NULL, len);
 223                 __rte_thash_xor_reduce(xor_acc, val + i, &val_zero);
 224         }
 225 }
 226
 227 #endif /* __GFNI__ && __AVX512F__ */
 228
 229 #ifdef __cplusplus
 230 }
 231 #endif
 232
 233 #endif /* _RTE_THASH_X86_GFNI_H_ */