lib/hash/rte_thash_x86_gfni.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2021 Intel Corporation
   3  */
   4
   5 #ifndef _RTE_THASH_X86_GFNI_H_
   6 #define _RTE_THASH_X86_GFNI_H_
   7
   8 /**
   9  * @file
  10  *
  11  * Optimized Toeplitz hash functions implementation
  12  * using Galois Fields New Instructions.
  13  */
  14
  15 #include <rte_vect.h>
  16
  17 #ifdef __cplusplus
  18 extern "C" {
  19 #endif
  20
  21 #ifdef __GFNI__
  22 #define RTE_THASH_GFNI_DEFINED
  23
  24 #define RTE_THASH_FIRST_ITER_MSK        0x0f0f0f0f0f0e0c08
  25 #define RTE_THASH_PERM_MSK              0x0f0f0f0f0f0f0f0f
  26 #define RTE_THASH_FIRST_ITER_MSK_2      0xf0f0f0f0f0e0c080
  27 #define RTE_THASH_PERM_MSK_2            0xf0f0f0f0f0f0f0f0
  28 #define RTE_THASH_REWIND_MSK            0x0000000000113377
  29
  30 __rte_internal
  31 static inline void
  32 __rte_thash_xor_reduce(__m512i xor_acc, uint32_t *val_1, uint32_t *val_2)
  33 {
  34         __m256i tmp_256_1, tmp_256_2;
  35         __m128i tmp128_1, tmp128_2;
  36         uint64_t tmp_1, tmp_2;
  37
  38         tmp_256_1 = _mm512_castsi512_si256(xor_acc);
  39         tmp_256_2 = _mm512_extracti32x8_epi32(xor_acc, 1);
  40         tmp_256_1 = _mm256_xor_si256(tmp_256_1, tmp_256_2);
  41
  42         tmp128_1 = _mm256_castsi256_si128(tmp_256_1);
  43         tmp128_2 = _mm256_extracti32x4_epi32(tmp_256_1, 1);
  44         tmp128_1 = _mm_xor_si128(tmp128_1, tmp128_2);
  45
  46         tmp_1 = _mm_extract_epi64(tmp128_1, 0);
  47         tmp_2 = _mm_extract_epi64(tmp128_1, 1);
  48         tmp_1 ^= tmp_2;
  49
  50         *val_1 = (uint32_t)tmp_1;
  51         *val_2 = (uint32_t)(tmp_1 >> 32);
  52 }
  53
  54 __rte_internal
  55 static inline __m512i
  56 __rte_thash_gfni(const uint64_t *mtrx, const uint8_t *tuple,
  57         const uint8_t *secondary_tuple, int len)
  58 {
  59         __m512i permute_idx = _mm512_set_epi8(7, 6, 5, 4, 7, 6, 5, 4,
  60                                                 6, 5, 4, 3, 6, 5, 4, 3,
  61                                                 5, 4, 3, 2, 5, 4, 3, 2,
  62                                                 4, 3, 2, 1, 4, 3, 2, 1,
  63                                                 3, 2, 1, 0, 3, 2, 1, 0,
  64                                                 2, 1, 0, -1, 2, 1, 0, -1,
  65                                                 1, 0, -1, -2, 1, 0, -1, -2,
  66                                                 0, -1, -2, -3, 0, -1, -2, -3);
  67
  68         const __m512i rewind_idx = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,
  69                                                 0, 0, 0, 0, 0, 0, 0, 0,
  70                                                 0, 0, 0, 0, 0, 0, 0, 0,
  71                                                 0, 0, 0, 0, 0, 0, 0, 0,
  72                                                 0, 0, 0, 0, 0, 0, 0, 0,
  73                                                 0, 0, 0, 59, 0, 0, 0, 59,
  74                                                 0, 0, 59, 58, 0, 0, 59, 58,
  75                                                 0, 59, 58, 57, 0, 59, 58, 57);
  76         const __mmask64 rewind_mask = RTE_THASH_REWIND_MSK;
  77         const __m512i shift_8 = _mm512_set1_epi8(8);
  78         __m512i xor_acc = _mm512_setzero_si512();
  79         __m512i perm_bytes = _mm512_setzero_si512();
  80         __m512i vals, matrixes, tuple_bytes, tuple_bytes_2;
  81         __mmask64 load_mask, permute_mask, permute_mask_2;
  82         int chunk_len = 0, i = 0;
  83         uint8_t mtrx_msk;
  84         const int prepend = 3;
  85
  86         for (; len > 0; len -= 64, tuple += 64) {
  87                 if (i == 8)
  88                         perm_bytes = _mm512_maskz_permutexvar_epi8(rewind_mask,
  89                                 rewind_idx, perm_bytes);
  90
  91                 permute_mask = RTE_THASH_FIRST_ITER_MSK;
  92                 load_mask = (len >= 64) ? UINT64_MAX : ((1ULL << len) - 1);
  93                 tuple_bytes = _mm512_maskz_loadu_epi8(load_mask, tuple);
  94                 if (secondary_tuple) {
  95                         permute_mask_2 = RTE_THASH_FIRST_ITER_MSK_2;
  96                         tuple_bytes_2 = _mm512_maskz_loadu_epi8(load_mask,
  97                                 secondary_tuple);
  98                 }
  99
 100                 chunk_len = __builtin_popcountll(load_mask);
 101                 for (i = 0; i < ((chunk_len + prepend) / 8); i++, mtrx += 8) {
 102                         perm_bytes = _mm512_mask_permutexvar_epi8(perm_bytes,
 103                                 permute_mask, permute_idx, tuple_bytes);
 104
 105                         if (secondary_tuple)
 106                                 perm_bytes =
 107                                         _mm512_mask_permutexvar_epi8(perm_bytes,
 108                                         permute_mask_2, permute_idx,
 109                                         tuple_bytes_2);
 110
 111                         matrixes = _mm512_maskz_loadu_epi64(UINT8_MAX, mtrx);
 112                         vals = _mm512_gf2p8affine_epi64_epi8(perm_bytes,
 113                                 matrixes, 0);
 114
 115                         xor_acc = _mm512_xor_si512(xor_acc, vals);
 116                         permute_idx = _mm512_add_epi8(permute_idx, shift_8);
 117                         permute_mask = RTE_THASH_PERM_MSK;
 118                         if (secondary_tuple)
 119                                 permute_mask_2 = RTE_THASH_PERM_MSK_2;
 120                 }
 121         }
 122
 123         int rest_len = (chunk_len + prepend) % 8;
 124         if (rest_len != 0) {
 125                 mtrx_msk = (1 << (rest_len % 8)) - 1;
 126                 matrixes = _mm512_maskz_loadu_epi64(mtrx_msk, mtrx);
 127                 if (i == 8) {
 128                         perm_bytes = _mm512_maskz_permutexvar_epi8(rewind_mask,
 129                                 rewind_idx, perm_bytes);
 130                 } else {
 131                         perm_bytes = _mm512_mask_permutexvar_epi8(perm_bytes,
 132                                 permute_mask, permute_idx, tuple_bytes);
 133
 134                         if (secondary_tuple)
 135                                 perm_bytes =
 136                                         _mm512_mask_permutexvar_epi8(
 137                                         perm_bytes, permute_mask_2,
 138                                         permute_idx, tuple_bytes_2);
 139                 }
 140
 141                 vals = _mm512_gf2p8affine_epi64_epi8(perm_bytes, matrixes, 0);
 142                 xor_acc = _mm512_xor_si512(xor_acc, vals);
 143         }
 144
 145         return xor_acc;
 146 }
 147
 148 /**
 149  * Calculate Toeplitz hash.
 150  *
 151  * @warning
 152  * @b EXPERIMENTAL: this API may change without prior notice.
 153  *
 154  * @param m
 155  *  Pointer to the matrices generated from the corresponding
 156  *  RSS hash key using rte_thash_complete_matrix().
 157  *  Note that @p len should not exceed the length of the rss_key minus 4.
 158  * @param tuple
 159  *  Pointer to the data to be hashed. Data must be in network byte order.
 160  * @param len
 161  *  Length of the data to be hashed.
 162  * @return
 163  *  Calculated Toeplitz hash value.
 164  */
 165 __rte_experimental
 166 static inline uint32_t
 167 rte_thash_gfni(const uint64_t *m, const uint8_t *tuple, int len)
 168 {
 169         uint32_t val, val_zero;
 170
 171         __m512i xor_acc = __rte_thash_gfni(m, tuple, NULL, len);
 172         __rte_thash_xor_reduce(xor_acc, &val, &val_zero);
 173
 174         return val;
 175 }
 176
 177 /**
 178  * Bulk implementation for Toeplitz hash.
 179  *
 180  * @warning
 181  * @b EXPERIMENTAL: this API may change without prior notice.
 182  *
 183  * @param m
 184  *  Pointer to the matrices generated from the corresponding
 185  *  RSS hash key using rte_thash_complete_matrix().
 186  *  Note that @p len should not exceed the length of the rss_key minus 4.
 187  * @param len
 188  *  Length of the largest data buffer to be hashed.
 189  * @param tuple
 190  *  Array of the pointers on data to be hashed.
 191  *  Data must be in network byte order.
 192  * @param val
 193  *  Array of uint32_t where to put calculated Toeplitz hash values
 194  * @param num
 195  *  Number of tuples to hash.
 196  */
 197 __rte_experimental
 198 static inline void
 199 rte_thash_gfni_bulk(const uint64_t *mtrx, int len, uint8_t *tuple[],
 200         uint32_t val[], uint32_t num)
 201 {
 202         uint32_t i;
 203         uint32_t val_zero;
 204         __m512i xor_acc;
 205
 206         for (i = 0; i != (num & ~1); i += 2) {
 207                 xor_acc = __rte_thash_gfni(mtrx, tuple[i], tuple[i + 1], len);
 208                 __rte_thash_xor_reduce(xor_acc, val + i, val + i + 1);
 209         }
 210
 211         if (num & 1) {
 212                 xor_acc = __rte_thash_gfni(mtrx, tuple[i], NULL, len);
 213                 __rte_thash_xor_reduce(xor_acc, val + i, &val_zero);
 214         }
 215 }
 216
 217 #endif /* _GFNI_ */
 218
 219 #ifdef __cplusplus
 220 }
 221 #endif
 222
 223 #endif /* _RTE_THASH_X86_GFNI_H_ */