1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2019 Vladimir Medvedkin <medvedkinv@gmail.com>
3 * Copyright(c) 2021 Intel Corporation
12 * Software implementation of the Toeplitz hash function used by RSS.
13 * Can be used either for packet distribution on single queue NIC
14 * or for simulating of RSS computation on specific NIC (for example
15 * after GRE header decapsulating)
23 #include <rte_byteorder.h>
24 #include <rte_config.h>
26 #include <rte_common.h>
27 #include <rte_thash_gfni.h>
29 #if defined(RTE_ARCH_X86) || defined(__ARM_NEON)
34 /* Byte swap mask used for converting IPv6 address
35 * 4-byte chunks to CPU byte order
37 static const __m128i rte_thash_ipv6_bswap_mask = {
38 0x0405060700010203ULL, 0x0C0D0E0F08090A0BULL};
42 * length in dwords of input tuple to
43 * calculate hash of ipv4 header only
45 #define RTE_THASH_V4_L3_LEN ((sizeof(struct rte_ipv4_tuple) - \
46 sizeof(((struct rte_ipv4_tuple *)0)->sctp_tag)) / 4)
49 * length in dwords of input tuple to
50 * calculate hash of ipv4 header +
53 #define RTE_THASH_V4_L4_LEN ((sizeof(struct rte_ipv4_tuple)) / 4)
56 * length in dwords of input tuple to
57 * calculate hash of ipv6 header only
59 #define RTE_THASH_V6_L3_LEN ((sizeof(struct rte_ipv6_tuple) - \
60 sizeof(((struct rte_ipv6_tuple *)0)->sctp_tag)) / 4)
63 * length in dwords of input tuple to
64 * calculate hash of ipv6 header +
67 #define RTE_THASH_V6_L4_LEN ((sizeof(struct rte_ipv6_tuple)) / 4)
71 * addresses and ports/sctp_tag have to be CPU byte order
73 struct rte_ipv4_tuple {
88 * Addresses have to be filled by rte_thash_load_v6_addr()
89 * ports/sctp_tag have to be CPU byte order
91 struct rte_ipv6_tuple {
104 union rte_thash_tuple {
105 struct rte_ipv4_tuple v4;
106 struct rte_ipv6_tuple v6;
108 } __rte_aligned(XMM_SIZE);
114 * Prepare special converted key to use with rte_softrss_be()
116 * pointer to original RSS key
118 * pointer to target RSS key
123 rte_convert_rss_key(const uint32_t *orig, uint32_t *targ, int len)
127 for (i = 0; i < (len >> 2); i++)
128 targ[i] = rte_be_to_cpu_32(orig[i]);
132 * Prepare and load IPv6 addresses (src and dst)
135 * Pointer to ipv6 header of the original packet
137 * Pointer to rte_ipv6_tuple structure
140 rte_thash_load_v6_addrs(const struct rte_ipv6_hdr *orig,
141 union rte_thash_tuple *targ)
144 __m128i ipv6 = _mm_loadu_si128((const __m128i *)orig->src_addr);
145 *(__m128i *)targ->v6.src_addr =
146 _mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
147 ipv6 = _mm_loadu_si128((const __m128i *)orig->dst_addr);
148 *(__m128i *)targ->v6.dst_addr =
149 _mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
150 #elif defined(__ARM_NEON)
151 uint8x16_t ipv6 = vld1q_u8((uint8_t const *)orig->src_addr);
152 vst1q_u8((uint8_t *)targ->v6.src_addr, vrev32q_u8(ipv6));
153 ipv6 = vld1q_u8((uint8_t const *)orig->dst_addr);
154 vst1q_u8((uint8_t *)targ->v6.dst_addr, vrev32q_u8(ipv6));
157 for (i = 0; i < 4; i++) {
158 *((uint32_t *)targ->v6.src_addr + i) =
159 rte_be_to_cpu_32(*((const uint32_t *)orig->src_addr + i));
160 *((uint32_t *)targ->v6.dst_addr + i) =
161 rte_be_to_cpu_32(*((const uint32_t *)orig->dst_addr + i));
167 * Generic implementation. Can be used with original rss_key
169 * Pointer to input tuple
171 * Length of input_tuple in 4-bytes chunks
173 * Pointer to RSS hash key.
175 * Calculated hash value.
177 static inline uint32_t
178 rte_softrss(uint32_t *input_tuple, uint32_t input_len,
179 const uint8_t *rss_key)
181 uint32_t i, j, map, ret = 0;
183 for (j = 0; j < input_len; j++) {
184 for (map = input_tuple[j]; map; map &= (map - 1)) {
186 ret ^= rte_cpu_to_be_32(((const uint32_t *)rss_key)[j]) << (31 - i) |
187 (uint32_t)((uint64_t)(rte_cpu_to_be_32(((const uint32_t *)rss_key)[j + 1])) >>
195 * Optimized implementation.
196 * If you want the calculated hash value matches NIC RSS value
197 * you have to use special converted key with rte_convert_rss_key() fn.
199 * Pointer to input tuple
201 * Length of input_tuple in 4-bytes chunks
203 * Pointer to RSS hash key.
205 * Calculated hash value.
207 static inline uint32_t
208 rte_softrss_be(uint32_t *input_tuple, uint32_t input_len,
209 const uint8_t *rss_key)
211 uint32_t i, j, map, ret = 0;
213 for (j = 0; j < input_len; j++) {
214 for (map = input_tuple[j]; map; map &= (map - 1)) {
216 ret ^= ((const uint32_t *)rss_key)[j] << (31 - i) |
217 (uint32_t)((uint64_t)(((const uint32_t *)rss_key)[j + 1]) >> (i + 1));
224 * Indicates if GFNI implementations of the Toeplitz hash are supported.
227 * @b EXPERIMENTAL: this API may change without prior notice.
230 * 1 if GFNI is supported
235 rte_thash_gfni_supported(void);
238 * Converts Toeplitz hash key (RSS key) into matrixes required
239 * for GFNI implementation
242 * @b EXPERIMENTAL: this API may change without prior notice.
245 * pointer to the memory where matrices will be written.
246 * Note: the size of this memory must be equal to size * 8
248 * pointer to the Toeplitz hash key
250 * Size of the rss_key in bytes.
254 rte_thash_complete_matrix(uint64_t *matrixes, const uint8_t *rss_key,
257 /** @internal Logarithm of minimum size of the RSS ReTa */
258 #define RTE_THASH_RETA_SZ_MIN 2U
259 /** @internal Logarithm of maximum size of the RSS ReTa */
260 #define RTE_THASH_RETA_SZ_MAX 16U
263 * LFSR will ignore if generated m-sequence has more than 2^n -1 bits,
264 * where n is the logarithm of the RSS ReTa size.
266 #define RTE_THASH_IGNORE_PERIOD_OVERFLOW 0x1
268 * Generate minimal required bit (equal to ReTa LSB) sequence into
271 #define RTE_THASH_MINIMAL_SEQ 0x2
273 /** @internal thash context structure. */
274 struct rte_thash_ctx;
275 /** @internal thash helper structure. */
276 struct rte_thash_subtuple_helper;
279 * Create a new thash context.
282 * @b EXPERIMENTAL: this API may change without prior notice.
287 * Length of the toeplitz hash key
289 * Logarithm of the NIC's Redirection Table (ReTa) size,
290 * i.e. number of the LSBs if the hash used to determine
293 * Pointer to the key used to init an internal key state.
294 * Could be NULL, in this case internal key will be inited with random.
296 * Supported flags are:
297 * RTE_THASH_IGNORE_PERIOD_OVERFLOW
298 * RTE_THASH_MINIMAL_SEQ
300 * A pointer to the created context on success
304 struct rte_thash_ctx *
305 rte_thash_init_ctx(const char *name, uint32_t key_len, uint32_t reta_sz,
306 uint8_t *key, uint32_t flags);
309 * Find an existing thash context and return a pointer to it.
312 * @b EXPERIMENTAL: this API may change without prior notice.
315 * Name of the thash context
317 * Pointer to the thash context or NULL if it was not found with rte_errno
318 * set appropriately. Possible rte_errno values include:
319 * - ENOENT - required entry not available to return.
322 struct rte_thash_ctx *
323 rte_thash_find_existing(const char *name);
326 * Free a thash context object
329 * @b EXPERIMENTAL: this API may change without prior notice.
338 rte_thash_free_ctx(struct rte_thash_ctx *ctx);
341 * Add a special properties to the toeplitz hash key inside a thash context.
342 * Creates an internal helper struct which has a complementary table
343 * to calculate toeplitz hash collisions.
344 * This function is not multi-thread safe.
347 * @b EXPERIMENTAL: this API may change without prior notice.
354 * Length in bits of the target subtuple
355 * Must be no shorter than reta_sz passed on rte_thash_init_ctx().
357 * Offset in bits of the subtuple
364 rte_thash_add_helper(struct rte_thash_ctx *ctx, const char *name, uint32_t len,
368 * Find a helper in the context by the given name
371 * @b EXPERIMENTAL: this API may change without prior notice.
378 * Pointer to the thash helper or NULL if it was not found.
381 struct rte_thash_subtuple_helper *
382 rte_thash_get_helper(struct rte_thash_ctx *ctx, const char *name);
385 * Get a complementary value for the subtuple to produce a
386 * partial toeplitz hash collision. It must be XOR'ed with the
387 * subtuple to produce the hash value with the desired hash LSB's
388 * This function is multi-thread safe.
391 * Pointer to the helper struct
393 * Toeplitz hash value calculated for the given tuple
394 * @param desired_hash
395 * Desired hash value to find a collision for
397 * A complementary value which must be xored with the corresponding subtuple
401 rte_thash_get_complement(struct rte_thash_subtuple_helper *h,
402 uint32_t hash, uint32_t desired_hash);
405 * Get a pointer to the toeplitz hash contained in the context.
406 * It changes after each addition of a helper. It should be installed to
410 * @b EXPERIMENTAL: this API may change without prior notice.
415 * A pointer to the toeplitz hash key
419 rte_thash_get_key(struct rte_thash_ctx *ctx);
422 * Get a pointer to the toeplitz hash matrices contained in the context.
423 * These matrices could be used with fast toeplitz hash implementation if
425 * Matrices changes after each addition of a helper.
428 * @b EXPERIMENTAL: this API may change without prior notice.
433 * A pointer to the toeplitz hash key matrices on success
434 * NULL if GFNI is not supported.
438 rte_thash_get_gfni_matrices(struct rte_thash_ctx *ctx);
441 * Function prototype for the rte_thash_adjust_tuple
442 * to check if adjusted tuple could be used.
443 * Generally it is some kind of lookup function to check
444 * if adjusted tuple is already in use.
447 * @b EXPERIMENTAL: this API may change without prior notice.
450 * Pointer to the userdata. It could be a pointer to the
451 * table with used tuples to search.
453 * Pointer to the tuple to check
459 typedef int (*rte_thash_check_tuple_t)(void *userdata, uint8_t *tuple);
462 * Adjusts tuple in the way to make Toeplitz hash has
463 * desired least significant bits.
464 * This function is multi-thread safe.
467 * @b EXPERIMENTAL: this API may change without prior notice.
472 * Pointer to the helper struct
474 * Pointer to the tuple to be adjusted
476 * Length of the tuple. Must be multiple of 4.
477 * @param desired_value
478 * Desired value of least significant bits of the hash
480 * Number of attempts to adjust tuple with fn() calling
482 * Callback function to check adjusted tuple. Could be NULL
484 * Pointer to the userdata to be passed to fn(). Could be NULL
492 rte_thash_adjust_tuple(struct rte_thash_ctx *ctx,
493 struct rte_thash_subtuple_helper *h,
494 uint8_t *tuple, unsigned int tuple_len,
495 uint32_t desired_value, unsigned int attempts,
496 rte_thash_check_tuple_t fn, void *userdata);
502 #endif /* _RTE_THASH_H */