1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
11 #include <sys/queue.h>
16 #include <netinet/in.h>
18 #include <rte_debug.h>
19 #include <rte_ether.h>
20 #include <rte_ethdev.h>
21 #include <rte_cycles.h>
29 #include "l3fwd_event.h"
31 #if defined(RTE_ARCH_X86) || defined(__ARM_FEATURE_CRC32)
36 #include <rte_hash_crc.h>
37 #define DEFAULT_HASH_FUNC rte_hash_crc
39 #include <rte_jhash.h>
40 #define DEFAULT_HASH_FUNC rte_jhash
43 #define IPV6_ADDR_LEN 16
53 union ipv4_5tuple_host {
66 #define XMM_NUM_IN_IPV6_5TUPLE 3
69 uint8_t ip_dst[IPV6_ADDR_LEN];
70 uint8_t ip_src[IPV6_ADDR_LEN];
76 union ipv6_5tuple_host {
81 uint8_t ip_src[IPV6_ADDR_LEN];
82 uint8_t ip_dst[IPV6_ADDR_LEN];
87 xmm_t xmm[XMM_NUM_IN_IPV6_5TUPLE];
92 struct ipv4_l3fwd_em_route {
93 struct ipv4_5tuple key;
97 struct ipv6_l3fwd_em_route {
98 struct ipv6_5tuple key;
102 /* 198.18.0.0/16 are set aside for RFC2544 benchmarking (RFC5735).
103 * Use RFC863 Discard Protocol.
105 static const struct ipv4_l3fwd_em_route ipv4_l3fwd_em_route_array[] = {
106 {{RTE_IPV4(198, 18, 0, 0), RTE_IPV4(198, 18, 0, 1), 9, 9, IPPROTO_UDP}, 0},
107 {{RTE_IPV4(198, 18, 1, 0), RTE_IPV4(198, 18, 1, 1), 9, 9, IPPROTO_UDP}, 1},
108 {{RTE_IPV4(198, 18, 2, 0), RTE_IPV4(198, 18, 2, 1), 9, 9, IPPROTO_UDP}, 2},
109 {{RTE_IPV4(198, 18, 3, 0), RTE_IPV4(198, 18, 3, 1), 9, 9, IPPROTO_UDP}, 3},
110 {{RTE_IPV4(198, 18, 4, 0), RTE_IPV4(198, 18, 4, 1), 9, 9, IPPROTO_UDP}, 4},
111 {{RTE_IPV4(198, 18, 5, 0), RTE_IPV4(198, 18, 5, 1), 9, 9, IPPROTO_UDP}, 5},
112 {{RTE_IPV4(198, 18, 6, 0), RTE_IPV4(198, 18, 6, 1), 9, 9, IPPROTO_UDP}, 6},
113 {{RTE_IPV4(198, 18, 7, 0), RTE_IPV4(198, 18, 7, 1), 9, 9, IPPROTO_UDP}, 7},
114 {{RTE_IPV4(198, 18, 8, 0), RTE_IPV4(198, 18, 8, 1), 9, 9, IPPROTO_UDP}, 8},
115 {{RTE_IPV4(198, 18, 9, 0), RTE_IPV4(198, 18, 9, 1), 9, 9, IPPROTO_UDP}, 9},
116 {{RTE_IPV4(198, 18, 10, 0), RTE_IPV4(198, 18, 10, 1), 9, 9, IPPROTO_UDP}, 10},
117 {{RTE_IPV4(198, 18, 11, 0), RTE_IPV4(198, 18, 11, 1), 9, 9, IPPROTO_UDP}, 11},
118 {{RTE_IPV4(198, 18, 12, 0), RTE_IPV4(198, 18, 12, 1), 9, 9, IPPROTO_UDP}, 12},
119 {{RTE_IPV4(198, 18, 13, 0), RTE_IPV4(198, 18, 13, 1), 9, 9, IPPROTO_UDP}, 13},
120 {{RTE_IPV4(198, 18, 14, 0), RTE_IPV4(198, 18, 14, 1), 9, 9, IPPROTO_UDP}, 14},
121 {{RTE_IPV4(198, 18, 15, 0), RTE_IPV4(198, 18, 15, 1), 9, 9, IPPROTO_UDP}, 15},
124 /* 2001:0200::/48 is IANA reserved range for IPv6 benchmarking (RFC5180).
125 * Use RFC863 Discard Protocol.
127 static const struct ipv6_l3fwd_em_route ipv6_l3fwd_em_route_array[] = {
128 {{{32, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
129 {32, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 0},
130 {{{32, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0},
131 {32, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 1},
132 {{{32, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0},
133 {32, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 2},
134 {{{32, 1, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0},
135 {32, 1, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 3},
136 {{{32, 1, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0},
137 {32, 1, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 4},
138 {{{32, 1, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0},
139 {32, 1, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 5},
140 {{{32, 1, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0},
141 {32, 1, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 6},
142 {{{32, 1, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0},
143 {32, 1, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 7},
144 {{{32, 1, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0},
145 {32, 1, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 8},
146 {{{32, 1, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0},
147 {32, 1, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 9},
148 {{{32, 1, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0},
149 {32, 1, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 10},
150 {{{32, 1, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0},
151 {32, 1, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 11},
152 {{{32, 1, 2, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0},
153 {32, 1, 2, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 12},
154 {{{32, 1, 2, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0},
155 {32, 1, 2, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 13},
156 {{{32, 1, 2, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0},
157 {32, 1, 2, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 14},
158 {{{32, 1, 2, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0},
159 {32, 1, 2, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 1}, 9, 9, IPPROTO_UDP}, 15},
162 struct rte_hash *ipv4_l3fwd_em_lookup_struct[NB_SOCKETS];
163 struct rte_hash *ipv6_l3fwd_em_lookup_struct[NB_SOCKETS];
165 static inline uint32_t
166 ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len,
169 const union ipv4_5tuple_host *k;
175 p = (const uint32_t *)&k->port_src;
178 init_val = rte_hash_crc_4byte(t, init_val);
179 init_val = rte_hash_crc_4byte(k->ip_src, init_val);
180 init_val = rte_hash_crc_4byte(k->ip_dst, init_val);
181 init_val = rte_hash_crc_4byte(*p, init_val);
183 init_val = rte_jhash_1word(t, init_val);
184 init_val = rte_jhash_1word(k->ip_src, init_val);
185 init_val = rte_jhash_1word(k->ip_dst, init_val);
186 init_val = rte_jhash_1word(*p, init_val);
192 static inline uint32_t
193 ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len,
196 const union ipv6_5tuple_host *k;
200 const uint32_t *ip_src0, *ip_src1, *ip_src2, *ip_src3;
201 const uint32_t *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3;
206 p = (const uint32_t *)&k->port_src;
209 ip_src0 = (const uint32_t *) k->ip_src;
210 ip_src1 = (const uint32_t *)(k->ip_src+4);
211 ip_src2 = (const uint32_t *)(k->ip_src+8);
212 ip_src3 = (const uint32_t *)(k->ip_src+12);
213 ip_dst0 = (const uint32_t *) k->ip_dst;
214 ip_dst1 = (const uint32_t *)(k->ip_dst+4);
215 ip_dst2 = (const uint32_t *)(k->ip_dst+8);
216 ip_dst3 = (const uint32_t *)(k->ip_dst+12);
217 init_val = rte_hash_crc_4byte(t, init_val);
218 init_val = rte_hash_crc_4byte(*ip_src0, init_val);
219 init_val = rte_hash_crc_4byte(*ip_src1, init_val);
220 init_val = rte_hash_crc_4byte(*ip_src2, init_val);
221 init_val = rte_hash_crc_4byte(*ip_src3, init_val);
222 init_val = rte_hash_crc_4byte(*ip_dst0, init_val);
223 init_val = rte_hash_crc_4byte(*ip_dst1, init_val);
224 init_val = rte_hash_crc_4byte(*ip_dst2, init_val);
225 init_val = rte_hash_crc_4byte(*ip_dst3, init_val);
226 init_val = rte_hash_crc_4byte(*p, init_val);
228 init_val = rte_jhash_1word(t, init_val);
229 init_val = rte_jhash(k->ip_src,
230 sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
231 init_val = rte_jhash(k->ip_dst,
232 sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
233 init_val = rte_jhash_1word(*p, init_val);
238 #define IPV4_L3FWD_EM_NUM_ROUTES RTE_DIM(ipv4_l3fwd_em_route_array)
240 #define IPV6_L3FWD_EM_NUM_ROUTES RTE_DIM(ipv6_l3fwd_em_route_array)
242 static uint8_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
243 static uint8_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
245 static rte_xmm_t mask0;
246 static rte_xmm_t mask1;
247 static rte_xmm_t mask2;
249 #if defined(__SSE2__)
251 em_mask_key(void *key, xmm_t mask)
253 __m128i data = _mm_loadu_si128((__m128i *)(key));
255 return _mm_and_si128(data, mask);
257 #elif defined(__ARM_NEON)
259 em_mask_key(void *key, xmm_t mask)
261 int32x4_t data = vld1q_s32((int32_t *)key);
263 return vandq_s32(data, mask);
265 #elif defined(__ALTIVEC__)
267 em_mask_key(void *key, xmm_t mask)
269 xmm_t data = vec_ld(0, (xmm_t *)(key));
271 return vec_and(data, mask);
274 #error No vector engine (SSE, NEON, ALTIVEC) available, check your toolchain
277 /* Performing hash-based lookups. 8< */
278 static inline uint16_t
279 em_get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, void *lookup_struct)
282 union ipv4_5tuple_host key;
283 struct rte_hash *ipv4_l3fwd_lookup_struct =
284 (struct rte_hash *)lookup_struct;
286 ipv4_hdr = (uint8_t *)ipv4_hdr +
287 offsetof(struct rte_ipv4_hdr, time_to_live);
290 * Get 5 tuple: dst port, src port, dst IP address,
291 * src IP address and protocol.
293 key.xmm = em_mask_key(ipv4_hdr, mask0.x);
295 /* Find destination port */
296 ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key);
297 return (ret < 0) ? portid : ipv4_l3fwd_out_if[ret];
299 /* >8 End of performing hash-based lookups. */
301 static inline uint16_t
302 em_get_ipv6_dst_port(void *ipv6_hdr, uint16_t portid, void *lookup_struct)
305 union ipv6_5tuple_host key;
306 struct rte_hash *ipv6_l3fwd_lookup_struct =
307 (struct rte_hash *)lookup_struct;
309 ipv6_hdr = (uint8_t *)ipv6_hdr +
310 offsetof(struct rte_ipv6_hdr, payload_len);
311 void *data0 = ipv6_hdr;
312 void *data1 = ((uint8_t *)ipv6_hdr) + sizeof(xmm_t);
313 void *data2 = ((uint8_t *)ipv6_hdr) + sizeof(xmm_t) + sizeof(xmm_t);
315 /* Get part of 5 tuple: src IP address lower 96 bits and protocol */
316 key.xmm[0] = em_mask_key(data0, mask1.x);
319 * Get part of 5 tuple: dst IP address lower 96 bits
320 * and src IP address higher 32 bits.
322 #if defined RTE_ARCH_X86
323 key.xmm[1] = _mm_loadu_si128(data1);
325 key.xmm[1] = *(xmm_t *)data1;
329 * Get part of 5 tuple: dst port and src port
330 * and dst IP address higher 32 bits.
332 key.xmm[2] = em_mask_key(data2, mask2.x);
334 /* Find destination port */
335 ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key);
336 return (ret < 0) ? portid : ipv6_l3fwd_out_if[ret];
339 #if defined RTE_ARCH_X86 || defined __ARM_NEON
340 #if defined(NO_HASH_MULTI_LOOKUP)
341 #include "l3fwd_em_sequential.h"
343 #include "l3fwd_em_hlm.h"
346 #include "l3fwd_em.h"
350 convert_ipv4_5tuple(struct ipv4_5tuple *key1,
351 union ipv4_5tuple_host *key2)
353 key2->ip_dst = rte_cpu_to_be_32(key1->ip_dst);
354 key2->ip_src = rte_cpu_to_be_32(key1->ip_src);
355 key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
356 key2->port_src = rte_cpu_to_be_16(key1->port_src);
357 key2->proto = key1->proto;
363 convert_ipv6_5tuple(struct ipv6_5tuple *key1,
364 union ipv6_5tuple_host *key2)
368 for (i = 0; i < 16; i++) {
369 key2->ip_dst[i] = key1->ip_dst[i];
370 key2->ip_src[i] = key1->ip_src[i];
372 key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
373 key2->port_src = rte_cpu_to_be_16(key1->port_src);
374 key2->proto = key1->proto;
380 #define BYTE_VALUE_MAX 256
381 #define ALL_32_BITS 0xffffffff
382 #define BIT_8_TO_15 0x0000ff00
385 populate_ipv4_few_flow_into_table(const struct rte_hash *h)
390 mask0 = (rte_xmm_t){.u32 = {BIT_8_TO_15, ALL_32_BITS,
391 ALL_32_BITS, ALL_32_BITS} };
393 for (i = 0; i < IPV4_L3FWD_EM_NUM_ROUTES; i++) {
394 struct ipv4_l3fwd_em_route entry;
395 union ipv4_5tuple_host newkey;
397 entry = ipv4_l3fwd_em_route_array[i];
398 convert_ipv4_5tuple(&entry.key, &newkey);
399 ret = rte_hash_add_key(h, (void *) &newkey);
401 rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
402 " to the l3fwd hash.\n", i);
404 ipv4_l3fwd_out_if[ret] = entry.if_out;
406 printf("Hash: Adding 0x%" PRIx64 " keys\n",
407 (uint64_t)IPV4_L3FWD_EM_NUM_ROUTES);
410 #define BIT_16_TO_23 0x00ff0000
412 populate_ipv6_few_flow_into_table(const struct rte_hash *h)
417 mask1 = (rte_xmm_t){.u32 = {BIT_16_TO_23, ALL_32_BITS,
418 ALL_32_BITS, ALL_32_BITS} };
420 mask2 = (rte_xmm_t){.u32 = {ALL_32_BITS, ALL_32_BITS, 0, 0} };
422 for (i = 0; i < IPV6_L3FWD_EM_NUM_ROUTES; i++) {
423 struct ipv6_l3fwd_em_route entry;
424 union ipv6_5tuple_host newkey;
426 entry = ipv6_l3fwd_em_route_array[i];
427 convert_ipv6_5tuple(&entry.key, &newkey);
428 ret = rte_hash_add_key(h, (void *) &newkey);
430 rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
431 " to the l3fwd hash.\n", i);
433 ipv6_l3fwd_out_if[ret] = entry.if_out;
435 printf("Hash: Adding 0x%" PRIx64 "keys\n",
436 (uint64_t)IPV6_L3FWD_EM_NUM_ROUTES);
439 #define NUMBER_PORT_USED 16
441 populate_ipv4_many_flow_into_table(const struct rte_hash *h,
442 unsigned int nr_flow)
446 mask0 = (rte_xmm_t){.u32 = {BIT_8_TO_15, ALL_32_BITS,
447 ALL_32_BITS, ALL_32_BITS} };
449 for (i = 0; i < nr_flow; i++) {
450 uint8_t port = i % NUMBER_PORT_USED;
451 struct ipv4_l3fwd_em_route entry;
452 union ipv4_5tuple_host newkey;
454 uint8_t a = (uint8_t)((port + 1) % BYTE_VALUE_MAX);
456 /* Create the ipv4 exact match flow */
457 memset(&entry, 0, sizeof(entry));
458 entry = ipv4_l3fwd_em_route_array[port];
459 entry.key.ip_dst = RTE_IPV4(198, 18, port, a);
460 convert_ipv4_5tuple(&entry.key, &newkey);
461 int32_t ret = rte_hash_add_key(h, (void *) &newkey);
464 rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
466 ipv4_l3fwd_out_if[ret] = (uint8_t) entry.if_out;
469 printf("Hash: Adding 0x%x keys\n", nr_flow);
473 populate_ipv6_many_flow_into_table(const struct rte_hash *h,
474 unsigned int nr_flow)
478 mask1 = (rte_xmm_t){.u32 = {BIT_16_TO_23, ALL_32_BITS,
479 ALL_32_BITS, ALL_32_BITS} };
480 mask2 = (rte_xmm_t){.u32 = {ALL_32_BITS, ALL_32_BITS, 0, 0} };
482 for (i = 0; i < nr_flow; i++) {
483 uint8_t port = i % NUMBER_PORT_USED;
484 struct ipv6_l3fwd_em_route entry;
485 union ipv6_5tuple_host newkey;
487 /* Create the ipv6 exact match flow */
488 memset(&entry, 0, sizeof(entry));
489 entry = ipv6_l3fwd_em_route_array[port];
490 entry.key.ip_dst[15] = (port + 1) % BYTE_VALUE_MAX;
491 convert_ipv6_5tuple(&entry.key, &newkey);
492 int32_t ret = rte_hash_add_key(h, (void *) &newkey);
495 rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
497 ipv6_l3fwd_out_if[ret] = (uint8_t) entry.if_out;
500 printf("Hash: Adding 0x%x keys\n", nr_flow);
504 * 1. IP packets without extension;
505 * 2. L4 payload should be either TCP or UDP.
508 em_check_ptype(int portid)
511 int ptype_l3_ipv4_ext = 0;
512 int ptype_l3_ipv6_ext = 0;
513 int ptype_l4_tcp = 0;
514 int ptype_l4_udp = 0;
515 uint32_t ptype_mask = RTE_PTYPE_L3_MASK | RTE_PTYPE_L4_MASK;
517 ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, NULL, 0);
521 uint32_t ptypes[ret];
523 ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, ptypes, ret);
524 for (i = 0; i < ret; ++i) {
526 case RTE_PTYPE_L3_IPV4_EXT:
527 ptype_l3_ipv4_ext = 1;
529 case RTE_PTYPE_L3_IPV6_EXT:
530 ptype_l3_ipv6_ext = 1;
532 case RTE_PTYPE_L4_TCP:
535 case RTE_PTYPE_L4_UDP:
541 if (ptype_l3_ipv4_ext == 0)
542 printf("port %d cannot parse RTE_PTYPE_L3_IPV4_EXT\n", portid);
543 if (ptype_l3_ipv6_ext == 0)
544 printf("port %d cannot parse RTE_PTYPE_L3_IPV6_EXT\n", portid);
545 if (!ptype_l3_ipv4_ext || !ptype_l3_ipv6_ext)
548 if (ptype_l4_tcp == 0)
549 printf("port %d cannot parse RTE_PTYPE_L4_TCP\n", portid);
550 if (ptype_l4_udp == 0)
551 printf("port %d cannot parse RTE_PTYPE_L4_UDP\n", portid);
552 if (ptype_l4_tcp && ptype_l4_udp)
559 em_parse_ptype(struct rte_mbuf *m)
561 struct rte_ether_hdr *eth_hdr;
562 uint32_t packet_type = RTE_PTYPE_UNKNOWN;
566 struct rte_ipv4_hdr *ipv4_hdr;
567 struct rte_ipv6_hdr *ipv6_hdr;
569 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
570 ether_type = eth_hdr->ether_type;
571 l3 = (uint8_t *)eth_hdr + sizeof(struct rte_ether_hdr);
572 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
573 ipv4_hdr = (struct rte_ipv4_hdr *)l3;
574 hdr_len = rte_ipv4_hdr_len(ipv4_hdr);
575 if (hdr_len == sizeof(struct rte_ipv4_hdr)) {
576 packet_type |= RTE_PTYPE_L3_IPV4;
577 if (ipv4_hdr->next_proto_id == IPPROTO_TCP)
578 packet_type |= RTE_PTYPE_L4_TCP;
579 else if (ipv4_hdr->next_proto_id == IPPROTO_UDP)
580 packet_type |= RTE_PTYPE_L4_UDP;
582 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
583 } else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
584 ipv6_hdr = (struct rte_ipv6_hdr *)l3;
585 if (ipv6_hdr->proto == IPPROTO_TCP)
586 packet_type |= RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP;
587 else if (ipv6_hdr->proto == IPPROTO_UDP)
588 packet_type |= RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP;
590 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
593 m->packet_type = packet_type;
597 em_cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused,
598 struct rte_mbuf *pkts[], uint16_t nb_pkts,
599 uint16_t max_pkts __rte_unused,
600 void *user_param __rte_unused)
604 for (i = 0; i < nb_pkts; ++i)
605 em_parse_ptype(pkts[i]);
610 /* main processing loop */
612 em_main_loop(__rte_unused void *dummy)
614 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
616 uint64_t prev_tsc, diff_tsc, cur_tsc;
620 struct lcore_conf *qconf;
621 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
622 US_PER_S * BURST_TX_DRAIN_US;
624 lcore_id = rte_lcore_id();
625 qconf = &lcore_conf[lcore_id];
627 const uint16_t n_rx_q = qconf->n_rx_queue;
628 const uint16_t n_tx_p = qconf->n_tx_port;
630 RTE_LOG(INFO, L3FWD, "lcore %u has nothing to do\n", lcore_id);
634 RTE_LOG(INFO, L3FWD, "entering main loop on lcore %u\n", lcore_id);
636 for (i = 0; i < n_rx_q; i++) {
638 portid = qconf->rx_queue_list[i].port_id;
639 queueid = qconf->rx_queue_list[i].queue_id;
641 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n",
642 lcore_id, portid, queueid);
645 cur_tsc = rte_rdtsc();
648 while (!force_quit) {
651 * TX burst queue drain
653 diff_tsc = cur_tsc - prev_tsc;
654 if (unlikely(diff_tsc > drain_tsc)) {
656 for (i = 0; i < n_tx_p; ++i) {
657 portid = qconf->tx_port_id[i];
658 if (qconf->tx_mbufs[portid].len == 0)
661 qconf->tx_mbufs[portid].len,
663 qconf->tx_mbufs[portid].len = 0;
670 * Read packet from RX queues
672 for (i = 0; i < n_rx_q; ++i) {
673 portid = qconf->rx_queue_list[i].port_id;
674 queueid = qconf->rx_queue_list[i].queue_id;
675 nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
680 #if defined RTE_ARCH_X86 || defined __ARM_NEON
681 l3fwd_em_send_packets(nb_rx, pkts_burst,
684 l3fwd_em_no_opt_send_packets(nb_rx, pkts_burst,
689 cur_tsc = rte_rdtsc();
695 static __rte_always_inline void
696 em_event_loop_single(struct l3fwd_event_resources *evt_rsrc,
699 const int event_p_id = l3fwd_get_free_event_port(evt_rsrc);
700 const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
701 evt_rsrc->evq.nb_queues - 1];
702 const uint8_t event_d_id = evt_rsrc->event_d_id;
703 struct lcore_conf *lconf;
704 unsigned int lcore_id;
710 lcore_id = rte_lcore_id();
711 lconf = &lcore_conf[lcore_id];
713 RTE_LOG(INFO, L3FWD, "entering %s on lcore %u\n", __func__, lcore_id);
714 while (!force_quit) {
715 if (!rte_event_dequeue_burst(event_d_id, event_p_id, &ev, 1, 0))
718 struct rte_mbuf *mbuf = ev.mbuf;
720 #if defined RTE_ARCH_X86 || defined __ARM_NEON
721 mbuf->port = em_get_dst_port(lconf, mbuf, mbuf->port);
722 process_packet(mbuf, &mbuf->port);
724 l3fwd_em_simple_process(mbuf, lconf);
726 if (mbuf->port == BAD_PORT) {
727 rte_pktmbuf_free(mbuf);
731 if (flags & L3FWD_EVENT_TX_ENQ) {
732 ev.queue_id = tx_q_id;
733 ev.op = RTE_EVENT_OP_FORWARD;
734 while (rte_event_enqueue_burst(event_d_id, event_p_id,
735 &ev, 1) && !force_quit)
739 if (flags & L3FWD_EVENT_TX_DIRECT) {
740 rte_event_eth_tx_adapter_txq_set(mbuf, 0);
741 while (!rte_event_eth_tx_adapter_enqueue(event_d_id,
742 event_p_id, &ev, 1, 0) &&
749 static __rte_always_inline void
750 em_event_loop_burst(struct l3fwd_event_resources *evt_rsrc,
753 const int event_p_id = l3fwd_get_free_event_port(evt_rsrc);
754 const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
755 evt_rsrc->evq.nb_queues - 1];
756 const uint8_t event_d_id = evt_rsrc->event_d_id;
757 const uint16_t deq_len = evt_rsrc->deq_depth;
758 struct rte_event events[MAX_PKT_BURST];
759 struct lcore_conf *lconf;
760 unsigned int lcore_id;
761 int i, nb_enq, nb_deq;
766 lcore_id = rte_lcore_id();
768 lconf = &lcore_conf[lcore_id];
770 RTE_LOG(INFO, L3FWD, "entering %s on lcore %u\n", __func__, lcore_id);
772 while (!force_quit) {
773 /* Read events from RX queues */
774 nb_deq = rte_event_dequeue_burst(event_d_id, event_p_id,
781 #if defined RTE_ARCH_X86 || defined __ARM_NEON
782 l3fwd_em_process_events(nb_deq, (struct rte_event **)&events,
785 l3fwd_em_no_opt_process_events(nb_deq,
786 (struct rte_event **)&events,
789 for (i = 0; i < nb_deq; i++) {
790 if (flags & L3FWD_EVENT_TX_ENQ) {
791 events[i].queue_id = tx_q_id;
792 events[i].op = RTE_EVENT_OP_FORWARD;
795 if (flags & L3FWD_EVENT_TX_DIRECT)
796 rte_event_eth_tx_adapter_txq_set(events[i].mbuf,
800 if (flags & L3FWD_EVENT_TX_ENQ) {
801 nb_enq = rte_event_enqueue_burst(event_d_id, event_p_id,
803 while (nb_enq < nb_deq && !force_quit)
804 nb_enq += rte_event_enqueue_burst(event_d_id,
805 event_p_id, events + nb_enq,
809 if (flags & L3FWD_EVENT_TX_DIRECT) {
810 nb_enq = rte_event_eth_tx_adapter_enqueue(event_d_id,
811 event_p_id, events, nb_deq, 0);
812 while (nb_enq < nb_deq && !force_quit)
813 nb_enq += rte_event_eth_tx_adapter_enqueue(
814 event_d_id, event_p_id,
821 static __rte_always_inline void
822 em_event_loop(struct l3fwd_event_resources *evt_rsrc,
825 if (flags & L3FWD_EVENT_SINGLE)
826 em_event_loop_single(evt_rsrc, flags);
827 if (flags & L3FWD_EVENT_BURST)
828 em_event_loop_burst(evt_rsrc, flags);
832 em_event_main_loop_tx_d(__rte_unused void *dummy)
834 struct l3fwd_event_resources *evt_rsrc =
835 l3fwd_get_eventdev_rsrc();
837 em_event_loop(evt_rsrc, L3FWD_EVENT_TX_DIRECT | L3FWD_EVENT_SINGLE);
842 em_event_main_loop_tx_d_burst(__rte_unused void *dummy)
844 struct l3fwd_event_resources *evt_rsrc =
845 l3fwd_get_eventdev_rsrc();
847 em_event_loop(evt_rsrc, L3FWD_EVENT_TX_DIRECT | L3FWD_EVENT_BURST);
852 em_event_main_loop_tx_q(__rte_unused void *dummy)
854 struct l3fwd_event_resources *evt_rsrc =
855 l3fwd_get_eventdev_rsrc();
857 em_event_loop(evt_rsrc, L3FWD_EVENT_TX_ENQ | L3FWD_EVENT_SINGLE);
862 em_event_main_loop_tx_q_burst(__rte_unused void *dummy)
864 struct l3fwd_event_resources *evt_rsrc =
865 l3fwd_get_eventdev_rsrc();
867 em_event_loop(evt_rsrc, L3FWD_EVENT_TX_ENQ | L3FWD_EVENT_BURST);
871 /* Same eventdev loop for single and burst of vector */
872 static __rte_always_inline void
873 em_event_loop_vector(struct l3fwd_event_resources *evt_rsrc,
876 const int event_p_id = l3fwd_get_free_event_port(evt_rsrc);
877 const uint8_t tx_q_id =
878 evt_rsrc->evq.event_q_id[evt_rsrc->evq.nb_queues - 1];
879 const uint8_t event_d_id = evt_rsrc->event_d_id;
880 const uint16_t deq_len = evt_rsrc->deq_depth;
881 struct rte_event events[MAX_PKT_BURST];
882 struct lcore_conf *lconf;
883 unsigned int lcore_id;
884 int i, nb_enq, nb_deq;
889 lcore_id = rte_lcore_id();
890 lconf = &lcore_conf[lcore_id];
892 RTE_LOG(INFO, L3FWD, "entering %s on lcore %u\n", __func__, lcore_id);
894 while (!force_quit) {
895 /* Read events from RX queues */
896 nb_deq = rte_event_dequeue_burst(event_d_id, event_p_id, events,
903 for (i = 0; i < nb_deq; i++) {
904 if (flags & L3FWD_EVENT_TX_ENQ) {
905 events[i].queue_id = tx_q_id;
906 events[i].op = RTE_EVENT_OP_FORWARD;
909 #if defined RTE_ARCH_X86 || defined __ARM_NEON
910 l3fwd_em_process_event_vector(events[i].vec, lconf);
912 l3fwd_em_no_opt_process_event_vector(events[i].vec,
915 if (flags & L3FWD_EVENT_TX_DIRECT)
916 event_vector_txq_set(events[i].vec, 0);
919 if (flags & L3FWD_EVENT_TX_ENQ) {
920 nb_enq = rte_event_enqueue_burst(event_d_id, event_p_id,
922 while (nb_enq < nb_deq && !force_quit)
923 nb_enq += rte_event_enqueue_burst(
924 event_d_id, event_p_id, events + nb_enq,
928 if (flags & L3FWD_EVENT_TX_DIRECT) {
929 nb_enq = rte_event_eth_tx_adapter_enqueue(
930 event_d_id, event_p_id, events, nb_deq, 0);
931 while (nb_enq < nb_deq && !force_quit)
932 nb_enq += rte_event_eth_tx_adapter_enqueue(
933 event_d_id, event_p_id, events + nb_enq,
940 em_event_main_loop_tx_d_vector(__rte_unused void *dummy)
942 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
944 em_event_loop_vector(evt_rsrc, L3FWD_EVENT_TX_DIRECT);
949 em_event_main_loop_tx_d_burst_vector(__rte_unused void *dummy)
951 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
953 em_event_loop_vector(evt_rsrc, L3FWD_EVENT_TX_DIRECT);
958 em_event_main_loop_tx_q_vector(__rte_unused void *dummy)
960 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
962 em_event_loop_vector(evt_rsrc, L3FWD_EVENT_TX_ENQ);
967 em_event_main_loop_tx_q_burst_vector(__rte_unused void *dummy)
969 struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
971 em_event_loop_vector(evt_rsrc, L3FWD_EVENT_TX_ENQ);
975 /* Initialize exact match (hash) parameters. 8< */
977 setup_hash(const int socketid)
979 struct rte_hash_parameters ipv4_l3fwd_hash_params = {
981 .entries = L3FWD_HASH_ENTRIES,
982 .key_len = sizeof(union ipv4_5tuple_host),
983 .hash_func = ipv4_hash_crc,
984 .hash_func_init_val = 0,
987 struct rte_hash_parameters ipv6_l3fwd_hash_params = {
989 .entries = L3FWD_HASH_ENTRIES,
990 .key_len = sizeof(union ipv6_5tuple_host),
991 .hash_func = ipv6_hash_crc,
992 .hash_func_init_val = 0,
997 /* create ipv4 hash */
998 snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid);
999 ipv4_l3fwd_hash_params.name = s;
1000 ipv4_l3fwd_hash_params.socket_id = socketid;
1001 ipv4_l3fwd_em_lookup_struct[socketid] =
1002 rte_hash_create(&ipv4_l3fwd_hash_params);
1003 if (ipv4_l3fwd_em_lookup_struct[socketid] == NULL)
1004 rte_exit(EXIT_FAILURE,
1005 "Unable to create the l3fwd hash on socket %d\n",
1008 /* create ipv6 hash */
1009 snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid);
1010 ipv6_l3fwd_hash_params.name = s;
1011 ipv6_l3fwd_hash_params.socket_id = socketid;
1012 ipv6_l3fwd_em_lookup_struct[socketid] =
1013 rte_hash_create(&ipv6_l3fwd_hash_params);
1014 if (ipv6_l3fwd_em_lookup_struct[socketid] == NULL)
1015 rte_exit(EXIT_FAILURE,
1016 "Unable to create the l3fwd hash on socket %d\n",
1019 if (hash_entry_number != HASH_ENTRY_NUMBER_DEFAULT) {
1020 /* For testing hash matching with a large number of flows we
1021 * generate millions of IP 5-tuples with an incremented dst
1022 * address to initialize the hash table. */
1024 /* populate the ipv4 hash */
1025 populate_ipv4_many_flow_into_table(
1026 ipv4_l3fwd_em_lookup_struct[socketid],
1029 /* populate the ipv6 hash */
1030 populate_ipv6_many_flow_into_table(
1031 ipv6_l3fwd_em_lookup_struct[socketid],
1036 * Use data in ipv4/ipv6 l3fwd lookup table
1037 * directly to initialize the hash table.
1040 /* populate the ipv4 hash */
1041 populate_ipv4_few_flow_into_table(
1042 ipv4_l3fwd_em_lookup_struct[socketid]);
1044 /* populate the ipv6 hash */
1045 populate_ipv6_few_flow_into_table(
1046 ipv6_l3fwd_em_lookup_struct[socketid]);
1050 /* >8 End of initialization of hash parameters. */
1052 /* Return ipv4/ipv6 em fwd lookup struct. */
1054 em_get_ipv4_l3fwd_lookup_struct(const int socketid)
1056 return ipv4_l3fwd_em_lookup_struct[socketid];
1060 em_get_ipv6_l3fwd_lookup_struct(const int socketid)
1062 return ipv6_l3fwd_em_lookup_struct[socketid];