net/ice: fix memory leak when releasing VSI
[dpdk.git] / examples / l3fwd / l3fwd_em.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2016 Intel Corporation
3  */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <stdint.h>
8 #include <inttypes.h>
9 #include <sys/types.h>
10 #include <string.h>
11 #include <sys/queue.h>
12 #include <stdarg.h>
13 #include <errno.h>
14 #include <getopt.h>
15 #include <stdbool.h>
16 #include <netinet/in.h>
17
18 #include <rte_debug.h>
19 #include <rte_ether.h>
20 #include <rte_ethdev.h>
21 #include <rte_cycles.h>
22 #include <rte_mbuf.h>
23 #include <rte_ip.h>
24 #include <rte_tcp.h>
25 #include <rte_udp.h>
26 #include <rte_hash.h>
27
28 #include "l3fwd.h"
29 #include "l3fwd_event.h"
30
31 #if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32)
32 #define EM_HASH_CRC 1
33 #endif
34
35 #ifdef EM_HASH_CRC
36 #include <rte_hash_crc.h>
37 #define DEFAULT_HASH_FUNC       rte_hash_crc
38 #else
39 #include <rte_jhash.h>
40 #define DEFAULT_HASH_FUNC       rte_jhash
41 #endif
42
43 #define IPV6_ADDR_LEN 16
44
45 struct ipv4_5tuple {
46         uint32_t ip_dst;
47         uint32_t ip_src;
48         uint16_t port_dst;
49         uint16_t port_src;
50         uint8_t  proto;
51 } __rte_packed;
52
53 union ipv4_5tuple_host {
54         struct {
55                 uint8_t  pad0;
56                 uint8_t  proto;
57                 uint16_t pad1;
58                 uint32_t ip_src;
59                 uint32_t ip_dst;
60                 uint16_t port_src;
61                 uint16_t port_dst;
62         };
63         xmm_t xmm;
64 };
65
66 #define XMM_NUM_IN_IPV6_5TUPLE 3
67
68 struct ipv6_5tuple {
69         uint8_t  ip_dst[IPV6_ADDR_LEN];
70         uint8_t  ip_src[IPV6_ADDR_LEN];
71         uint16_t port_dst;
72         uint16_t port_src;
73         uint8_t  proto;
74 } __rte_packed;
75
76 union ipv6_5tuple_host {
77         struct {
78                 uint16_t pad0;
79                 uint8_t  proto;
80                 uint8_t  pad1;
81                 uint8_t  ip_src[IPV6_ADDR_LEN];
82                 uint8_t  ip_dst[IPV6_ADDR_LEN];
83                 uint16_t port_src;
84                 uint16_t port_dst;
85                 uint64_t reserve;
86         };
87         xmm_t xmm[XMM_NUM_IN_IPV6_5TUPLE];
88 };
89
90
91
92 struct ipv4_l3fwd_em_route {
93         struct ipv4_5tuple key;
94         uint8_t if_out;
95 };
96
97 struct ipv6_l3fwd_em_route {
98         struct ipv6_5tuple key;
99         uint8_t if_out;
100 };
101
102 static struct ipv4_l3fwd_em_route ipv4_l3fwd_em_route_array[] = {
103         {{RTE_IPV4(101, 0, 0, 0), RTE_IPV4(100, 10, 0, 1),  101, 11, IPPROTO_TCP}, 0},
104         {{RTE_IPV4(201, 0, 0, 0), RTE_IPV4(200, 20, 0, 1),  102, 12, IPPROTO_TCP}, 1},
105         {{RTE_IPV4(111, 0, 0, 0), RTE_IPV4(100, 30, 0, 1),  101, 11, IPPROTO_TCP}, 2},
106         {{RTE_IPV4(211, 0, 0, 0), RTE_IPV4(200, 40, 0, 1),  102, 12, IPPROTO_TCP}, 3},
107 };
108
109 static struct ipv6_l3fwd_em_route ipv6_l3fwd_em_route_array[] = {
110         {{
111         {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
112         {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
113         101, 11, IPPROTO_TCP}, 0},
114
115         {{
116         {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
117         {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
118         102, 12, IPPROTO_TCP}, 1},
119
120         {{
121         {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
122         {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
123         101, 11, IPPROTO_TCP}, 2},
124
125         {{
126         {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
127         {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
128         102, 12, IPPROTO_TCP}, 3},
129 };
130
131 struct rte_hash *ipv4_l3fwd_em_lookup_struct[NB_SOCKETS];
132 struct rte_hash *ipv6_l3fwd_em_lookup_struct[NB_SOCKETS];
133
134 static inline uint32_t
135 ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len,
136                 uint32_t init_val)
137 {
138         const union ipv4_5tuple_host *k;
139         uint32_t t;
140         const uint32_t *p;
141
142         k = data;
143         t = k->proto;
144         p = (const uint32_t *)&k->port_src;
145
146 #ifdef EM_HASH_CRC
147         init_val = rte_hash_crc_4byte(t, init_val);
148         init_val = rte_hash_crc_4byte(k->ip_src, init_val);
149         init_val = rte_hash_crc_4byte(k->ip_dst, init_val);
150         init_val = rte_hash_crc_4byte(*p, init_val);
151 #else
152         init_val = rte_jhash_1word(t, init_val);
153         init_val = rte_jhash_1word(k->ip_src, init_val);
154         init_val = rte_jhash_1word(k->ip_dst, init_val);
155         init_val = rte_jhash_1word(*p, init_val);
156 #endif
157
158         return init_val;
159 }
160
161 static inline uint32_t
162 ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len,
163                 uint32_t init_val)
164 {
165         const union ipv6_5tuple_host *k;
166         uint32_t t;
167         const uint32_t *p;
168 #ifdef EM_HASH_CRC
169         const uint32_t  *ip_src0, *ip_src1, *ip_src2, *ip_src3;
170         const uint32_t  *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3;
171 #endif
172
173         k = data;
174         t = k->proto;
175         p = (const uint32_t *)&k->port_src;
176
177 #ifdef EM_HASH_CRC
178         ip_src0 = (const uint32_t *) k->ip_src;
179         ip_src1 = (const uint32_t *)(k->ip_src+4);
180         ip_src2 = (const uint32_t *)(k->ip_src+8);
181         ip_src3 = (const uint32_t *)(k->ip_src+12);
182         ip_dst0 = (const uint32_t *) k->ip_dst;
183         ip_dst1 = (const uint32_t *)(k->ip_dst+4);
184         ip_dst2 = (const uint32_t *)(k->ip_dst+8);
185         ip_dst3 = (const uint32_t *)(k->ip_dst+12);
186         init_val = rte_hash_crc_4byte(t, init_val);
187         init_val = rte_hash_crc_4byte(*ip_src0, init_val);
188         init_val = rte_hash_crc_4byte(*ip_src1, init_val);
189         init_val = rte_hash_crc_4byte(*ip_src2, init_val);
190         init_val = rte_hash_crc_4byte(*ip_src3, init_val);
191         init_val = rte_hash_crc_4byte(*ip_dst0, init_val);
192         init_val = rte_hash_crc_4byte(*ip_dst1, init_val);
193         init_val = rte_hash_crc_4byte(*ip_dst2, init_val);
194         init_val = rte_hash_crc_4byte(*ip_dst3, init_val);
195         init_val = rte_hash_crc_4byte(*p, init_val);
196 #else
197         init_val = rte_jhash_1word(t, init_val);
198         init_val = rte_jhash(k->ip_src,
199                         sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
200         init_val = rte_jhash(k->ip_dst,
201                         sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
202         init_val = rte_jhash_1word(*p, init_val);
203 #endif
204         return init_val;
205 }
206
207 #define IPV4_L3FWD_EM_NUM_ROUTES RTE_DIM(ipv4_l3fwd_em_route_array)
208
209 #define IPV6_L3FWD_EM_NUM_ROUTES RTE_DIM(ipv6_l3fwd_em_route_array)
210
211 static uint8_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
212 static uint8_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
213
214 static rte_xmm_t mask0;
215 static rte_xmm_t mask1;
216 static rte_xmm_t mask2;
217
218 #if defined(RTE_MACHINE_CPUFLAG_SSE2)
219 static inline xmm_t
220 em_mask_key(void *key, xmm_t mask)
221 {
222         __m128i data = _mm_loadu_si128((__m128i *)(key));
223
224         return _mm_and_si128(data, mask);
225 }
226 #elif defined(RTE_MACHINE_CPUFLAG_NEON)
227 static inline xmm_t
228 em_mask_key(void *key, xmm_t mask)
229 {
230         int32x4_t data = vld1q_s32((int32_t *)key);
231
232         return vandq_s32(data, mask);
233 }
234 #elif defined(RTE_MACHINE_CPUFLAG_ALTIVEC)
235 static inline xmm_t
236 em_mask_key(void *key, xmm_t mask)
237 {
238         xmm_t data = vec_ld(0, (xmm_t *)(key));
239
240         return vec_and(data, mask);
241 }
242 #else
243 #error No vector engine (SSE, NEON, ALTIVEC) available, check your toolchain
244 #endif
245
246 static inline uint16_t
247 em_get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, void *lookup_struct)
248 {
249         int ret = 0;
250         union ipv4_5tuple_host key;
251         struct rte_hash *ipv4_l3fwd_lookup_struct =
252                 (struct rte_hash *)lookup_struct;
253
254         ipv4_hdr = (uint8_t *)ipv4_hdr +
255                 offsetof(struct rte_ipv4_hdr, time_to_live);
256
257         /*
258          * Get 5 tuple: dst port, src port, dst IP address,
259          * src IP address and protocol.
260          */
261         key.xmm = em_mask_key(ipv4_hdr, mask0.x);
262
263         /* Find destination port */
264         ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key);
265         return (ret < 0) ? portid : ipv4_l3fwd_out_if[ret];
266 }
267
268 static inline uint16_t
269 em_get_ipv6_dst_port(void *ipv6_hdr, uint16_t portid, void *lookup_struct)
270 {
271         int ret = 0;
272         union ipv6_5tuple_host key;
273         struct rte_hash *ipv6_l3fwd_lookup_struct =
274                 (struct rte_hash *)lookup_struct;
275
276         ipv6_hdr = (uint8_t *)ipv6_hdr +
277                 offsetof(struct rte_ipv6_hdr, payload_len);
278         void *data0 = ipv6_hdr;
279         void *data1 = ((uint8_t *)ipv6_hdr) + sizeof(xmm_t);
280         void *data2 = ((uint8_t *)ipv6_hdr) + sizeof(xmm_t) + sizeof(xmm_t);
281
282         /* Get part of 5 tuple: src IP address lower 96 bits and protocol */
283         key.xmm[0] = em_mask_key(data0, mask1.x);
284
285         /*
286          * Get part of 5 tuple: dst IP address lower 96 bits
287          * and src IP address higher 32 bits.
288          */
289 #if defined RTE_ARCH_X86
290         key.xmm[1] = _mm_loadu_si128(data1);
291 #else
292         key.xmm[1] = *(xmm_t *)data1;
293 #endif
294
295         /*
296          * Get part of 5 tuple: dst port and src port
297          * and dst IP address higher 32 bits.
298          */
299         key.xmm[2] = em_mask_key(data2, mask2.x);
300
301         /* Find destination port */
302         ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key);
303         return (ret < 0) ? portid : ipv6_l3fwd_out_if[ret];
304 }
305
306 #if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON
307 #if defined(NO_HASH_MULTI_LOOKUP)
308 #include "l3fwd_em_sequential.h"
309 #else
310 #include "l3fwd_em_hlm.h"
311 #endif
312 #else
313 #include "l3fwd_em.h"
314 #endif
315
316 static void
317 convert_ipv4_5tuple(struct ipv4_5tuple *key1,
318                 union ipv4_5tuple_host *key2)
319 {
320         key2->ip_dst = rte_cpu_to_be_32(key1->ip_dst);
321         key2->ip_src = rte_cpu_to_be_32(key1->ip_src);
322         key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
323         key2->port_src = rte_cpu_to_be_16(key1->port_src);
324         key2->proto = key1->proto;
325         key2->pad0 = 0;
326         key2->pad1 = 0;
327 }
328
329 static void
330 convert_ipv6_5tuple(struct ipv6_5tuple *key1,
331                 union ipv6_5tuple_host *key2)
332 {
333         uint32_t i;
334
335         for (i = 0; i < 16; i++) {
336                 key2->ip_dst[i] = key1->ip_dst[i];
337                 key2->ip_src[i] = key1->ip_src[i];
338         }
339         key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
340         key2->port_src = rte_cpu_to_be_16(key1->port_src);
341         key2->proto = key1->proto;
342         key2->pad0 = 0;
343         key2->pad1 = 0;
344         key2->reserve = 0;
345 }
346
347 #define BYTE_VALUE_MAX 256
348 #define ALL_32_BITS 0xffffffff
349 #define BIT_8_TO_15 0x0000ff00
350
351 static inline void
352 populate_ipv4_few_flow_into_table(const struct rte_hash *h)
353 {
354         uint32_t i;
355         int32_t ret;
356
357         mask0 = (rte_xmm_t){.u32 = {BIT_8_TO_15, ALL_32_BITS,
358                                 ALL_32_BITS, ALL_32_BITS} };
359
360         for (i = 0; i < IPV4_L3FWD_EM_NUM_ROUTES; i++) {
361                 struct ipv4_l3fwd_em_route  entry;
362                 union ipv4_5tuple_host newkey;
363
364                 entry = ipv4_l3fwd_em_route_array[i];
365                 convert_ipv4_5tuple(&entry.key, &newkey);
366                 ret = rte_hash_add_key(h, (void *) &newkey);
367                 if (ret < 0) {
368                         rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
369                                 " to the l3fwd hash.\n", i);
370                 }
371                 ipv4_l3fwd_out_if[ret] = entry.if_out;
372         }
373         printf("Hash: Adding 0x%" PRIx64 " keys\n",
374                 (uint64_t)IPV4_L3FWD_EM_NUM_ROUTES);
375 }
376
377 #define BIT_16_TO_23 0x00ff0000
378 static inline void
379 populate_ipv6_few_flow_into_table(const struct rte_hash *h)
380 {
381         uint32_t i;
382         int32_t ret;
383
384         mask1 = (rte_xmm_t){.u32 = {BIT_16_TO_23, ALL_32_BITS,
385                                 ALL_32_BITS, ALL_32_BITS} };
386
387         mask2 = (rte_xmm_t){.u32 = {ALL_32_BITS, ALL_32_BITS, 0, 0} };
388
389         for (i = 0; i < IPV6_L3FWD_EM_NUM_ROUTES; i++) {
390                 struct ipv6_l3fwd_em_route entry;
391                 union ipv6_5tuple_host newkey;
392
393                 entry = ipv6_l3fwd_em_route_array[i];
394                 convert_ipv6_5tuple(&entry.key, &newkey);
395                 ret = rte_hash_add_key(h, (void *) &newkey);
396                 if (ret < 0) {
397                         rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
398                                 " to the l3fwd hash.\n", i);
399                 }
400                 ipv6_l3fwd_out_if[ret] = entry.if_out;
401         }
402         printf("Hash: Adding 0x%" PRIx64 "keys\n",
403                 (uint64_t)IPV6_L3FWD_EM_NUM_ROUTES);
404 }
405
406 #define NUMBER_PORT_USED 4
407 static inline void
408 populate_ipv4_many_flow_into_table(const struct rte_hash *h,
409                 unsigned int nr_flow)
410 {
411         unsigned i;
412
413         mask0 = (rte_xmm_t){.u32 = {BIT_8_TO_15, ALL_32_BITS,
414                                 ALL_32_BITS, ALL_32_BITS} };
415
416         for (i = 0; i < nr_flow; i++) {
417                 struct ipv4_l3fwd_em_route entry;
418                 union ipv4_5tuple_host newkey;
419
420                 uint8_t a = (uint8_t)
421                         ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
422                 uint8_t b = (uint8_t)
423                         (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
424                 uint8_t c = (uint8_t)
425                         ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
426
427                 /* Create the ipv4 exact match flow */
428                 memset(&entry, 0, sizeof(entry));
429                 switch (i & (NUMBER_PORT_USED - 1)) {
430                 case 0:
431                         entry = ipv4_l3fwd_em_route_array[0];
432                         entry.key.ip_dst = RTE_IPV4(101, c, b, a);
433                         break;
434                 case 1:
435                         entry = ipv4_l3fwd_em_route_array[1];
436                         entry.key.ip_dst = RTE_IPV4(201, c, b, a);
437                         break;
438                 case 2:
439                         entry = ipv4_l3fwd_em_route_array[2];
440                         entry.key.ip_dst = RTE_IPV4(111, c, b, a);
441                         break;
442                 case 3:
443                         entry = ipv4_l3fwd_em_route_array[3];
444                         entry.key.ip_dst = RTE_IPV4(211, c, b, a);
445                         break;
446                 };
447                 convert_ipv4_5tuple(&entry.key, &newkey);
448                 int32_t ret = rte_hash_add_key(h, (void *) &newkey);
449
450                 if (ret < 0)
451                         rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
452
453                 ipv4_l3fwd_out_if[ret] = (uint8_t) entry.if_out;
454
455         }
456         printf("Hash: Adding 0x%x keys\n", nr_flow);
457 }
458
459 static inline void
460 populate_ipv6_many_flow_into_table(const struct rte_hash *h,
461                 unsigned int nr_flow)
462 {
463         unsigned i;
464
465         mask1 = (rte_xmm_t){.u32 = {BIT_16_TO_23, ALL_32_BITS,
466                                 ALL_32_BITS, ALL_32_BITS} };
467         mask2 = (rte_xmm_t){.u32 = {ALL_32_BITS, ALL_32_BITS, 0, 0} };
468
469         for (i = 0; i < nr_flow; i++) {
470                 struct ipv6_l3fwd_em_route entry;
471                 union ipv6_5tuple_host newkey;
472
473                 uint8_t a = (uint8_t)
474                         ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
475                 uint8_t b = (uint8_t)
476                         (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
477                 uint8_t c = (uint8_t)
478                         ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
479
480                 /* Create the ipv6 exact match flow */
481                 memset(&entry, 0, sizeof(entry));
482                 switch (i & (NUMBER_PORT_USED - 1)) {
483                 case 0:
484                         entry = ipv6_l3fwd_em_route_array[0];
485                         break;
486                 case 1:
487                         entry = ipv6_l3fwd_em_route_array[1];
488                         break;
489                 case 2:
490                         entry = ipv6_l3fwd_em_route_array[2];
491                         break;
492                 case 3:
493                         entry = ipv6_l3fwd_em_route_array[3];
494                         break;
495                 };
496                 entry.key.ip_dst[13] = c;
497                 entry.key.ip_dst[14] = b;
498                 entry.key.ip_dst[15] = a;
499                 convert_ipv6_5tuple(&entry.key, &newkey);
500                 int32_t ret = rte_hash_add_key(h, (void *) &newkey);
501
502                 if (ret < 0)
503                         rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
504
505                 ipv6_l3fwd_out_if[ret] = (uint8_t) entry.if_out;
506
507         }
508         printf("Hash: Adding 0x%x keys\n", nr_flow);
509 }
510
511 /* Requirements:
512  * 1. IP packets without extension;
513  * 2. L4 payload should be either TCP or UDP.
514  */
515 int
516 em_check_ptype(int portid)
517 {
518         int i, ret;
519         int ptype_l3_ipv4_ext = 0;
520         int ptype_l3_ipv6_ext = 0;
521         int ptype_l4_tcp = 0;
522         int ptype_l4_udp = 0;
523         uint32_t ptype_mask = RTE_PTYPE_L3_MASK | RTE_PTYPE_L4_MASK;
524
525         ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, NULL, 0);
526         if (ret <= 0)
527                 return 0;
528
529         uint32_t ptypes[ret];
530
531         ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, ptypes, ret);
532         for (i = 0; i < ret; ++i) {
533                 switch (ptypes[i]) {
534                 case RTE_PTYPE_L3_IPV4_EXT:
535                         ptype_l3_ipv4_ext = 1;
536                         break;
537                 case RTE_PTYPE_L3_IPV6_EXT:
538                         ptype_l3_ipv6_ext = 1;
539                         break;
540                 case RTE_PTYPE_L4_TCP:
541                         ptype_l4_tcp = 1;
542                         break;
543                 case RTE_PTYPE_L4_UDP:
544                         ptype_l4_udp = 1;
545                         break;
546                 }
547         }
548
549         if (ptype_l3_ipv4_ext == 0)
550                 printf("port %d cannot parse RTE_PTYPE_L3_IPV4_EXT\n", portid);
551         if (ptype_l3_ipv6_ext == 0)
552                 printf("port %d cannot parse RTE_PTYPE_L3_IPV6_EXT\n", portid);
553         if (!ptype_l3_ipv4_ext || !ptype_l3_ipv6_ext)
554                 return 0;
555
556         if (ptype_l4_tcp == 0)
557                 printf("port %d cannot parse RTE_PTYPE_L4_TCP\n", portid);
558         if (ptype_l4_udp == 0)
559                 printf("port %d cannot parse RTE_PTYPE_L4_UDP\n", portid);
560         if (ptype_l4_tcp && ptype_l4_udp)
561                 return 1;
562
563         return 0;
564 }
565
566 static inline void
567 em_parse_ptype(struct rte_mbuf *m)
568 {
569         struct rte_ether_hdr *eth_hdr;
570         uint32_t packet_type = RTE_PTYPE_UNKNOWN;
571         uint16_t ether_type;
572         void *l3;
573         int hdr_len;
574         struct rte_ipv4_hdr *ipv4_hdr;
575         struct rte_ipv6_hdr *ipv6_hdr;
576
577         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
578         ether_type = eth_hdr->ether_type;
579         l3 = (uint8_t *)eth_hdr + sizeof(struct rte_ether_hdr);
580         if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
581                 ipv4_hdr = (struct rte_ipv4_hdr *)l3;
582                 hdr_len = (ipv4_hdr->version_ihl & RTE_IPV4_HDR_IHL_MASK) *
583                           RTE_IPV4_IHL_MULTIPLIER;
584                 if (hdr_len == sizeof(struct rte_ipv4_hdr)) {
585                         packet_type |= RTE_PTYPE_L3_IPV4;
586                         if (ipv4_hdr->next_proto_id == IPPROTO_TCP)
587                                 packet_type |= RTE_PTYPE_L4_TCP;
588                         else if (ipv4_hdr->next_proto_id == IPPROTO_UDP)
589                                 packet_type |= RTE_PTYPE_L4_UDP;
590                 } else
591                         packet_type |= RTE_PTYPE_L3_IPV4_EXT;
592         } else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
593                 ipv6_hdr = (struct rte_ipv6_hdr *)l3;
594                 if (ipv6_hdr->proto == IPPROTO_TCP)
595                         packet_type |= RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP;
596                 else if (ipv6_hdr->proto == IPPROTO_UDP)
597                         packet_type |= RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP;
598                 else
599                         packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
600         }
601
602         m->packet_type = packet_type;
603 }
604
605 uint16_t
606 em_cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused,
607                   struct rte_mbuf *pkts[], uint16_t nb_pkts,
608                   uint16_t max_pkts __rte_unused,
609                   void *user_param __rte_unused)
610 {
611         unsigned i;
612
613         for (i = 0; i < nb_pkts; ++i)
614                 em_parse_ptype(pkts[i]);
615
616         return nb_pkts;
617 }
618
619 /* main processing loop */
620 int
621 em_main_loop(__rte_unused void *dummy)
622 {
623         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
624         unsigned lcore_id;
625         uint64_t prev_tsc, diff_tsc, cur_tsc;
626         int i, nb_rx;
627         uint8_t queueid;
628         uint16_t portid;
629         struct lcore_conf *qconf;
630         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
631                 US_PER_S * BURST_TX_DRAIN_US;
632
633         prev_tsc = 0;
634
635         lcore_id = rte_lcore_id();
636         qconf = &lcore_conf[lcore_id];
637
638         if (qconf->n_rx_queue == 0) {
639                 RTE_LOG(INFO, L3FWD, "lcore %u has nothing to do\n", lcore_id);
640                 return 0;
641         }
642
643         RTE_LOG(INFO, L3FWD, "entering main loop on lcore %u\n", lcore_id);
644
645         for (i = 0; i < qconf->n_rx_queue; i++) {
646
647                 portid = qconf->rx_queue_list[i].port_id;
648                 queueid = qconf->rx_queue_list[i].queue_id;
649                 RTE_LOG(INFO, L3FWD,
650                         " -- lcoreid=%u portid=%u rxqueueid=%hhu\n",
651                         lcore_id, portid, queueid);
652         }
653
654         while (!force_quit) {
655
656                 cur_tsc = rte_rdtsc();
657
658                 /*
659                  * TX burst queue drain
660                  */
661                 diff_tsc = cur_tsc - prev_tsc;
662                 if (unlikely(diff_tsc > drain_tsc)) {
663
664                         for (i = 0; i < qconf->n_tx_port; ++i) {
665                                 portid = qconf->tx_port_id[i];
666                                 if (qconf->tx_mbufs[portid].len == 0)
667                                         continue;
668                                 send_burst(qconf,
669                                         qconf->tx_mbufs[portid].len,
670                                         portid);
671                                 qconf->tx_mbufs[portid].len = 0;
672                         }
673
674                         prev_tsc = cur_tsc;
675                 }
676
677                 /*
678                  * Read packet from RX queues
679                  */
680                 for (i = 0; i < qconf->n_rx_queue; ++i) {
681                         portid = qconf->rx_queue_list[i].port_id;
682                         queueid = qconf->rx_queue_list[i].queue_id;
683                         nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
684                                 MAX_PKT_BURST);
685                         if (nb_rx == 0)
686                                 continue;
687
688 #if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON
689                         l3fwd_em_send_packets(nb_rx, pkts_burst,
690                                                         portid, qconf);
691 #else
692                         l3fwd_em_no_opt_send_packets(nb_rx, pkts_burst,
693                                                         portid, qconf);
694 #endif
695                 }
696         }
697
698         return 0;
699 }
700
701 static __rte_always_inline void
702 em_event_loop_single(struct l3fwd_event_resources *evt_rsrc,
703                 const uint8_t flags)
704 {
705         const int event_p_id = l3fwd_get_free_event_port(evt_rsrc);
706         const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
707                 evt_rsrc->evq.nb_queues - 1];
708         const uint8_t event_d_id = evt_rsrc->event_d_id;
709         struct lcore_conf *lconf;
710         unsigned int lcore_id;
711         struct rte_event ev;
712
713         if (event_p_id < 0)
714                 return;
715
716         lcore_id = rte_lcore_id();
717         lconf = &lcore_conf[lcore_id];
718
719         RTE_LOG(INFO, L3FWD, "entering %s on lcore %u\n", __func__, lcore_id);
720         while (!force_quit) {
721                 if (!rte_event_dequeue_burst(event_d_id, event_p_id, &ev, 1, 0))
722                         continue;
723
724                 struct rte_mbuf *mbuf = ev.mbuf;
725
726 #if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON
727                 mbuf->port = em_get_dst_port(lconf, mbuf, mbuf->port);
728                 process_packet(mbuf, &mbuf->port);
729 #else
730                 l3fwd_em_simple_process(mbuf, lconf);
731 #endif
732                 if (mbuf->port == BAD_PORT) {
733                         rte_pktmbuf_free(mbuf);
734                         continue;
735                 }
736
737                 if (flags & L3FWD_EVENT_TX_ENQ) {
738                         ev.queue_id = tx_q_id;
739                         ev.op = RTE_EVENT_OP_FORWARD;
740                         while (rte_event_enqueue_burst(event_d_id, event_p_id,
741                                                 &ev, 1) && !force_quit)
742                                 ;
743                 }
744
745                 if (flags & L3FWD_EVENT_TX_DIRECT) {
746                         rte_event_eth_tx_adapter_txq_set(mbuf, 0);
747                         while (!rte_event_eth_tx_adapter_enqueue(event_d_id,
748                                                 event_p_id, &ev, 1, 0) &&
749                                         !force_quit)
750                                 ;
751                 }
752         }
753 }
754
755 static __rte_always_inline void
756 em_event_loop_burst(struct l3fwd_event_resources *evt_rsrc,
757                 const uint8_t flags)
758 {
759         const int event_p_id = l3fwd_get_free_event_port(evt_rsrc);
760         const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
761                 evt_rsrc->evq.nb_queues - 1];
762         const uint8_t event_d_id = evt_rsrc->event_d_id;
763         const uint16_t deq_len = evt_rsrc->deq_depth;
764         struct rte_event events[MAX_PKT_BURST];
765         struct lcore_conf *lconf;
766         unsigned int lcore_id;
767         int i, nb_enq, nb_deq;
768
769         if (event_p_id < 0)
770                 return;
771
772         lcore_id = rte_lcore_id();
773
774         lconf = &lcore_conf[lcore_id];
775
776         RTE_LOG(INFO, L3FWD, "entering %s on lcore %u\n", __func__, lcore_id);
777
778         while (!force_quit) {
779                 /* Read events from RX queues */
780                 nb_deq = rte_event_dequeue_burst(event_d_id, event_p_id,
781                                 events, deq_len, 0);
782                 if (nb_deq == 0) {
783                         rte_pause();
784                         continue;
785                 }
786
787 #if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON
788                 l3fwd_em_process_events(nb_deq, (struct rte_event **)&events,
789                                         lconf);
790 #else
791                 l3fwd_em_no_opt_process_events(nb_deq,
792                                                (struct rte_event **)&events,
793                                                lconf);
794 #endif
795                 for (i = 0; i < nb_deq; i++) {
796                         if (flags & L3FWD_EVENT_TX_ENQ) {
797                                 events[i].queue_id = tx_q_id;
798                                 events[i].op = RTE_EVENT_OP_FORWARD;
799                         }
800
801                         if (flags & L3FWD_EVENT_TX_DIRECT)
802                                 rte_event_eth_tx_adapter_txq_set(events[i].mbuf,
803                                                                  0);
804                 }
805
806                 if (flags & L3FWD_EVENT_TX_ENQ) {
807                         nb_enq = rte_event_enqueue_burst(event_d_id, event_p_id,
808                                         events, nb_deq);
809                         while (nb_enq < nb_deq && !force_quit)
810                                 nb_enq += rte_event_enqueue_burst(event_d_id,
811                                                 event_p_id, events + nb_enq,
812                                                 nb_deq - nb_enq);
813                 }
814
815                 if (flags & L3FWD_EVENT_TX_DIRECT) {
816                         nb_enq = rte_event_eth_tx_adapter_enqueue(event_d_id,
817                                         event_p_id, events, nb_deq, 0);
818                         while (nb_enq < nb_deq && !force_quit)
819                                 nb_enq += rte_event_eth_tx_adapter_enqueue(
820                                                 event_d_id, event_p_id,
821                                                 events + nb_enq,
822                                                 nb_deq - nb_enq, 0);
823                 }
824         }
825 }
826
827 static __rte_always_inline void
828 em_event_loop(struct l3fwd_event_resources *evt_rsrc,
829                  const uint8_t flags)
830 {
831         if (flags & L3FWD_EVENT_SINGLE)
832                 em_event_loop_single(evt_rsrc, flags);
833         if (flags & L3FWD_EVENT_BURST)
834                 em_event_loop_burst(evt_rsrc, flags);
835 }
836
837 int __rte_noinline
838 em_event_main_loop_tx_d(__rte_unused void *dummy)
839 {
840         struct l3fwd_event_resources *evt_rsrc =
841                                         l3fwd_get_eventdev_rsrc();
842
843         em_event_loop(evt_rsrc, L3FWD_EVENT_TX_DIRECT | L3FWD_EVENT_SINGLE);
844         return 0;
845 }
846
847 int __rte_noinline
848 em_event_main_loop_tx_d_burst(__rte_unused void *dummy)
849 {
850         struct l3fwd_event_resources *evt_rsrc =
851                                         l3fwd_get_eventdev_rsrc();
852
853         em_event_loop(evt_rsrc, L3FWD_EVENT_TX_DIRECT | L3FWD_EVENT_BURST);
854         return 0;
855 }
856
857 int __rte_noinline
858 em_event_main_loop_tx_q(__rte_unused void *dummy)
859 {
860         struct l3fwd_event_resources *evt_rsrc =
861                                         l3fwd_get_eventdev_rsrc();
862
863         em_event_loop(evt_rsrc, L3FWD_EVENT_TX_ENQ | L3FWD_EVENT_SINGLE);
864         return 0;
865 }
866
867 int __rte_noinline
868 em_event_main_loop_tx_q_burst(__rte_unused void *dummy)
869 {
870         struct l3fwd_event_resources *evt_rsrc =
871                                         l3fwd_get_eventdev_rsrc();
872
873         em_event_loop(evt_rsrc, L3FWD_EVENT_TX_ENQ | L3FWD_EVENT_BURST);
874         return 0;
875 }
876
877 /*
878  * Initialize exact match (hash) parameters.
879  */
880 void
881 setup_hash(const int socketid)
882 {
883         struct rte_hash_parameters ipv4_l3fwd_hash_params = {
884                 .name = NULL,
885                 .entries = L3FWD_HASH_ENTRIES,
886                 .key_len = sizeof(union ipv4_5tuple_host),
887                 .hash_func = ipv4_hash_crc,
888                 .hash_func_init_val = 0,
889         };
890
891         struct rte_hash_parameters ipv6_l3fwd_hash_params = {
892                 .name = NULL,
893                 .entries = L3FWD_HASH_ENTRIES,
894                 .key_len = sizeof(union ipv6_5tuple_host),
895                 .hash_func = ipv6_hash_crc,
896                 .hash_func_init_val = 0,
897         };
898
899         char s[64];
900
901         /* create ipv4 hash */
902         snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid);
903         ipv4_l3fwd_hash_params.name = s;
904         ipv4_l3fwd_hash_params.socket_id = socketid;
905         ipv4_l3fwd_em_lookup_struct[socketid] =
906                 rte_hash_create(&ipv4_l3fwd_hash_params);
907         if (ipv4_l3fwd_em_lookup_struct[socketid] == NULL)
908                 rte_exit(EXIT_FAILURE,
909                         "Unable to create the l3fwd hash on socket %d\n",
910                         socketid);
911
912         /* create ipv6 hash */
913         snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid);
914         ipv6_l3fwd_hash_params.name = s;
915         ipv6_l3fwd_hash_params.socket_id = socketid;
916         ipv6_l3fwd_em_lookup_struct[socketid] =
917                 rte_hash_create(&ipv6_l3fwd_hash_params);
918         if (ipv6_l3fwd_em_lookup_struct[socketid] == NULL)
919                 rte_exit(EXIT_FAILURE,
920                         "Unable to create the l3fwd hash on socket %d\n",
921                         socketid);
922
923         if (hash_entry_number != HASH_ENTRY_NUMBER_DEFAULT) {
924                 /* For testing hash matching with a large number of flows we
925                  * generate millions of IP 5-tuples with an incremented dst
926                  * address to initialize the hash table. */
927                 if (ipv6 == 0) {
928                         /* populate the ipv4 hash */
929                         populate_ipv4_many_flow_into_table(
930                                 ipv4_l3fwd_em_lookup_struct[socketid],
931                                 hash_entry_number);
932                 } else {
933                         /* populate the ipv6 hash */
934                         populate_ipv6_many_flow_into_table(
935                                 ipv6_l3fwd_em_lookup_struct[socketid],
936                                 hash_entry_number);
937                 }
938         } else {
939                 /*
940                  * Use data in ipv4/ipv6 l3fwd lookup table
941                  * directly to initialize the hash table.
942                  */
943                 if (ipv6 == 0) {
944                         /* populate the ipv4 hash */
945                         populate_ipv4_few_flow_into_table(
946                                 ipv4_l3fwd_em_lookup_struct[socketid]);
947                 } else {
948                         /* populate the ipv6 hash */
949                         populate_ipv6_few_flow_into_table(
950                                 ipv6_l3fwd_em_lookup_struct[socketid]);
951                 }
952         }
953 }
954
955 /* Return ipv4/ipv6 em fwd lookup struct. */
956 void *
957 em_get_ipv4_l3fwd_lookup_struct(const int socketid)
958 {
959         return ipv4_l3fwd_em_lookup_struct[socketid];
960 }
961
962 void *
963 em_get_ipv6_l3fwd_lookup_struct(const int socketid)
964 {
965         return ipv6_l3fwd_em_lookup_struct[socketid];
966 }