net/virtio: fix incorrect cast of void *
[dpdk.git] / gro / rte_gro.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Intel Corporation
3  */
4
5 #include <rte_malloc.h>
6 #include <rte_mbuf.h>
7 #include <rte_cycles.h>
8 #include <rte_ethdev.h>
9
10 #include "rte_gro.h"
11 #include "gro_tcp4.h"
12 #include "gro_udp4.h"
13 #include "gro_vxlan_tcp4.h"
14 #include "gro_vxlan_udp4.h"
15
16 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
17                 uint16_t max_flow_num,
18                 uint16_t max_item_per_flow);
19 typedef void (*gro_tbl_destroy_fn)(void *tbl);
20 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
21
22 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
23                 gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
24                 gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
25 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
26                         gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
27                         gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
28                         NULL};
29 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
30                         gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
31                         gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
32                         NULL};
33
34 #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
35                 ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
36                 (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
37
38 #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
39                 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
40                 (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
41
42 #define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
43                 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
44                 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
45                  RTE_PTYPE_TUNNEL_VXLAN) && \
46                 ((ptype & RTE_PTYPE_INNER_L4_TCP) == \
47                  RTE_PTYPE_INNER_L4_TCP) && \
48                 (((ptype & RTE_PTYPE_INNER_L3_MASK) == \
49                   RTE_PTYPE_INNER_L3_IPV4) || \
50                  ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
51                   RTE_PTYPE_INNER_L3_IPV4_EXT) || \
52                  ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
53                   RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)))
54
55 #define IS_IPV4_VXLAN_UDP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
56                 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
57                 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
58                  RTE_PTYPE_TUNNEL_VXLAN) && \
59                 ((ptype & RTE_PTYPE_INNER_L4_UDP) == \
60                  RTE_PTYPE_INNER_L4_UDP) && \
61                 (((ptype & RTE_PTYPE_INNER_L3_MASK) == \
62                   RTE_PTYPE_INNER_L3_IPV4) || \
63                  ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
64                   RTE_PTYPE_INNER_L3_IPV4_EXT) || \
65                  ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
66                   RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)))
67
68 /*
69  * GRO context structure. It keeps the table structures, which are
70  * used to merge packets, for different GRO types. Before using
71  * rte_gro_reassemble(), applications need to create the GRO context
72  * first.
73  */
74 struct gro_ctx {
75         /* GRO types to perform */
76         uint64_t gro_types;
77         /* reassembly tables */
78         void *tbls[RTE_GRO_TYPE_MAX_NUM];
79 };
80
81 void *
82 rte_gro_ctx_create(const struct rte_gro_param *param)
83 {
84         struct gro_ctx *gro_ctx;
85         gro_tbl_create_fn create_tbl_fn;
86         uint64_t gro_type_flag = 0;
87         uint64_t gro_types = 0;
88         uint8_t i;
89
90         gro_ctx = rte_zmalloc_socket(__func__,
91                         sizeof(struct gro_ctx),
92                         RTE_CACHE_LINE_SIZE,
93                         param->socket_id);
94         if (gro_ctx == NULL)
95                 return NULL;
96
97         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
98                 gro_type_flag = 1ULL << i;
99                 if ((param->gro_types & gro_type_flag) == 0)
100                         continue;
101
102                 create_tbl_fn = tbl_create_fn[i];
103                 if (create_tbl_fn == NULL)
104                         continue;
105
106                 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id,
107                                 param->max_flow_num,
108                                 param->max_item_per_flow);
109                 if (gro_ctx->tbls[i] == NULL) {
110                         /* destroy all created tables */
111                         gro_ctx->gro_types = gro_types;
112                         rte_gro_ctx_destroy(gro_ctx);
113                         return NULL;
114                 }
115                 gro_types |= gro_type_flag;
116         }
117         gro_ctx->gro_types = param->gro_types;
118
119         return gro_ctx;
120 }
121
122 void
123 rte_gro_ctx_destroy(void *ctx)
124 {
125         gro_tbl_destroy_fn destroy_tbl_fn;
126         struct gro_ctx *gro_ctx = ctx;
127         uint64_t gro_type_flag;
128         uint8_t i;
129
130         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
131                 gro_type_flag = 1ULL << i;
132                 if ((gro_ctx->gro_types & gro_type_flag) == 0)
133                         continue;
134                 destroy_tbl_fn = tbl_destroy_fn[i];
135                 if (destroy_tbl_fn)
136                         destroy_tbl_fn(gro_ctx->tbls[i]);
137         }
138         rte_free(gro_ctx);
139 }
140
141 uint16_t
142 rte_gro_reassemble_burst(struct rte_mbuf **pkts,
143                 uint16_t nb_pkts,
144                 const struct rte_gro_param *param)
145 {
146         /* allocate a reassembly table for TCP/IPv4 GRO */
147         struct gro_tcp4_tbl tcp_tbl;
148         struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
149         struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
150
151         /* allocate a reassembly table for UDP/IPv4 GRO */
152         struct gro_udp4_tbl udp_tbl;
153         struct gro_udp4_flow udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
154         struct gro_udp4_item udp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
155
156         /* Allocate a reassembly table for VXLAN TCP GRO */
157         struct gro_vxlan_tcp4_tbl vxlan_tcp_tbl;
158         struct gro_vxlan_tcp4_flow vxlan_tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
159         struct gro_vxlan_tcp4_item vxlan_tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
160                         = {{{0}, 0, 0} };
161
162         /* Allocate a reassembly table for VXLAN UDP GRO */
163         struct gro_vxlan_udp4_tbl vxlan_udp_tbl;
164         struct gro_vxlan_udp4_flow vxlan_udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
165         struct gro_vxlan_udp4_item vxlan_udp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
166                         = {{{0}} };
167
168         struct rte_mbuf *unprocess_pkts[nb_pkts];
169         uint32_t item_num;
170         int32_t ret;
171         uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
172         uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
173                 do_vxlan_udp_gro = 0;
174
175         if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
176                                         RTE_GRO_TCP_IPV4 |
177                                         RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
178                                         RTE_GRO_UDP_IPV4)) == 0))
179                 return nb_pkts;
180
181         /* Get the maximum number of packets */
182         item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
183                                 param->max_item_per_flow));
184         item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
185
186         if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
187                 for (i = 0; i < item_num; i++)
188                         vxlan_tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
189
190                 vxlan_tcp_tbl.flows = vxlan_tcp_flows;
191                 vxlan_tcp_tbl.items = vxlan_tcp_items;
192                 vxlan_tcp_tbl.flow_num = 0;
193                 vxlan_tcp_tbl.item_num = 0;
194                 vxlan_tcp_tbl.max_flow_num = item_num;
195                 vxlan_tcp_tbl.max_item_num = item_num;
196                 do_vxlan_tcp_gro = 1;
197         }
198
199         if (param->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) {
200                 for (i = 0; i < item_num; i++)
201                         vxlan_udp_flows[i].start_index = INVALID_ARRAY_INDEX;
202
203                 vxlan_udp_tbl.flows = vxlan_udp_flows;
204                 vxlan_udp_tbl.items = vxlan_udp_items;
205                 vxlan_udp_tbl.flow_num = 0;
206                 vxlan_udp_tbl.item_num = 0;
207                 vxlan_udp_tbl.max_flow_num = item_num;
208                 vxlan_udp_tbl.max_item_num = item_num;
209                 do_vxlan_udp_gro = 1;
210         }
211
212         if (param->gro_types & RTE_GRO_TCP_IPV4) {
213                 for (i = 0; i < item_num; i++)
214                         tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
215
216                 tcp_tbl.flows = tcp_flows;
217                 tcp_tbl.items = tcp_items;
218                 tcp_tbl.flow_num = 0;
219                 tcp_tbl.item_num = 0;
220                 tcp_tbl.max_flow_num = item_num;
221                 tcp_tbl.max_item_num = item_num;
222                 do_tcp4_gro = 1;
223         }
224
225         if (param->gro_types & RTE_GRO_UDP_IPV4) {
226                 for (i = 0; i < item_num; i++)
227                         udp_flows[i].start_index = INVALID_ARRAY_INDEX;
228
229                 udp_tbl.flows = udp_flows;
230                 udp_tbl.items = udp_items;
231                 udp_tbl.flow_num = 0;
232                 udp_tbl.item_num = 0;
233                 udp_tbl.max_flow_num = item_num;
234                 udp_tbl.max_item_num = item_num;
235                 do_udp4_gro = 1;
236         }
237
238
239         for (i = 0; i < nb_pkts; i++) {
240                 /*
241                  * The timestamp is ignored, since all packets
242                  * will be flushed from the tables.
243                  */
244                 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
245                                 do_vxlan_tcp_gro) {
246                         ret = gro_vxlan_tcp4_reassemble(pkts[i],
247                                                         &vxlan_tcp_tbl, 0);
248                         if (ret > 0)
249                                 /* Merge successfully */
250                                 nb_after_gro--;
251                         else if (ret < 0)
252                                 unprocess_pkts[unprocess_num++] = pkts[i];
253                 } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) &&
254                                 do_vxlan_udp_gro) {
255                         ret = gro_vxlan_udp4_reassemble(pkts[i],
256                                                         &vxlan_udp_tbl, 0);
257                         if (ret > 0)
258                                 /* Merge successfully */
259                                 nb_after_gro--;
260                         else if (ret < 0)
261                                 unprocess_pkts[unprocess_num++] = pkts[i];
262                 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
263                                 do_tcp4_gro) {
264                         ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0);
265                         if (ret > 0)
266                                 /* merge successfully */
267                                 nb_after_gro--;
268                         else if (ret < 0)
269                                 unprocess_pkts[unprocess_num++] = pkts[i];
270                 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
271                                 do_udp4_gro) {
272                         ret = gro_udp4_reassemble(pkts[i], &udp_tbl, 0);
273                         if (ret > 0)
274                                 /* merge successfully */
275                                 nb_after_gro--;
276                         else if (ret < 0)
277                                 unprocess_pkts[unprocess_num++] = pkts[i];
278                 } else
279                         unprocess_pkts[unprocess_num++] = pkts[i];
280         }
281
282         if ((nb_after_gro < nb_pkts)
283                  || (unprocess_num < nb_pkts)) {
284                 i = 0;
285                 /* Flush all packets from the tables */
286                 if (do_vxlan_tcp_gro) {
287                         i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
288                                         0, pkts, nb_pkts);
289                 }
290
291                 if (do_vxlan_udp_gro) {
292                         i += gro_vxlan_udp4_tbl_timeout_flush(&vxlan_udp_tbl,
293                                         0, &pkts[i], nb_pkts - i);
294
295                 }
296
297                 if (do_tcp4_gro) {
298                         i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0,
299                                         &pkts[i], nb_pkts - i);
300                 }
301
302                 if (do_udp4_gro) {
303                         i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
304                                         &pkts[i], nb_pkts - i);
305                 }
306                 /* Copy unprocessed packets */
307                 if (unprocess_num > 0) {
308                         memcpy(&pkts[i], unprocess_pkts,
309                                         sizeof(struct rte_mbuf *) *
310                                         unprocess_num);
311                 }
312                 nb_after_gro = i + unprocess_num;
313         }
314
315         return nb_after_gro;
316 }
317
318 uint16_t
319 rte_gro_reassemble(struct rte_mbuf **pkts,
320                 uint16_t nb_pkts,
321                 void *ctx)
322 {
323         struct rte_mbuf *unprocess_pkts[nb_pkts];
324         struct gro_ctx *gro_ctx = ctx;
325         void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
326         uint64_t current_time;
327         uint16_t i, unprocess_num = 0;
328         uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
329
330         if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
331                                         RTE_GRO_TCP_IPV4 |
332                                         RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
333                                         RTE_GRO_UDP_IPV4)) == 0))
334                 return nb_pkts;
335
336         tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX];
337         vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
338         udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
339         vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
340
341         do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
342                 RTE_GRO_TCP_IPV4;
343         do_vxlan_tcp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) ==
344                 RTE_GRO_IPV4_VXLAN_TCP_IPV4;
345         do_udp4_gro = (gro_ctx->gro_types & RTE_GRO_UDP_IPV4) ==
346                 RTE_GRO_UDP_IPV4;
347         do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
348                 RTE_GRO_IPV4_VXLAN_UDP_IPV4;
349
350         current_time = rte_rdtsc();
351
352         for (i = 0; i < nb_pkts; i++) {
353                 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
354                                 do_vxlan_tcp_gro) {
355                         if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tcp_tbl,
356                                                 current_time) < 0)
357                                 unprocess_pkts[unprocess_num++] = pkts[i];
358                 } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) &&
359                                 do_vxlan_udp_gro) {
360                         if (gro_vxlan_udp4_reassemble(pkts[i], vxlan_udp_tbl,
361                                                 current_time) < 0)
362                                 unprocess_pkts[unprocess_num++] = pkts[i];
363                 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
364                                 do_tcp4_gro) {
365                         if (gro_tcp4_reassemble(pkts[i], tcp_tbl,
366                                                 current_time) < 0)
367                                 unprocess_pkts[unprocess_num++] = pkts[i];
368                 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
369                                 do_udp4_gro) {
370                         if (gro_udp4_reassemble(pkts[i], udp_tbl,
371                                                 current_time) < 0)
372                                 unprocess_pkts[unprocess_num++] = pkts[i];
373                 } else
374                         unprocess_pkts[unprocess_num++] = pkts[i];
375         }
376         if (unprocess_num > 0) {
377                 memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) *
378                                 unprocess_num);
379         }
380
381         return unprocess_num;
382 }
383
384 uint16_t
385 rte_gro_timeout_flush(void *ctx,
386                 uint64_t timeout_cycles,
387                 uint64_t gro_types,
388                 struct rte_mbuf **out,
389                 uint16_t max_nb_out)
390 {
391         struct gro_ctx *gro_ctx = ctx;
392         uint64_t flush_timestamp;
393         uint16_t num = 0;
394         uint16_t left_nb_out = max_nb_out;
395
396         gro_types = gro_types & gro_ctx->gro_types;
397         flush_timestamp = rte_rdtsc() - timeout_cycles;
398
399         if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
400                 num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[
401                                 RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX],
402                                 flush_timestamp, out, left_nb_out);
403                 left_nb_out = max_nb_out - num;
404         }
405
406         if ((gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) && left_nb_out > 0) {
407                 num += gro_vxlan_udp4_tbl_timeout_flush(gro_ctx->tbls[
408                                 RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX],
409                                 flush_timestamp, &out[num], left_nb_out);
410                 left_nb_out = max_nb_out - num;
411         }
412
413         /* If no available space in 'out', stop flushing. */
414         if ((gro_types & RTE_GRO_TCP_IPV4) && left_nb_out > 0) {
415                 num += gro_tcp4_tbl_timeout_flush(
416                                 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
417                                 flush_timestamp,
418                                 &out[num], left_nb_out);
419                 left_nb_out = max_nb_out - num;
420         }
421
422         /* If no available space in 'out', stop flushing. */
423         if ((gro_types & RTE_GRO_UDP_IPV4) && left_nb_out > 0) {
424                 num += gro_udp4_tbl_timeout_flush(
425                                 gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
426                                 flush_timestamp,
427                                 &out[num], left_nb_out);
428         }
429
430         return num;
431 }
432
433 uint64_t
434 rte_gro_get_pkt_count(void *ctx)
435 {
436         struct gro_ctx *gro_ctx = ctx;
437         gro_tbl_pkt_count_fn pkt_count_fn;
438         uint64_t gro_types = gro_ctx->gro_types, flag;
439         uint64_t item_num = 0;
440         uint8_t i;
441
442         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) {
443                 flag = 1ULL << i;
444                 if ((gro_types & flag) == 0)
445                         continue;
446
447                 gro_types ^= flag;
448                 pkt_count_fn = tbl_pkt_count_fn[i];
449                 if (pkt_count_fn)
450                         item_num += pkt_count_fn(gro_ctx->tbls[i]);
451         }
452
453         return item_num;
454 }