-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2017 Intel Corporation. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
*/
#include <rte_malloc.h>
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_udp4.h"
+#include "gro_vxlan_tcp4.h"
+#include "gro_vxlan_udp4.h"
typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
uint16_t max_flow_num,
typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
- gro_tcp4_tbl_create, NULL};
+ gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
- gro_tcp4_tbl_destroy, NULL};
+ gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
+ gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
- gro_tcp4_tbl_pkt_count, NULL};
+ gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
+ gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ NULL};
+
+#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
+#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
+#define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
+ ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
+ RTE_PTYPE_TUNNEL_VXLAN) && \
+ ((ptype & RTE_PTYPE_INNER_L4_TCP) == \
+ RTE_PTYPE_INNER_L4_TCP) && \
+ (((ptype & RTE_PTYPE_INNER_L3_MASK) == \
+ RTE_PTYPE_INNER_L3_IPV4) || \
+ ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
+ RTE_PTYPE_INNER_L3_IPV4_EXT) || \
+ ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)))
+
+#define IS_IPV4_VXLAN_UDP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
+ ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
+ RTE_PTYPE_TUNNEL_VXLAN) && \
+ ((ptype & RTE_PTYPE_INNER_L4_UDP) == \
+ RTE_PTYPE_INNER_L4_UDP) && \
+ (((ptype & RTE_PTYPE_INNER_L3_MASK) == \
+ RTE_PTYPE_INNER_L3_IPV4) || \
+ ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
+ RTE_PTYPE_INNER_L3_IPV4_EXT) || \
+ ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)))
/*
- * GRO context structure, which is used to merge packets. It keeps
- * many reassembly tables of desired GRO types. Applications need to
- * create GRO context objects before using rte_gro_reassemble to
- * perform GRO.
+ * GRO context structure. It keeps the table structures, which are
+ * used to merge packets, for different GRO types. Before using
+ * rte_gro_reassemble(), applications need to create the GRO context
+ * first.
*/
struct gro_ctx {
/* GRO types to perform */
return NULL;
for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
- gro_type_flag = 1 << i;
+ gro_type_flag = 1ULL << i;
if ((param->gro_types & gro_type_flag) == 0)
continue;
uint64_t gro_type_flag;
uint8_t i;
- if (gro_ctx == NULL)
- return;
for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
- gro_type_flag = 1 << i;
+ gro_type_flag = 1ULL << i;
if ((gro_ctx->gro_types & gro_type_flag) == 0)
continue;
destroy_tbl_fn = tbl_destroy_fn[i];
uint16_t nb_pkts,
const struct rte_gro_param *param)
{
- uint16_t i;
- uint16_t nb_after_gro = nb_pkts;
- uint32_t item_num;
-
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
- struct gro_tcp4_key tcp_keys[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ /* allocate a reassembly table for UDP/IPv4 GRO */
+ struct gro_udp4_tbl udp_tbl;
+ struct gro_udp4_flow udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_udp4_item udp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ /* Allocate a reassembly table for VXLAN TCP GRO */
+ struct gro_vxlan_tcp4_tbl vxlan_tcp_tbl;
+ struct gro_vxlan_tcp4_flow vxlan_tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_vxlan_tcp4_item vxlan_tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
+ = {{{0}, 0, 0} };
+
+ /* Allocate a reassembly table for VXLAN UDP GRO */
+ struct gro_vxlan_udp4_tbl vxlan_udp_tbl;
+ struct gro_vxlan_udp4_flow vxlan_udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_vxlan_udp4_item vxlan_udp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
+ = {{{0}} };
+
struct rte_mbuf *unprocess_pkts[nb_pkts];
- uint16_t unprocess_num = 0;
+ uint32_t item_num;
int32_t ret;
- uint64_t current_time;
-
- if ((param->gro_types & RTE_GRO_TCP_IPV4) == 0)
+ uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
+ uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
+ do_vxlan_udp_gro = 0;
+
+ if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 |
+ RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
+ RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
- /* get the actual number of packets */
+ /* Get the maximum number of packets */
item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
- param->max_item_per_flow));
+ param->max_item_per_flow));
item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
- for (i = 0; i < item_num; i++)
- tcp_keys[i].start_index = INVALID_ARRAY_INDEX;
+ if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
+ for (i = 0; i < item_num; i++)
+ vxlan_tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ vxlan_tcp_tbl.flows = vxlan_tcp_flows;
+ vxlan_tcp_tbl.items = vxlan_tcp_items;
+ vxlan_tcp_tbl.flow_num = 0;
+ vxlan_tcp_tbl.item_num = 0;
+ vxlan_tcp_tbl.max_flow_num = item_num;
+ vxlan_tcp_tbl.max_item_num = item_num;
+ do_vxlan_tcp_gro = 1;
+ }
- tcp_tbl.keys = tcp_keys;
- tcp_tbl.items = tcp_items;
- tcp_tbl.key_num = 0;
- tcp_tbl.item_num = 0;
- tcp_tbl.max_key_num = item_num;
- tcp_tbl.max_item_num = item_num;
+ if (param->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) {
+ for (i = 0; i < item_num; i++)
+ vxlan_udp_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ vxlan_udp_tbl.flows = vxlan_udp_flows;
+ vxlan_udp_tbl.items = vxlan_udp_items;
+ vxlan_udp_tbl.flow_num = 0;
+ vxlan_udp_tbl.item_num = 0;
+ vxlan_udp_tbl.max_flow_num = item_num;
+ vxlan_udp_tbl.max_item_num = item_num;
+ do_vxlan_udp_gro = 1;
+ }
+
+ if (param->gro_types & RTE_GRO_TCP_IPV4) {
+ for (i = 0; i < item_num; i++)
+ tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp_tbl.flows = tcp_flows;
+ tcp_tbl.items = tcp_items;
+ tcp_tbl.flow_num = 0;
+ tcp_tbl.item_num = 0;
+ tcp_tbl.max_flow_num = item_num;
+ tcp_tbl.max_item_num = item_num;
+ do_tcp4_gro = 1;
+ }
+
+ if (param->gro_types & RTE_GRO_UDP_IPV4) {
+ for (i = 0; i < item_num; i++)
+ udp_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ udp_tbl.flows = udp_flows;
+ udp_tbl.items = udp_items;
+ udp_tbl.flow_num = 0;
+ udp_tbl.item_num = 0;
+ udp_tbl.max_flow_num = item_num;
+ udp_tbl.max_item_num = item_num;
+ do_udp4_gro = 1;
+ }
- current_time = rte_rdtsc();
for (i = 0; i < nb_pkts; i++) {
- if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 |
- RTE_PTYPE_L4_TCP)) ==
- (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) {
- ret = gro_tcp4_reassemble(pkts[i],
- &tcp_tbl,
- current_time);
+ /*
+ * The timestamp is ignored, since all packets
+ * will be flushed from the tables.
+ */
+ if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
+ do_vxlan_tcp_gro) {
+ ret = gro_vxlan_tcp4_reassemble(pkts[i],
+ &vxlan_tcp_tbl, 0);
+ if (ret > 0)
+ /* Merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) &&
+ do_vxlan_udp_gro) {
+ ret = gro_vxlan_udp4_reassemble(pkts[i],
+ &vxlan_udp_tbl, 0);
+ if (ret > 0)
+ /* Merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp4_gro) {
+ ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0);
if (ret > 0)
/* merge successfully */
nb_after_gro--;
- else if (ret < 0) {
- unprocess_pkts[unprocess_num++] =
- pkts[i];
- }
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
+ do_udp4_gro) {
+ ret = gro_udp4_reassemble(pkts[i], &udp_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
- /* re-arrange GROed packets */
- if (nb_after_gro < nb_pkts) {
- i = gro_tcp4_tbl_timeout_flush(&tcp_tbl, current_time,
- pkts, nb_pkts);
+ if ((nb_after_gro < nb_pkts)
+ || (unprocess_num < nb_pkts)) {
+ i = 0;
+ /* Flush all packets from the tables */
+ if (do_vxlan_tcp_gro) {
+ i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ 0, pkts, nb_pkts);
+ }
+
+ if (do_vxlan_udp_gro) {
+ i += gro_vxlan_udp4_tbl_timeout_flush(&vxlan_udp_tbl,
+ 0, &pkts[i], nb_pkts - i);
+
+ }
+
+ if (do_tcp4_gro) {
+ i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0,
+ &pkts[i], nb_pkts - i);
+ }
+
+ if (do_udp4_gro) {
+ i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
+ &pkts[i], nb_pkts - i);
+ }
+ /* Copy unprocessed packets */
if (unprocess_num > 0) {
memcpy(&pkts[i], unprocess_pkts,
sizeof(struct rte_mbuf *) *
unprocess_num);
}
+ nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
uint16_t nb_pkts,
void *ctx)
{
- uint16_t i, unprocess_num = 0;
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
uint64_t current_time;
+ uint16_t i, unprocess_num = 0;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
- if ((gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 0)
+ if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 |
+ RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
+ RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
+ tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX];
+ vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
+ udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
+ vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+
+ do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
+ RTE_GRO_TCP_IPV4;
+ do_vxlan_tcp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) ==
+ RTE_GRO_IPV4_VXLAN_TCP_IPV4;
+ do_udp4_gro = (gro_ctx->gro_types & RTE_GRO_UDP_IPV4) ==
+ RTE_GRO_UDP_IPV4;
+ do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
+ RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+
current_time = rte_rdtsc();
for (i = 0; i < nb_pkts; i++) {
- if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 |
- RTE_PTYPE_L4_TCP)) ==
- (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) {
- if (gro_tcp4_reassemble(pkts[i],
- gro_ctx->tbls
- [RTE_GRO_TCP_IPV4_INDEX],
+ if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
+ do_vxlan_tcp_gro) {
+ if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tcp_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) &&
+ do_vxlan_udp_gro) {
+ if (gro_vxlan_udp4_reassemble(pkts[i], vxlan_udp_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp4_gro) {
+ if (gro_tcp4_reassemble(pkts[i], tcp_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
+ do_udp4_gro) {
+ if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
if (unprocess_num > 0) {
- memcpy(pkts, unprocess_pkts,
- sizeof(struct rte_mbuf *) *
+ memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) *
unprocess_num);
}
{
struct gro_ctx *gro_ctx = ctx;
uint64_t flush_timestamp;
+ uint16_t num = 0;
+ uint16_t left_nb_out = max_nb_out;
gro_types = gro_types & gro_ctx->gro_types;
flush_timestamp = rte_rdtsc() - timeout_cycles;
- if (gro_types & RTE_GRO_TCP_IPV4) {
- return gro_tcp4_tbl_timeout_flush(
+ if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
+ num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[
+ RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX],
+ flush_timestamp, out, left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) && left_nb_out > 0) {
+ num += gro_vxlan_udp4_tbl_timeout_flush(gro_ctx->tbls[
+ RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX],
+ flush_timestamp, &out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ /* If no available space in 'out', stop flushing. */
+ if ((gro_types & RTE_GRO_TCP_IPV4) && left_nb_out > 0) {
+ num += gro_tcp4_tbl_timeout_flush(
gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
flush_timestamp,
- out, max_nb_out);
+ &out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ /* If no available space in 'out', stop flushing. */
+ if ((gro_types & RTE_GRO_UDP_IPV4) && left_nb_out > 0) {
+ num += gro_udp4_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
}
- return 0;
+
+ return num;
}
uint64_t
{
struct gro_ctx *gro_ctx = ctx;
gro_tbl_pkt_count_fn pkt_count_fn;
+ uint64_t gro_types = gro_ctx->gro_types, flag;
uint64_t item_num = 0;
- uint64_t gro_type_flag;
uint8_t i;
- for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
- gro_type_flag = 1 << i;
- if ((gro_ctx->gro_types & gro_type_flag) == 0)
+ for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) {
+ flag = 1ULL << i;
+ if ((gro_types & flag) == 0)
continue;
+ gro_types ^= flag;
pkt_count_fn = tbl_pkt_count_fn[i];
- if (pkt_count_fn == NULL)
- continue;
- item_num += pkt_count_fn(gro_ctx->tbls[i]);
+ if (pkt_count_fn)
+ item_num += pkt_count_fn(gro_ctx->tbls[i]);
}
+
return item_num;
}
git.droids-corp.org / dpdk.git / blobdiff