X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_gro%2Frte_gro.c;h=6618f4d32dbed8ab8c893597412116cb11054f5b;hb=fa21a6fec78592ec95b52f412f5d0514cde5f802;hp=fa6d7ce5b263eedfcb04ebd754f6e9b4fb7657bf;hpb=e996506a1c07f145aa15ea6756cb11f2c705d1d3;p=dpdk.git diff --git a/lib/librte_gro/rte_gro.c b/lib/librte_gro/rte_gro.c index fa6d7ce5b2..6618f4d32d 100644 --- a/lib/librte_gro/rte_gro.c +++ b/lib/librte_gro/rte_gro.c @@ -1,39 +1,15 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation */ #include #include +#include +#include #include "rte_gro.h" +#include "gro_tcp4.h" +#include "gro_vxlan_tcp4.h" typedef void *(*gro_tbl_create_fn)(uint16_t socket_id, uint16_t max_flow_num, @@ -41,15 +17,34 @@ typedef void *(*gro_tbl_create_fn)(uint16_t socket_id, typedef void (*gro_tbl_destroy_fn)(void *tbl); typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl); -static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM]; -static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM]; -static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM]; +static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = { + gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create, NULL}; +static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = { + gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy, + NULL}; +static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = { + gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count, + NULL}; + +#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ + ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP)) + +#define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ + ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \ + ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \ + RTE_PTYPE_TUNNEL_VXLAN) && \ + ((ptype & RTE_PTYPE_INNER_L4_TCP) == \ + RTE_PTYPE_INNER_L4_TCP) && \ + (((ptype & RTE_PTYPE_INNER_L3_MASK) & \ + (RTE_PTYPE_INNER_L3_IPV4 | \ + RTE_PTYPE_INNER_L3_IPV4_EXT | \ + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)) != 0)) /* - * GRO context structure, which is used to merge packets. It keeps - * many reassembly tables of desired GRO types. Applications need to - * create GRO context objects before using rte_gro_reassemble to - * perform GRO. + * GRO context structure. It keeps the table structures, which are + * used to merge packets, for different GRO types. Before using + * rte_gro_reassemble(), applications need to create the GRO context + * first. */ struct gro_ctx { /* GRO types to perform */ @@ -75,7 +70,7 @@ rte_gro_ctx_create(const struct rte_gro_param *param) return NULL; for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { - gro_type_flag = 1 << i; + gro_type_flag = 1ULL << i; if ((param->gro_types & gro_type_flag) == 0) continue; @@ -107,10 +102,8 @@ rte_gro_ctx_destroy(void *ctx) uint64_t gro_type_flag; uint8_t i; - if (gro_ctx == NULL) - return; for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { - gro_type_flag = 1 << i; + gro_type_flag = 1ULL << i; if ((gro_ctx->gro_types & gro_type_flag) == 0) continue; destroy_tbl_fn = tbl_destroy_fn[i]; @@ -121,29 +114,187 @@ rte_gro_ctx_destroy(void *ctx) } uint16_t -rte_gro_reassemble_burst(struct rte_mbuf **pkts __rte_unused, +rte_gro_reassemble_burst(struct rte_mbuf **pkts, uint16_t nb_pkts, - const struct rte_gro_param *param __rte_unused) + const struct rte_gro_param *param) { - return nb_pkts; + /* allocate a reassembly table for TCP/IPv4 GRO */ + struct gro_tcp4_tbl tcp_tbl; + struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM]; + struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} }; + + /* Allocate a reassembly table for VXLAN GRO */ + struct gro_vxlan_tcp4_tbl vxlan_tbl; + struct gro_vxlan_tcp4_flow vxlan_flows[RTE_GRO_MAX_BURST_ITEM_NUM]; + struct gro_vxlan_tcp4_item vxlan_items[RTE_GRO_MAX_BURST_ITEM_NUM] = { + {{0}, 0, 0} }; + + struct rte_mbuf *unprocess_pkts[nb_pkts]; + uint32_t item_num; + int32_t ret; + uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts; + uint8_t do_tcp4_gro = 0, do_vxlan_gro = 0; + + if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 | + RTE_GRO_TCP_IPV4)) == 0)) + return nb_pkts; + + /* Get the maximum number of packets */ + item_num = RTE_MIN(nb_pkts, (param->max_flow_num * + param->max_item_per_flow)); + item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM); + + if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) { + for (i = 0; i < item_num; i++) + vxlan_flows[i].start_index = INVALID_ARRAY_INDEX; + + vxlan_tbl.flows = vxlan_flows; + vxlan_tbl.items = vxlan_items; + vxlan_tbl.flow_num = 0; + vxlan_tbl.item_num = 0; + vxlan_tbl.max_flow_num = item_num; + vxlan_tbl.max_item_num = item_num; + do_vxlan_gro = 1; + } + + if (param->gro_types & RTE_GRO_TCP_IPV4) { + for (i = 0; i < item_num; i++) + tcp_flows[i].start_index = INVALID_ARRAY_INDEX; + + tcp_tbl.flows = tcp_flows; + tcp_tbl.items = tcp_items; + tcp_tbl.flow_num = 0; + tcp_tbl.item_num = 0; + tcp_tbl.max_flow_num = item_num; + tcp_tbl.max_item_num = item_num; + do_tcp4_gro = 1; + } + + for (i = 0; i < nb_pkts; i++) { + /* + * The timestamp is ignored, since all packets + * will be flushed from the tables. + */ + if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) && + do_vxlan_gro) { + ret = gro_vxlan_tcp4_reassemble(pkts[i], &vxlan_tbl, 0); + if (ret > 0) + /* Merge successfully */ + nb_after_gro--; + else if (ret < 0) + unprocess_pkts[unprocess_num++] = pkts[i]; + } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) && + do_tcp4_gro) { + ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0); + if (ret > 0) + /* merge successfully */ + nb_after_gro--; + else if (ret < 0) + unprocess_pkts[unprocess_num++] = pkts[i]; + } else + unprocess_pkts[unprocess_num++] = pkts[i]; + } + + if (nb_after_gro < nb_pkts) { + i = 0; + /* Flush all packets from the tables */ + if (do_vxlan_gro) { + i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tbl, + 0, pkts, nb_pkts); + } + if (do_tcp4_gro) { + i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0, + &pkts[i], nb_pkts - i); + } + /* Copy unprocessed packets */ + if (unprocess_num > 0) { + memcpy(&pkts[i], unprocess_pkts, + sizeof(struct rte_mbuf *) * + unprocess_num); + } + } + + return nb_after_gro; } uint16_t -rte_gro_reassemble(struct rte_mbuf **pkts __rte_unused, +rte_gro_reassemble(struct rte_mbuf **pkts, uint16_t nb_pkts, - void *ctx __rte_unused) + void *ctx) { - return nb_pkts; + struct rte_mbuf *unprocess_pkts[nb_pkts]; + struct gro_ctx *gro_ctx = ctx; + void *tcp_tbl, *vxlan_tbl; + uint64_t current_time; + uint16_t i, unprocess_num = 0; + uint8_t do_tcp4_gro, do_vxlan_gro; + + if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 | + RTE_GRO_TCP_IPV4)) == 0)) + return nb_pkts; + + tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX]; + vxlan_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX]; + + do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == + RTE_GRO_TCP_IPV4; + do_vxlan_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) == + RTE_GRO_IPV4_VXLAN_TCP_IPV4; + + current_time = rte_rdtsc(); + + for (i = 0; i < nb_pkts; i++) { + if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) && + do_vxlan_gro) { + if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tbl, + current_time) < 0) + unprocess_pkts[unprocess_num++] = pkts[i]; + } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) && + do_tcp4_gro) { + if (gro_tcp4_reassemble(pkts[i], tcp_tbl, + current_time) < 0) + unprocess_pkts[unprocess_num++] = pkts[i]; + } else + unprocess_pkts[unprocess_num++] = pkts[i]; + } + if (unprocess_num > 0) { + memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) * + unprocess_num); + } + + return unprocess_num; } uint16_t -rte_gro_timeout_flush(void *ctx __rte_unused, - uint64_t timeout_cycles __rte_unused, - uint64_t gro_types __rte_unused, - struct rte_mbuf **out __rte_unused, - uint16_t max_nb_out __rte_unused) +rte_gro_timeout_flush(void *ctx, + uint64_t timeout_cycles, + uint64_t gro_types, + struct rte_mbuf **out, + uint16_t max_nb_out) { - return 0; + struct gro_ctx *gro_ctx = ctx; + uint64_t flush_timestamp; + uint16_t num = 0; + + gro_types = gro_types & gro_ctx->gro_types; + flush_timestamp = rte_rdtsc() - timeout_cycles; + + if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) { + num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[ + RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX], + flush_timestamp, out, max_nb_out); + max_nb_out -= num; + } + + /* If no available space in 'out', stop flushing. */ + if ((gro_types & RTE_GRO_TCP_IPV4) && max_nb_out > 0) { + num += gro_tcp4_tbl_timeout_flush( + gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX], + flush_timestamp, + &out[num], max_nb_out); + } + + return num; } uint64_t @@ -151,19 +302,20 @@ rte_gro_get_pkt_count(void *ctx) { struct gro_ctx *gro_ctx = ctx; gro_tbl_pkt_count_fn pkt_count_fn; + uint64_t gro_types = gro_ctx->gro_types, flag; uint64_t item_num = 0; - uint64_t gro_type_flag; uint8_t i; - for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { - gro_type_flag = 1 << i; - if ((gro_ctx->gro_types & gro_type_flag) == 0) + for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) { + flag = 1ULL << i; + if ((gro_types & flag) == 0) continue; + gro_types ^= flag; pkt_count_fn = tbl_pkt_count_fn[i]; - if (pkt_count_fn == NULL) - continue; - item_num += pkt_count_fn(gro_ctx->tbls[i]); + if (pkt_count_fn) + item_num += pkt_count_fn(gro_ctx->tbls[i]); } + return item_num; }