lib/librte_gro/rte_gro.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cycles.h>
   8 #include <rte_ethdev.h>
   9
  10 #include "rte_gro.h"
  11 #include "gro_tcp4.h"
  12
  13 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
  14                 uint16_t max_flow_num,
  15                 uint16_t max_item_per_flow);
  16 typedef void (*gro_tbl_destroy_fn)(void *tbl);
  17 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
  18
  19 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
  20                 gro_tcp4_tbl_create, NULL};
  21 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
  22                         gro_tcp4_tbl_destroy, NULL};
  23 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
  24                         gro_tcp4_tbl_pkt_count, NULL};
  25
  26 #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
  27                 ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP))
  28
  29 /*
  30  * GRO context structure. It keeps the table structures, which are
  31  * used to merge packets, for different GRO types. Before using
  32  * rte_gro_reassemble(), applications need to create the GRO context
  33  * first.
  34  */
  35 struct gro_ctx {
  36         /* GRO types to perform */
  37         uint64_t gro_types;
  38         /* reassembly tables */
  39         void *tbls[RTE_GRO_TYPE_MAX_NUM];
  40 };
  41
  42 void *
  43 rte_gro_ctx_create(const struct rte_gro_param *param)
  44 {
  45         struct gro_ctx *gro_ctx;
  46         gro_tbl_create_fn create_tbl_fn;
  47         uint64_t gro_type_flag = 0;
  48         uint64_t gro_types = 0;
  49         uint8_t i;
  50
  51         gro_ctx = rte_zmalloc_socket(__func__,
  52                         sizeof(struct gro_ctx),
  53                         RTE_CACHE_LINE_SIZE,
  54                         param->socket_id);
  55         if (gro_ctx == NULL)
  56                 return NULL;
  57
  58         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
  59                 gro_type_flag = 1ULL << i;
  60                 if ((param->gro_types & gro_type_flag) == 0)
  61                         continue;
  62
  63                 create_tbl_fn = tbl_create_fn[i];
  64                 if (create_tbl_fn == NULL)
  65                         continue;
  66
  67                 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id,
  68                                 param->max_flow_num,
  69                                 param->max_item_per_flow);
  70                 if (gro_ctx->tbls[i] == NULL) {
  71                         /* destroy all created tables */
  72                         gro_ctx->gro_types = gro_types;
  73                         rte_gro_ctx_destroy(gro_ctx);
  74                         return NULL;
  75                 }
  76                 gro_types |= gro_type_flag;
  77         }
  78         gro_ctx->gro_types = param->gro_types;
  79
  80         return gro_ctx;
  81 }
  82
  83 void
  84 rte_gro_ctx_destroy(void *ctx)
  85 {
  86         gro_tbl_destroy_fn destroy_tbl_fn;
  87         struct gro_ctx *gro_ctx = ctx;
  88         uint64_t gro_type_flag;
  89         uint8_t i;
  90
  91         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
  92                 gro_type_flag = 1ULL << i;
  93                 if ((gro_ctx->gro_types & gro_type_flag) == 0)
  94                         continue;
  95                 destroy_tbl_fn = tbl_destroy_fn[i];
  96                 if (destroy_tbl_fn)
  97                         destroy_tbl_fn(gro_ctx->tbls[i]);
  98         }
  99         rte_free(gro_ctx);
 100 }
 101
 102 uint16_t
 103 rte_gro_reassemble_burst(struct rte_mbuf **pkts,
 104                 uint16_t nb_pkts,
 105                 const struct rte_gro_param *param)
 106 {
 107         /* allocate a reassembly table for TCP/IPv4 GRO */
 108         struct gro_tcp4_tbl tcp_tbl;
 109         struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 110         struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
 111
 112         struct rte_mbuf *unprocess_pkts[nb_pkts];
 113         uint32_t item_num;
 114         int32_t ret;
 115         uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
 116
 117         if (unlikely((param->gro_types & RTE_GRO_TCP_IPV4) == 0))
 118                 return nb_pkts;
 119
 120         /* Get the maximum number of packets */
 121         item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
 122                                 param->max_item_per_flow));
 123         item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
 124
 125         for (i = 0; i < item_num; i++)
 126                 tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
 127
 128         tcp_tbl.flows = tcp_flows;
 129         tcp_tbl.items = tcp_items;
 130         tcp_tbl.flow_num = 0;
 131         tcp_tbl.item_num = 0;
 132         tcp_tbl.max_flow_num = item_num;
 133         tcp_tbl.max_item_num = item_num;
 134
 135         for (i = 0; i < nb_pkts; i++) {
 136                 if (IS_IPV4_TCP_PKT(pkts[i]->packet_type)) {
 137                         /*
 138                          * The timestamp is ignored, since all packets
 139                          * will be flushed from the tables.
 140                          */
 141                         ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0);
 142                         if (ret > 0)
 143                                 /* merge successfully */
 144                                 nb_after_gro--;
 145                         else if (ret < 0)
 146                                 unprocess_pkts[unprocess_num++] = pkts[i];
 147                 } else
 148                         unprocess_pkts[unprocess_num++] = pkts[i];
 149         }
 150
 151         if (nb_after_gro < nb_pkts) {
 152                 /* Flush all packets from the tables */
 153                 i = gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0, pkts, nb_pkts);
 154                 /* Copy unprocessed packets */
 155                 if (unprocess_num > 0) {
 156                         memcpy(&pkts[i], unprocess_pkts,
 157                                         sizeof(struct rte_mbuf *) *
 158                                         unprocess_num);
 159                 }
 160         }
 161
 162         return nb_after_gro;
 163 }
 164
 165 uint16_t
 166 rte_gro_reassemble(struct rte_mbuf **pkts,
 167                 uint16_t nb_pkts,
 168                 void *ctx)
 169 {
 170         struct rte_mbuf *unprocess_pkts[nb_pkts];
 171         struct gro_ctx *gro_ctx = ctx;
 172         void *tcp_tbl;
 173         uint64_t current_time;
 174         uint16_t i, unprocess_num = 0;
 175
 176         if (unlikely((gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 0))
 177                 return nb_pkts;
 178
 179         tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX];
 180         current_time = rte_rdtsc();
 181
 182         for (i = 0; i < nb_pkts; i++) {
 183                 if (IS_IPV4_TCP_PKT(pkts[i]->packet_type)) {
 184                         if (gro_tcp4_reassemble(pkts[i], tcp_tbl,
 185                                                 current_time) < 0)
 186                                 unprocess_pkts[unprocess_num++] = pkts[i];
 187                 } else
 188                         unprocess_pkts[unprocess_num++] = pkts[i];
 189         }
 190         if (unprocess_num > 0) {
 191                 memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) *
 192                                 unprocess_num);
 193         }
 194
 195         return unprocess_num;
 196 }
 197
 198 uint16_t
 199 rte_gro_timeout_flush(void *ctx,
 200                 uint64_t timeout_cycles,
 201                 uint64_t gro_types,
 202                 struct rte_mbuf **out,
 203                 uint16_t max_nb_out)
 204 {
 205         struct gro_ctx *gro_ctx = ctx;
 206         uint64_t flush_timestamp;
 207
 208         gro_types = gro_types & gro_ctx->gro_types;
 209         flush_timestamp = rte_rdtsc() - timeout_cycles;
 210
 211         if (gro_types & RTE_GRO_TCP_IPV4) {
 212                 return gro_tcp4_tbl_timeout_flush(
 213                                 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
 214                                 flush_timestamp,
 215                                 out, max_nb_out);
 216         }
 217
 218         return 0;
 219 }
 220
 221 uint64_t
 222 rte_gro_get_pkt_count(void *ctx)
 223 {
 224         struct gro_ctx *gro_ctx = ctx;
 225         gro_tbl_pkt_count_fn pkt_count_fn;
 226         uint64_t gro_types = gro_ctx->gro_types, flag;
 227         uint64_t item_num = 0;
 228         uint8_t i;
 229
 230         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) {
 231                 flag = 1ULL << i;
 232                 if ((gro_types & flag) == 0)
 233                         continue;
 234
 235                 gro_types ^= flag;
 236                 pkt_count_fn = tbl_pkt_count_fn[i];
 237                 if (pkt_count_fn)
 238                         item_num += pkt_count_fn(gro_ctx->tbls[i]);
 239         }
 240
 241         return item_num;
 242 }