lib/librte_gro/rte_gro.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cycles.h>
   8 #include <rte_ethdev.h>
   9
  10 #include "rte_gro.h"
  11 #include "gro_tcp4.h"
  12
  13 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
  14                 uint16_t max_flow_num,
  15                 uint16_t max_item_per_flow);
  16 typedef void (*gro_tbl_destroy_fn)(void *tbl);
  17 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
  18
  19 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
  20                 gro_tcp4_tbl_create, NULL};
  21 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
  22                         gro_tcp4_tbl_destroy, NULL};
  23 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
  24                         gro_tcp4_tbl_pkt_count, NULL};
  25
  26 /*
  27  * GRO context structure, which is used to merge packets. It keeps
  28  * many reassembly tables of desired GRO types. Applications need to
  29  * create GRO context objects before using rte_gro_reassemble to
  30  * perform GRO.
  31  */
  32 struct gro_ctx {
  33         /* GRO types to perform */
  34         uint64_t gro_types;
  35         /* reassembly tables */
  36         void *tbls[RTE_GRO_TYPE_MAX_NUM];
  37 };
  38
  39 void *
  40 rte_gro_ctx_create(const struct rte_gro_param *param)
  41 {
  42         struct gro_ctx *gro_ctx;
  43         gro_tbl_create_fn create_tbl_fn;
  44         uint64_t gro_type_flag = 0;
  45         uint64_t gro_types = 0;
  46         uint8_t i;
  47
  48         gro_ctx = rte_zmalloc_socket(__func__,
  49                         sizeof(struct gro_ctx),
  50                         RTE_CACHE_LINE_SIZE,
  51                         param->socket_id);
  52         if (gro_ctx == NULL)
  53                 return NULL;
  54
  55         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
  56                 gro_type_flag = 1ULL << i;
  57                 if ((param->gro_types & gro_type_flag) == 0)
  58                         continue;
  59
  60                 create_tbl_fn = tbl_create_fn[i];
  61                 if (create_tbl_fn == NULL)
  62                         continue;
  63
  64                 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id,
  65                                 param->max_flow_num,
  66                                 param->max_item_per_flow);
  67                 if (gro_ctx->tbls[i] == NULL) {
  68                         /* destroy all created tables */
  69                         gro_ctx->gro_types = gro_types;
  70                         rte_gro_ctx_destroy(gro_ctx);
  71                         return NULL;
  72                 }
  73                 gro_types |= gro_type_flag;
  74         }
  75         gro_ctx->gro_types = param->gro_types;
  76
  77         return gro_ctx;
  78 }
  79
  80 void
  81 rte_gro_ctx_destroy(void *ctx)
  82 {
  83         gro_tbl_destroy_fn destroy_tbl_fn;
  84         struct gro_ctx *gro_ctx = ctx;
  85         uint64_t gro_type_flag;
  86         uint8_t i;
  87
  88         if (gro_ctx == NULL)
  89                 return;
  90         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
  91                 gro_type_flag = 1ULL << i;
  92                 if ((gro_ctx->gro_types & gro_type_flag) == 0)
  93                         continue;
  94                 destroy_tbl_fn = tbl_destroy_fn[i];
  95                 if (destroy_tbl_fn)
  96                         destroy_tbl_fn(gro_ctx->tbls[i]);
  97         }
  98         rte_free(gro_ctx);
  99 }
 100
 101 uint16_t
 102 rte_gro_reassemble_burst(struct rte_mbuf **pkts,
 103                 uint16_t nb_pkts,
 104                 const struct rte_gro_param *param)
 105 {
 106         uint16_t i;
 107         uint16_t nb_after_gro = nb_pkts;
 108         uint32_t item_num;
 109
 110         /* allocate a reassembly table for TCP/IPv4 GRO */
 111         struct gro_tcp4_tbl tcp_tbl;
 112         struct gro_tcp4_key tcp_keys[RTE_GRO_MAX_BURST_ITEM_NUM];
 113         struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
 114
 115         struct rte_mbuf *unprocess_pkts[nb_pkts];
 116         uint16_t unprocess_num = 0;
 117         int32_t ret;
 118         uint64_t current_time;
 119
 120         if ((param->gro_types & RTE_GRO_TCP_IPV4) == 0)
 121                 return nb_pkts;
 122
 123         /* get the actual number of packets */
 124         item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
 125                         param->max_item_per_flow));
 126         item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
 127
 128         for (i = 0; i < item_num; i++)
 129                 tcp_keys[i].start_index = INVALID_ARRAY_INDEX;
 130
 131         tcp_tbl.keys = tcp_keys;
 132         tcp_tbl.items = tcp_items;
 133         tcp_tbl.key_num = 0;
 134         tcp_tbl.item_num = 0;
 135         tcp_tbl.max_key_num = item_num;
 136         tcp_tbl.max_item_num = item_num;
 137
 138         current_time = rte_rdtsc();
 139
 140         for (i = 0; i < nb_pkts; i++) {
 141                 if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 |
 142                                         RTE_PTYPE_L4_TCP)) ==
 143                                 (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) {
 144                         ret = gro_tcp4_reassemble(pkts[i],
 145                                         &tcp_tbl,
 146                                         current_time);
 147                         if (ret > 0)
 148                                 /* merge successfully */
 149                                 nb_after_gro--;
 150                         else if (ret < 0) {
 151                                 unprocess_pkts[unprocess_num++] =
 152                                         pkts[i];
 153                         }
 154                 } else
 155                         unprocess_pkts[unprocess_num++] = pkts[i];
 156         }
 157
 158         /* re-arrange GROed packets */
 159         if (nb_after_gro < nb_pkts) {
 160                 i = gro_tcp4_tbl_timeout_flush(&tcp_tbl, current_time,
 161                                 pkts, nb_pkts);
 162                 if (unprocess_num > 0) {
 163                         memcpy(&pkts[i], unprocess_pkts,
 164                                         sizeof(struct rte_mbuf *) *
 165                                         unprocess_num);
 166                 }
 167         }
 168
 169         return nb_after_gro;
 170 }
 171
 172 uint16_t
 173 rte_gro_reassemble(struct rte_mbuf **pkts,
 174                 uint16_t nb_pkts,
 175                 void *ctx)
 176 {
 177         uint16_t i, unprocess_num = 0;
 178         struct rte_mbuf *unprocess_pkts[nb_pkts];
 179         struct gro_ctx *gro_ctx = ctx;
 180         uint64_t current_time;
 181
 182         if ((gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 0)
 183                 return nb_pkts;
 184
 185         current_time = rte_rdtsc();
 186
 187         for (i = 0; i < nb_pkts; i++) {
 188                 if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 |
 189                                         RTE_PTYPE_L4_TCP)) ==
 190                                 (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) {
 191                         if (gro_tcp4_reassemble(pkts[i],
 192                                                 gro_ctx->tbls
 193                                                 [RTE_GRO_TCP_IPV4_INDEX],
 194                                                 current_time) < 0)
 195                                 unprocess_pkts[unprocess_num++] = pkts[i];
 196                 } else
 197                         unprocess_pkts[unprocess_num++] = pkts[i];
 198         }
 199         if (unprocess_num > 0) {
 200                 memcpy(pkts, unprocess_pkts,
 201                                 sizeof(struct rte_mbuf *) *
 202                                 unprocess_num);
 203         }
 204
 205         return unprocess_num;
 206 }
 207
 208 uint16_t
 209 rte_gro_timeout_flush(void *ctx,
 210                 uint64_t timeout_cycles,
 211                 uint64_t gro_types,
 212                 struct rte_mbuf **out,
 213                 uint16_t max_nb_out)
 214 {
 215         struct gro_ctx *gro_ctx = ctx;
 216         uint64_t flush_timestamp;
 217
 218         gro_types = gro_types & gro_ctx->gro_types;
 219         flush_timestamp = rte_rdtsc() - timeout_cycles;
 220
 221         if (gro_types & RTE_GRO_TCP_IPV4) {
 222                 return gro_tcp4_tbl_timeout_flush(
 223                                 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
 224                                 flush_timestamp,
 225                                 out, max_nb_out);
 226         }
 227         return 0;
 228 }
 229
 230 uint64_t
 231 rte_gro_get_pkt_count(void *ctx)
 232 {
 233         struct gro_ctx *gro_ctx = ctx;
 234         gro_tbl_pkt_count_fn pkt_count_fn;
 235         uint64_t item_num = 0;
 236         uint64_t gro_type_flag;
 237         uint8_t i;
 238
 239         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
 240                 gro_type_flag = 1ULL << i;
 241                 if ((gro_ctx->gro_types & gro_type_flag) == 0)
 242                         continue;
 243
 244                 pkt_count_fn = tbl_pkt_count_fn[i];
 245                 if (pkt_count_fn == NULL)
 246                         continue;
 247                 item_num += pkt_count_fn(gro_ctx->tbls[i]);
 248         }
 249         return item_num;
 250 }