lib/librte_gro/rte_gro.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cycles.h>
   8 #include <rte_ethdev.h>
   9
  10 #include "rte_gro.h"
  11 #include "gro_tcp4.h"
  12 #include "gro_udp4.h"
  13 #include "gro_vxlan_tcp4.h"
  14
  15 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
  16                 uint16_t max_flow_num,
  17                 uint16_t max_item_per_flow);
  18 typedef void (*gro_tbl_destroy_fn)(void *tbl);
  19 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
  20
  21 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
  22                 gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
  23                 gro_udp4_tbl_create, NULL};
  24 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
  25                         gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
  26                         gro_udp4_tbl_destroy,
  27                         NULL};
  28 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
  29                         gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
  30                         gro_udp4_tbl_pkt_count,
  31                         NULL};
  32
  33 #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
  34                 ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP))
  35
  36 #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
  37                 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP))
  38
  39 #define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
  40                 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
  41                 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
  42                  RTE_PTYPE_TUNNEL_VXLAN) && \
  43                  ((ptype & RTE_PTYPE_INNER_L4_TCP) == \
  44                   RTE_PTYPE_INNER_L4_TCP) && \
  45                   (((ptype & RTE_PTYPE_INNER_L3_MASK) & \
  46                     (RTE_PTYPE_INNER_L3_IPV4 | \
  47                      RTE_PTYPE_INNER_L3_IPV4_EXT | \
  48                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)) != 0))
  49
  50
  51 /*
  52  * GRO context structure. It keeps the table structures, which are
  53  * used to merge packets, for different GRO types. Before using
  54  * rte_gro_reassemble(), applications need to create the GRO context
  55  * first.
  56  */
  57 struct gro_ctx {
  58         /* GRO types to perform */
  59         uint64_t gro_types;
  60         /* reassembly tables */
  61         void *tbls[RTE_GRO_TYPE_MAX_NUM];
  62 };
  63
  64 void *
  65 rte_gro_ctx_create(const struct rte_gro_param *param)
  66 {
  67         struct gro_ctx *gro_ctx;
  68         gro_tbl_create_fn create_tbl_fn;
  69         uint64_t gro_type_flag = 0;
  70         uint64_t gro_types = 0;
  71         uint8_t i;
  72
  73         gro_ctx = rte_zmalloc_socket(__func__,
  74                         sizeof(struct gro_ctx),
  75                         RTE_CACHE_LINE_SIZE,
  76                         param->socket_id);
  77         if (gro_ctx == NULL)
  78                 return NULL;
  79
  80         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
  81                 gro_type_flag = 1ULL << i;
  82                 if ((param->gro_types & gro_type_flag) == 0)
  83                         continue;
  84
  85                 create_tbl_fn = tbl_create_fn[i];
  86                 if (create_tbl_fn == NULL)
  87                         continue;
  88
  89                 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id,
  90                                 param->max_flow_num,
  91                                 param->max_item_per_flow);
  92                 if (gro_ctx->tbls[i] == NULL) {
  93                         /* destroy all created tables */
  94                         gro_ctx->gro_types = gro_types;
  95                         rte_gro_ctx_destroy(gro_ctx);
  96                         return NULL;
  97                 }
  98                 gro_types |= gro_type_flag;
  99         }
 100         gro_ctx->gro_types = param->gro_types;
 101
 102         return gro_ctx;
 103 }
 104
 105 void
 106 rte_gro_ctx_destroy(void *ctx)
 107 {
 108         gro_tbl_destroy_fn destroy_tbl_fn;
 109         struct gro_ctx *gro_ctx = ctx;
 110         uint64_t gro_type_flag;
 111         uint8_t i;
 112
 113         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
 114                 gro_type_flag = 1ULL << i;
 115                 if ((gro_ctx->gro_types & gro_type_flag) == 0)
 116                         continue;
 117                 destroy_tbl_fn = tbl_destroy_fn[i];
 118                 if (destroy_tbl_fn)
 119                         destroy_tbl_fn(gro_ctx->tbls[i]);
 120         }
 121         rte_free(gro_ctx);
 122 }
 123
 124 uint16_t
 125 rte_gro_reassemble_burst(struct rte_mbuf **pkts,
 126                 uint16_t nb_pkts,
 127                 const struct rte_gro_param *param)
 128 {
 129         /* allocate a reassembly table for TCP/IPv4 GRO */
 130         struct gro_tcp4_tbl tcp_tbl;
 131         struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 132         struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
 133
 134         /* allocate a reassembly table for UDP/IPv4 GRO */
 135         struct gro_udp4_tbl udp_tbl;
 136         struct gro_udp4_flow udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 137         struct gro_udp4_item udp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
 138
 139         /* Allocate a reassembly table for VXLAN TCP GRO */
 140         struct gro_vxlan_tcp4_tbl vxlan_tbl;
 141         struct gro_vxlan_tcp4_flow vxlan_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 142         struct gro_vxlan_tcp4_item vxlan_items[RTE_GRO_MAX_BURST_ITEM_NUM]
 143                         = {{{0}, 0, 0} };
 144
 145         struct rte_mbuf *unprocess_pkts[nb_pkts];
 146         uint32_t item_num;
 147         int32_t ret;
 148         uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
 149         uint8_t do_tcp4_gro = 0, do_vxlan_gro = 0, do_udp4_gro = 0;
 150
 151         if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
 152                                         RTE_GRO_TCP_IPV4 |
 153                                         RTE_GRO_UDP_IPV4)) == 0))
 154                 return nb_pkts;
 155
 156         /* Get the maximum number of packets */
 157         item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
 158                                 param->max_item_per_flow));
 159         item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
 160
 161         if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
 162                 for (i = 0; i < item_num; i++)
 163                         vxlan_flows[i].start_index = INVALID_ARRAY_INDEX;
 164
 165                 vxlan_tbl.flows = vxlan_flows;
 166                 vxlan_tbl.items = vxlan_items;
 167                 vxlan_tbl.flow_num = 0;
 168                 vxlan_tbl.item_num = 0;
 169                 vxlan_tbl.max_flow_num = item_num;
 170                 vxlan_tbl.max_item_num = item_num;
 171                 do_vxlan_gro = 1;
 172         }
 173
 174         if (param->gro_types & RTE_GRO_TCP_IPV4) {
 175                 for (i = 0; i < item_num; i++)
 176                         tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
 177
 178                 tcp_tbl.flows = tcp_flows;
 179                 tcp_tbl.items = tcp_items;
 180                 tcp_tbl.flow_num = 0;
 181                 tcp_tbl.item_num = 0;
 182                 tcp_tbl.max_flow_num = item_num;
 183                 tcp_tbl.max_item_num = item_num;
 184                 do_tcp4_gro = 1;
 185         }
 186
 187         if (param->gro_types & RTE_GRO_UDP_IPV4) {
 188                 for (i = 0; i < item_num; i++)
 189                         udp_flows[i].start_index = INVALID_ARRAY_INDEX;
 190
 191                 udp_tbl.flows = udp_flows;
 192                 udp_tbl.items = udp_items;
 193                 udp_tbl.flow_num = 0;
 194                 udp_tbl.item_num = 0;
 195                 udp_tbl.max_flow_num = item_num;
 196                 udp_tbl.max_item_num = item_num;
 197                 do_udp4_gro = 1;
 198         }
 199
 200
 201         for (i = 0; i < nb_pkts; i++) {
 202                 /*
 203                  * The timestamp is ignored, since all packets
 204                  * will be flushed from the tables.
 205                  */
 206                 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
 207                                 do_vxlan_gro) {
 208                         ret = gro_vxlan_tcp4_reassemble(pkts[i],
 209                                                         &vxlan_tbl, 0);
 210                         if (ret > 0)
 211                                 /* Merge successfully */
 212                                 nb_after_gro--;
 213                         else if (ret < 0)
 214                                 unprocess_pkts[unprocess_num++] = pkts[i];
 215                 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
 216                                 do_tcp4_gro) {
 217                         ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0);
 218                         if (ret > 0)
 219                                 /* merge successfully */
 220                                 nb_after_gro--;
 221                         else if (ret < 0)
 222                                 unprocess_pkts[unprocess_num++] = pkts[i];
 223                 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
 224                                 do_udp4_gro) {
 225                         ret = gro_udp4_reassemble(pkts[i], &udp_tbl, 0);
 226                         if (ret > 0)
 227                                 /* merge successfully */
 228                                 nb_after_gro--;
 229                         else if (ret < 0)
 230                                 unprocess_pkts[unprocess_num++] = pkts[i];
 231                 } else
 232                         unprocess_pkts[unprocess_num++] = pkts[i];
 233         }
 234
 235         if ((nb_after_gro < nb_pkts)
 236                  || (unprocess_num < nb_pkts)) {
 237                 i = 0;
 238                 /* Flush all packets from the tables */
 239                 if (do_vxlan_gro) {
 240                         i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tbl,
 241                                         0, pkts, nb_pkts);
 242                 }
 243
 244                 if (do_tcp4_gro) {
 245                         i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0,
 246                                         &pkts[i], nb_pkts - i);
 247                 }
 248
 249                 if (do_udp4_gro) {
 250                         i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
 251                                         &pkts[i], nb_pkts - i);
 252                 }
 253                 /* Copy unprocessed packets */
 254                 if (unprocess_num > 0) {
 255                         memcpy(&pkts[i], unprocess_pkts,
 256                                         sizeof(struct rte_mbuf *) *
 257                                         unprocess_num);
 258                 }
 259                 nb_after_gro = i + unprocess_num;
 260         }
 261
 262         return nb_after_gro;
 263 }
 264
 265 uint16_t
 266 rte_gro_reassemble(struct rte_mbuf **pkts,
 267                 uint16_t nb_pkts,
 268                 void *ctx)
 269 {
 270         struct rte_mbuf *unprocess_pkts[nb_pkts];
 271         struct gro_ctx *gro_ctx = ctx;
 272         void *tcp_tbl, *udp_tbl, *vxlan_tbl;
 273         uint64_t current_time;
 274         uint16_t i, unprocess_num = 0;
 275         uint8_t do_tcp4_gro, do_vxlan_gro, do_udp4_gro;
 276
 277         if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
 278                                         RTE_GRO_TCP_IPV4 |
 279                                         RTE_GRO_UDP_IPV4)) == 0))
 280                 return nb_pkts;
 281
 282         tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX];
 283         vxlan_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
 284         udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
 285
 286         do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
 287                 RTE_GRO_TCP_IPV4;
 288         do_vxlan_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) ==
 289                 RTE_GRO_IPV4_VXLAN_TCP_IPV4;
 290         do_udp4_gro = (gro_ctx->gro_types & RTE_GRO_UDP_IPV4) ==
 291                 RTE_GRO_UDP_IPV4;
 292
 293         current_time = rte_rdtsc();
 294
 295         for (i = 0; i < nb_pkts; i++) {
 296                 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
 297                                 do_vxlan_gro) {
 298                         if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tbl,
 299                                                 current_time) < 0)
 300                                 unprocess_pkts[unprocess_num++] = pkts[i];
 301                 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
 302                                 do_tcp4_gro) {
 303                         if (gro_tcp4_reassemble(pkts[i], tcp_tbl,
 304                                                 current_time) < 0)
 305                                 unprocess_pkts[unprocess_num++] = pkts[i];
 306                 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
 307                                 do_udp4_gro) {
 308                         if (gro_udp4_reassemble(pkts[i], udp_tbl,
 309                                                 current_time) < 0)
 310                                 unprocess_pkts[unprocess_num++] = pkts[i];
 311                 } else
 312                         unprocess_pkts[unprocess_num++] = pkts[i];
 313         }
 314         if (unprocess_num > 0) {
 315                 memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) *
 316                                 unprocess_num);
 317         }
 318
 319         return unprocess_num;
 320 }
 321
 322 uint16_t
 323 rte_gro_timeout_flush(void *ctx,
 324                 uint64_t timeout_cycles,
 325                 uint64_t gro_types,
 326                 struct rte_mbuf **out,
 327                 uint16_t max_nb_out)
 328 {
 329         struct gro_ctx *gro_ctx = ctx;
 330         uint64_t flush_timestamp;
 331         uint16_t num = 0;
 332         uint16_t left_nb_out = max_nb_out;
 333
 334         gro_types = gro_types & gro_ctx->gro_types;
 335         flush_timestamp = rte_rdtsc() - timeout_cycles;
 336
 337         if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
 338                 num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[
 339                                 RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX],
 340                                 flush_timestamp, out, left_nb_out);
 341                 left_nb_out = max_nb_out - num;
 342         }
 343
 344         /* If no available space in 'out', stop flushing. */
 345         if ((gro_types & RTE_GRO_TCP_IPV4) && left_nb_out > 0) {
 346                 num += gro_tcp4_tbl_timeout_flush(
 347                                 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
 348                                 flush_timestamp,
 349                                 &out[num], left_nb_out);
 350                 left_nb_out = max_nb_out - num;
 351         }
 352
 353         /* If no available space in 'out', stop flushing. */
 354         if ((gro_types & RTE_GRO_UDP_IPV4) && left_nb_out > 0) {
 355                 num += gro_udp4_tbl_timeout_flush(
 356                                 gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
 357                                 flush_timestamp,
 358                                 &out[num], left_nb_out);
 359         }
 360
 361         return num;
 362 }
 363
 364 uint64_t
 365 rte_gro_get_pkt_count(void *ctx)
 366 {
 367         struct gro_ctx *gro_ctx = ctx;
 368         gro_tbl_pkt_count_fn pkt_count_fn;
 369         uint64_t gro_types = gro_ctx->gro_types, flag;
 370         uint64_t item_num = 0;
 371         uint8_t i;
 372
 373         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) {
 374                 flag = 1ULL << i;
 375                 if ((gro_types & flag) == 0)
 376                         continue;
 377
 378                 gro_types ^= flag;
 379                 pkt_count_fn = tbl_pkt_count_fn[i];
 380                 if (pkt_count_fn)
 381                         item_num += pkt_count_fn(gro_ctx->tbls[i]);
 382         }
 383
 384         return item_num;
 385 }