gro/rte_gro.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cycles.h>
   8 #include <rte_ethdev.h>
   9
  10 #include "rte_gro.h"
  11 #include "gro_tcp4.h"
  12 #include "gro_udp4.h"
  13 #include "gro_vxlan_tcp4.h"
  14 #include "gro_vxlan_udp4.h"
  15
  16 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
  17                 uint16_t max_flow_num,
  18                 uint16_t max_item_per_flow);
  19 typedef void (*gro_tbl_destroy_fn)(void *tbl);
  20 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
  21
  22 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
  23                 gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
  24                 gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
  25 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
  26                         gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
  27                         gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
  28                         NULL};
  29 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
  30                         gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
  31                         gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
  32                         NULL};
  33
  34 #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
  35                 ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
  36                 (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
  37
  38 #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
  39                 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
  40                 (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
  41
  42 #define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
  43                 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
  44                 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
  45                  RTE_PTYPE_TUNNEL_VXLAN) && \
  46                 ((ptype & RTE_PTYPE_INNER_L4_TCP) == \
  47                  RTE_PTYPE_INNER_L4_TCP) && \
  48                 (((ptype & RTE_PTYPE_INNER_L3_MASK) == \
  49                   RTE_PTYPE_INNER_L3_IPV4) || \
  50                  ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
  51                   RTE_PTYPE_INNER_L3_IPV4_EXT) || \
  52                  ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
  53                   RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)))
  54
  55 #define IS_IPV4_VXLAN_UDP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
  56                 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
  57                 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
  58                  RTE_PTYPE_TUNNEL_VXLAN) && \
  59                 ((ptype & RTE_PTYPE_INNER_L4_UDP) == \
  60                  RTE_PTYPE_INNER_L4_UDP) && \
  61                 (((ptype & RTE_PTYPE_INNER_L3_MASK) == \
  62                   RTE_PTYPE_INNER_L3_IPV4) || \
  63                  ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
  64                   RTE_PTYPE_INNER_L3_IPV4_EXT) || \
  65                  ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
  66                   RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)))
  67
  68 /*
  69  * GRO context structure. It keeps the table structures, which are
  70  * used to merge packets, for different GRO types. Before using
  71  * rte_gro_reassemble(), applications need to create the GRO context
  72  * first.
  73  */
  74 struct gro_ctx {
  75         /* GRO types to perform */
  76         uint64_t gro_types;
  77         /* reassembly tables */
  78         void *tbls[RTE_GRO_TYPE_MAX_NUM];
  79 };
  80
  81 void *
  82 rte_gro_ctx_create(const struct rte_gro_param *param)
  83 {
  84         struct gro_ctx *gro_ctx;
  85         gro_tbl_create_fn create_tbl_fn;
  86         uint64_t gro_type_flag = 0;
  87         uint64_t gro_types = 0;
  88         uint8_t i;
  89
  90         gro_ctx = rte_zmalloc_socket(__func__,
  91                         sizeof(struct gro_ctx),
  92                         RTE_CACHE_LINE_SIZE,
  93                         param->socket_id);
  94         if (gro_ctx == NULL)
  95                 return NULL;
  96
  97         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
  98                 gro_type_flag = 1ULL << i;
  99                 if ((param->gro_types & gro_type_flag) == 0)
 100                         continue;
 101
 102                 create_tbl_fn = tbl_create_fn[i];
 103                 if (create_tbl_fn == NULL)
 104                         continue;
 105
 106                 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id,
 107                                 param->max_flow_num,
 108                                 param->max_item_per_flow);
 109                 if (gro_ctx->tbls[i] == NULL) {
 110                         /* destroy all created tables */
 111                         gro_ctx->gro_types = gro_types;
 112                         rte_gro_ctx_destroy(gro_ctx);
 113                         return NULL;
 114                 }
 115                 gro_types |= gro_type_flag;
 116         }
 117         gro_ctx->gro_types = param->gro_types;
 118
 119         return gro_ctx;
 120 }
 121
 122 void
 123 rte_gro_ctx_destroy(void *ctx)
 124 {
 125         gro_tbl_destroy_fn destroy_tbl_fn;
 126         struct gro_ctx *gro_ctx = ctx;
 127         uint64_t gro_type_flag;
 128         uint8_t i;
 129
 130         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
 131                 gro_type_flag = 1ULL << i;
 132                 if ((gro_ctx->gro_types & gro_type_flag) == 0)
 133                         continue;
 134                 destroy_tbl_fn = tbl_destroy_fn[i];
 135                 if (destroy_tbl_fn)
 136                         destroy_tbl_fn(gro_ctx->tbls[i]);
 137         }
 138         rte_free(gro_ctx);
 139 }
 140
 141 uint16_t
 142 rte_gro_reassemble_burst(struct rte_mbuf **pkts,
 143                 uint16_t nb_pkts,
 144                 const struct rte_gro_param *param)
 145 {
 146         /* allocate a reassembly table for TCP/IPv4 GRO */
 147         struct gro_tcp4_tbl tcp_tbl;
 148         struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 149         struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
 150
 151         /* allocate a reassembly table for UDP/IPv4 GRO */
 152         struct gro_udp4_tbl udp_tbl;
 153         struct gro_udp4_flow udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 154         struct gro_udp4_item udp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
 155
 156         /* Allocate a reassembly table for VXLAN TCP GRO */
 157         struct gro_vxlan_tcp4_tbl vxlan_tcp_tbl;
 158         struct gro_vxlan_tcp4_flow vxlan_tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 159         struct gro_vxlan_tcp4_item vxlan_tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
 160                         = {{{0}, 0, 0} };
 161
 162         /* Allocate a reassembly table for VXLAN UDP GRO */
 163         struct gro_vxlan_udp4_tbl vxlan_udp_tbl;
 164         struct gro_vxlan_udp4_flow vxlan_udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 165         struct gro_vxlan_udp4_item vxlan_udp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
 166                         = {{{0}} };
 167
 168         struct rte_mbuf *unprocess_pkts[nb_pkts];
 169         uint32_t item_num;
 170         int32_t ret;
 171         uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
 172         uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
 173                 do_vxlan_udp_gro = 0;
 174
 175         if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
 176                                         RTE_GRO_TCP_IPV4 |
 177                                         RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
 178                                         RTE_GRO_UDP_IPV4)) == 0))
 179                 return nb_pkts;
 180
 181         /* Get the maximum number of packets */
 182         item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
 183                                 param->max_item_per_flow));
 184         item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
 185
 186         if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
 187                 for (i = 0; i < item_num; i++)
 188                         vxlan_tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
 189
 190                 vxlan_tcp_tbl.flows = vxlan_tcp_flows;
 191                 vxlan_tcp_tbl.items = vxlan_tcp_items;
 192                 vxlan_tcp_tbl.flow_num = 0;
 193                 vxlan_tcp_tbl.item_num = 0;
 194                 vxlan_tcp_tbl.max_flow_num = item_num;
 195                 vxlan_tcp_tbl.max_item_num = item_num;
 196                 do_vxlan_tcp_gro = 1;
 197         }
 198
 199         if (param->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) {
 200                 for (i = 0; i < item_num; i++)
 201                         vxlan_udp_flows[i].start_index = INVALID_ARRAY_INDEX;
 202
 203                 vxlan_udp_tbl.flows = vxlan_udp_flows;
 204                 vxlan_udp_tbl.items = vxlan_udp_items;
 205                 vxlan_udp_tbl.flow_num = 0;
 206                 vxlan_udp_tbl.item_num = 0;
 207                 vxlan_udp_tbl.max_flow_num = item_num;
 208                 vxlan_udp_tbl.max_item_num = item_num;
 209                 do_vxlan_udp_gro = 1;
 210         }
 211
 212         if (param->gro_types & RTE_GRO_TCP_IPV4) {
 213                 for (i = 0; i < item_num; i++)
 214                         tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
 215
 216                 tcp_tbl.flows = tcp_flows;
 217                 tcp_tbl.items = tcp_items;
 218                 tcp_tbl.flow_num = 0;
 219                 tcp_tbl.item_num = 0;
 220                 tcp_tbl.max_flow_num = item_num;
 221                 tcp_tbl.max_item_num = item_num;
 222                 do_tcp4_gro = 1;
 223         }
 224
 225         if (param->gro_types & RTE_GRO_UDP_IPV4) {
 226                 for (i = 0; i < item_num; i++)
 227                         udp_flows[i].start_index = INVALID_ARRAY_INDEX;
 228
 229                 udp_tbl.flows = udp_flows;
 230                 udp_tbl.items = udp_items;
 231                 udp_tbl.flow_num = 0;
 232                 udp_tbl.item_num = 0;
 233                 udp_tbl.max_flow_num = item_num;
 234                 udp_tbl.max_item_num = item_num;
 235                 do_udp4_gro = 1;
 236         }
 237
 238
 239         for (i = 0; i < nb_pkts; i++) {
 240                 /*
 241                  * The timestamp is ignored, since all packets
 242                  * will be flushed from the tables.
 243                  */
 244                 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
 245                                 do_vxlan_tcp_gro) {
 246                         ret = gro_vxlan_tcp4_reassemble(pkts[i],
 247                                                         &vxlan_tcp_tbl, 0);
 248                         if (ret > 0)
 249                                 /* Merge successfully */
 250                                 nb_after_gro--;
 251                         else if (ret < 0)
 252                                 unprocess_pkts[unprocess_num++] = pkts[i];
 253                 } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) &&
 254                                 do_vxlan_udp_gro) {
 255                         ret = gro_vxlan_udp4_reassemble(pkts[i],
 256                                                         &vxlan_udp_tbl, 0);
 257                         if (ret > 0)
 258                                 /* Merge successfully */
 259                                 nb_after_gro--;
 260                         else if (ret < 0)
 261                                 unprocess_pkts[unprocess_num++] = pkts[i];
 262                 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
 263                                 do_tcp4_gro) {
 264                         ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0);
 265                         if (ret > 0)
 266                                 /* merge successfully */
 267                                 nb_after_gro--;
 268                         else if (ret < 0)
 269                                 unprocess_pkts[unprocess_num++] = pkts[i];
 270                 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
 271                                 do_udp4_gro) {
 272                         ret = gro_udp4_reassemble(pkts[i], &udp_tbl, 0);
 273                         if (ret > 0)
 274                                 /* merge successfully */
 275                                 nb_after_gro--;
 276                         else if (ret < 0)
 277                                 unprocess_pkts[unprocess_num++] = pkts[i];
 278                 } else
 279                         unprocess_pkts[unprocess_num++] = pkts[i];
 280         }
 281
 282         if ((nb_after_gro < nb_pkts)
 283                  || (unprocess_num < nb_pkts)) {
 284                 i = 0;
 285                 /* Flush all packets from the tables */
 286                 if (do_vxlan_tcp_gro) {
 287                         i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
 288                                         0, pkts, nb_pkts);
 289                 }
 290
 291                 if (do_vxlan_udp_gro) {
 292                         i += gro_vxlan_udp4_tbl_timeout_flush(&vxlan_udp_tbl,
 293                                         0, &pkts[i], nb_pkts - i);
 294
 295                 }
 296
 297                 if (do_tcp4_gro) {
 298                         i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0,
 299                                         &pkts[i], nb_pkts - i);
 300                 }
 301
 302                 if (do_udp4_gro) {
 303                         i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
 304                                         &pkts[i], nb_pkts - i);
 305                 }
 306                 /* Copy unprocessed packets */
 307                 if (unprocess_num > 0) {
 308                         memcpy(&pkts[i], unprocess_pkts,
 309                                         sizeof(struct rte_mbuf *) *
 310                                         unprocess_num);
 311                 }
 312                 nb_after_gro = i + unprocess_num;
 313         }
 314
 315         return nb_after_gro;
 316 }
 317
 318 uint16_t
 319 rte_gro_reassemble(struct rte_mbuf **pkts,
 320                 uint16_t nb_pkts,
 321                 void *ctx)
 322 {
 323         struct rte_mbuf *unprocess_pkts[nb_pkts];
 324         struct gro_ctx *gro_ctx = ctx;
 325         void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
 326         uint64_t current_time;
 327         uint16_t i, unprocess_num = 0;
 328         uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
 329
 330         if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
 331                                         RTE_GRO_TCP_IPV4 |
 332                                         RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
 333                                         RTE_GRO_UDP_IPV4)) == 0))
 334                 return nb_pkts;
 335
 336         tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX];
 337         vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
 338         udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
 339         vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
 340
 341         do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
 342                 RTE_GRO_TCP_IPV4;
 343         do_vxlan_tcp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) ==
 344                 RTE_GRO_IPV4_VXLAN_TCP_IPV4;
 345         do_udp4_gro = (gro_ctx->gro_types & RTE_GRO_UDP_IPV4) ==
 346                 RTE_GRO_UDP_IPV4;
 347         do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
 348                 RTE_GRO_IPV4_VXLAN_UDP_IPV4;
 349
 350         current_time = rte_rdtsc();
 351
 352         for (i = 0; i < nb_pkts; i++) {
 353                 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
 354                                 do_vxlan_tcp_gro) {
 355                         if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tcp_tbl,
 356                                                 current_time) < 0)
 357                                 unprocess_pkts[unprocess_num++] = pkts[i];
 358                 } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) &&
 359                                 do_vxlan_udp_gro) {
 360                         if (gro_vxlan_udp4_reassemble(pkts[i], vxlan_udp_tbl,
 361                                                 current_time) < 0)
 362                                 unprocess_pkts[unprocess_num++] = pkts[i];
 363                 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
 364                                 do_tcp4_gro) {
 365                         if (gro_tcp4_reassemble(pkts[i], tcp_tbl,
 366                                                 current_time) < 0)
 367                                 unprocess_pkts[unprocess_num++] = pkts[i];
 368                 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
 369                                 do_udp4_gro) {
 370                         if (gro_udp4_reassemble(pkts[i], udp_tbl,
 371                                                 current_time) < 0)
 372                                 unprocess_pkts[unprocess_num++] = pkts[i];
 373                 } else
 374                         unprocess_pkts[unprocess_num++] = pkts[i];
 375         }
 376         if (unprocess_num > 0) {
 377                 memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) *
 378                                 unprocess_num);
 379         }
 380
 381         return unprocess_num;
 382 }
 383
 384 uint16_t
 385 rte_gro_timeout_flush(void *ctx,
 386                 uint64_t timeout_cycles,
 387                 uint64_t gro_types,
 388                 struct rte_mbuf **out,
 389                 uint16_t max_nb_out)
 390 {
 391         struct gro_ctx *gro_ctx = ctx;
 392         uint64_t flush_timestamp;
 393         uint16_t num = 0;
 394         uint16_t left_nb_out = max_nb_out;
 395
 396         gro_types = gro_types & gro_ctx->gro_types;
 397         flush_timestamp = rte_rdtsc() - timeout_cycles;
 398
 399         if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
 400                 num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[
 401                                 RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX],
 402                                 flush_timestamp, out, left_nb_out);
 403                 left_nb_out = max_nb_out - num;
 404         }
 405
 406         if ((gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) && left_nb_out > 0) {
 407                 num += gro_vxlan_udp4_tbl_timeout_flush(gro_ctx->tbls[
 408                                 RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX],
 409                                 flush_timestamp, &out[num], left_nb_out);
 410                 left_nb_out = max_nb_out - num;
 411         }
 412
 413         /* If no available space in 'out', stop flushing. */
 414         if ((gro_types & RTE_GRO_TCP_IPV4) && left_nb_out > 0) {
 415                 num += gro_tcp4_tbl_timeout_flush(
 416                                 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
 417                                 flush_timestamp,
 418                                 &out[num], left_nb_out);
 419                 left_nb_out = max_nb_out - num;
 420         }
 421
 422         /* If no available space in 'out', stop flushing. */
 423         if ((gro_types & RTE_GRO_UDP_IPV4) && left_nb_out > 0) {
 424                 num += gro_udp4_tbl_timeout_flush(
 425                                 gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
 426                                 flush_timestamp,
 427                                 &out[num], left_nb_out);
 428         }
 429
 430         return num;
 431 }
 432
 433 uint64_t
 434 rte_gro_get_pkt_count(void *ctx)
 435 {
 436         struct gro_ctx *gro_ctx = ctx;
 437         gro_tbl_pkt_count_fn pkt_count_fn;
 438         uint64_t gro_types = gro_ctx->gro_types, flag;
 439         uint64_t item_num = 0;
 440         uint8_t i;
 441
 442         for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) {
 443                 flag = 1ULL << i;
 444                 if ((gro_types & flag) == 0)
 445                         continue;
 446
 447                 gro_types ^= flag;
 448                 pkt_count_fn = tbl_pkt_count_fn[i];
 449                 if (pkt_count_fn)
 450                         item_num += pkt_count_fn(gro_ctx->tbls[i]);
 451         }
 452
 453         return item_num;
 454 }