lib/librte_gro/gro_tcp4.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cycles.h>
   8 #include <rte_ethdev.h>
   9 #include <rte_ip.h>
  10 #include <rte_tcp.h>
  11
  12 #include "gro_tcp4.h"
  13
  14 void *
  15 gro_tcp4_tbl_create(uint16_t socket_id,
  16                 uint16_t max_flow_num,
  17                 uint16_t max_item_per_flow)
  18 {
  19         struct gro_tcp4_tbl *tbl;
  20         size_t size;
  21         uint32_t entries_num, i;
  22
  23         entries_num = max_flow_num * max_item_per_flow;
  24         entries_num = RTE_MIN(entries_num, GRO_TCP4_TBL_MAX_ITEM_NUM);
  25
  26         if (entries_num == 0)
  27                 return NULL;
  28
  29         tbl = rte_zmalloc_socket(__func__,
  30                         sizeof(struct gro_tcp4_tbl),
  31                         RTE_CACHE_LINE_SIZE,
  32                         socket_id);
  33         if (tbl == NULL)
  34                 return NULL;
  35
  36         size = sizeof(struct gro_tcp4_item) * entries_num;
  37         tbl->items = rte_zmalloc_socket(__func__,
  38                         size,
  39                         RTE_CACHE_LINE_SIZE,
  40                         socket_id);
  41         if (tbl->items == NULL) {
  42                 rte_free(tbl);
  43                 return NULL;
  44         }
  45         tbl->max_item_num = entries_num;
  46
  47         size = sizeof(struct gro_tcp4_key) * entries_num;
  48         tbl->keys = rte_zmalloc_socket(__func__,
  49                         size,
  50                         RTE_CACHE_LINE_SIZE,
  51                         socket_id);
  52         if (tbl->keys == NULL) {
  53                 rte_free(tbl->items);
  54                 rte_free(tbl);
  55                 return NULL;
  56         }
  57         /* INVALID_ARRAY_INDEX indicates empty key */
  58         for (i = 0; i < entries_num; i++)
  59                 tbl->keys[i].start_index = INVALID_ARRAY_INDEX;
  60         tbl->max_key_num = entries_num;
  61
  62         return tbl;
  63 }
  64
  65 void
  66 gro_tcp4_tbl_destroy(void *tbl)
  67 {
  68         struct gro_tcp4_tbl *tcp_tbl = tbl;
  69
  70         if (tcp_tbl) {
  71                 rte_free(tcp_tbl->items);
  72                 rte_free(tcp_tbl->keys);
  73         }
  74         rte_free(tcp_tbl);
  75 }
  76
  77 /*
  78  * merge two TCP/IPv4 packets without updating checksums.
  79  * If cmp is larger than 0, append the new packet to the
  80  * original packet. Otherwise, pre-pend the new packet to
  81  * the original packet.
  82  */
  83 static inline int
  84 merge_two_tcp4_packets(struct gro_tcp4_item *item_src,
  85                 struct rte_mbuf *pkt,
  86                 uint16_t ip_id,
  87                 uint32_t sent_seq,
  88                 int cmp)
  89 {
  90         struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
  91         uint16_t tcp_datalen;
  92
  93         if (cmp > 0) {
  94                 pkt_head = item_src->firstseg;
  95                 pkt_tail = pkt;
  96         } else {
  97                 pkt_head = pkt;
  98                 pkt_tail = item_src->firstseg;
  99         }
 100
 101         /* check if the packet length will be beyond the max value */
 102         tcp_datalen = pkt_tail->pkt_len - pkt_tail->l2_len -
 103                 pkt_tail->l3_len - pkt_tail->l4_len;
 104         if (pkt_head->pkt_len - pkt_head->l2_len + tcp_datalen >
 105                         TCP4_MAX_L3_LENGTH)
 106                 return 0;
 107
 108         /* remove packet header for the tail packet */
 109         rte_pktmbuf_adj(pkt_tail,
 110                         pkt_tail->l2_len +
 111                         pkt_tail->l3_len +
 112                         pkt_tail->l4_len);
 113
 114         /* chain two packets together */
 115         if (cmp > 0) {
 116                 item_src->lastseg->next = pkt;
 117                 item_src->lastseg = rte_pktmbuf_lastseg(pkt);
 118                 /* update IP ID to the larger value */
 119                 item_src->ip_id = ip_id;
 120         } else {
 121                 lastseg = rte_pktmbuf_lastseg(pkt);
 122                 lastseg->next = item_src->firstseg;
 123                 item_src->firstseg = pkt;
 124                 /* update sent_seq to the smaller value */
 125                 item_src->sent_seq = sent_seq;
 126         }
 127         item_src->nb_merged++;
 128
 129         /* update mbuf metadata for the merged packet */
 130         pkt_head->nb_segs += pkt_tail->nb_segs;
 131         pkt_head->pkt_len += pkt_tail->pkt_len;
 132
 133         return 1;
 134 }
 135
 136 static inline int
 137 check_seq_option(struct gro_tcp4_item *item,
 138                 struct tcp_hdr *tcp_hdr,
 139                 uint16_t tcp_hl,
 140                 uint16_t tcp_dl,
 141                 uint16_t ip_id,
 142                 uint32_t sent_seq)
 143 {
 144         struct rte_mbuf *pkt0 = item->firstseg;
 145         struct ipv4_hdr *ipv4_hdr0;
 146         struct tcp_hdr *tcp_hdr0;
 147         uint16_t tcp_hl0, tcp_dl0;
 148         uint16_t len;
 149
 150         ipv4_hdr0 = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt0, char *) +
 151                         pkt0->l2_len);
 152         tcp_hdr0 = (struct tcp_hdr *)((char *)ipv4_hdr0 + pkt0->l3_len);
 153         tcp_hl0 = pkt0->l4_len;
 154
 155         /* check if TCP option fields equal. If not, return 0. */
 156         len = RTE_MAX(tcp_hl, tcp_hl0) - sizeof(struct tcp_hdr);
 157         if ((tcp_hl != tcp_hl0) ||
 158                         ((len > 0) && (memcmp(tcp_hdr + 1,
 159                                         tcp_hdr0 + 1,
 160                                         len) != 0)))
 161                 return 0;
 162
 163         /* check if the two packets are neighbors */
 164         tcp_dl0 = pkt0->pkt_len - pkt0->l2_len - pkt0->l3_len - tcp_hl0;
 165         if ((sent_seq == (item->sent_seq + tcp_dl0)) &&
 166                         (ip_id == (item->ip_id + 1)))
 167                 /* append the new packet */
 168                 return 1;
 169         else if (((sent_seq + tcp_dl) == item->sent_seq) &&
 170                         ((ip_id + item->nb_merged) == item->ip_id))
 171                 /* pre-pend the new packet */
 172                 return -1;
 173         else
 174                 return 0;
 175 }
 176
 177 static inline uint32_t
 178 find_an_empty_item(struct gro_tcp4_tbl *tbl)
 179 {
 180         uint32_t i;
 181         uint32_t max_item_num = tbl->max_item_num;
 182
 183         for (i = 0; i < max_item_num; i++)
 184                 if (tbl->items[i].firstseg == NULL)
 185                         return i;
 186         return INVALID_ARRAY_INDEX;
 187 }
 188
 189 static inline uint32_t
 190 find_an_empty_key(struct gro_tcp4_tbl *tbl)
 191 {
 192         uint32_t i;
 193         uint32_t max_key_num = tbl->max_key_num;
 194
 195         for (i = 0; i < max_key_num; i++)
 196                 if (tbl->keys[i].start_index == INVALID_ARRAY_INDEX)
 197                         return i;
 198         return INVALID_ARRAY_INDEX;
 199 }
 200
 201 static inline uint32_t
 202 insert_new_item(struct gro_tcp4_tbl *tbl,
 203                 struct rte_mbuf *pkt,
 204                 uint16_t ip_id,
 205                 uint32_t sent_seq,
 206                 uint32_t prev_idx,
 207                 uint64_t start_time)
 208 {
 209         uint32_t item_idx;
 210
 211         item_idx = find_an_empty_item(tbl);
 212         if (item_idx == INVALID_ARRAY_INDEX)
 213                 return INVALID_ARRAY_INDEX;
 214
 215         tbl->items[item_idx].firstseg = pkt;
 216         tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
 217         tbl->items[item_idx].start_time = start_time;
 218         tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
 219         tbl->items[item_idx].sent_seq = sent_seq;
 220         tbl->items[item_idx].ip_id = ip_id;
 221         tbl->items[item_idx].nb_merged = 1;
 222         tbl->item_num++;
 223
 224         /* if the previous packet exists, chain the new one with it */
 225         if (prev_idx != INVALID_ARRAY_INDEX) {
 226                 tbl->items[item_idx].next_pkt_idx =
 227                         tbl->items[prev_idx].next_pkt_idx;
 228                 tbl->items[prev_idx].next_pkt_idx = item_idx;
 229         }
 230
 231         return item_idx;
 232 }
 233
 234 static inline uint32_t
 235 delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
 236                 uint32_t prev_item_idx)
 237 {
 238         uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
 239
 240         /* set NULL to firstseg to indicate it's an empty item */
 241         tbl->items[item_idx].firstseg = NULL;
 242         tbl->item_num--;
 243         if (prev_item_idx != INVALID_ARRAY_INDEX)
 244                 tbl->items[prev_item_idx].next_pkt_idx = next_idx;
 245
 246         return next_idx;
 247 }
 248
 249 static inline uint32_t
 250 insert_new_key(struct gro_tcp4_tbl *tbl,
 251                 struct tcp4_key *key_src,
 252                 uint32_t item_idx)
 253 {
 254         struct tcp4_key *key_dst;
 255         uint32_t key_idx;
 256
 257         key_idx = find_an_empty_key(tbl);
 258         if (key_idx == INVALID_ARRAY_INDEX)
 259                 return INVALID_ARRAY_INDEX;
 260
 261         key_dst = &(tbl->keys[key_idx].key);
 262
 263         ether_addr_copy(&(key_src->eth_saddr), &(key_dst->eth_saddr));
 264         ether_addr_copy(&(key_src->eth_daddr), &(key_dst->eth_daddr));
 265         key_dst->ip_src_addr = key_src->ip_src_addr;
 266         key_dst->ip_dst_addr = key_src->ip_dst_addr;
 267         key_dst->recv_ack = key_src->recv_ack;
 268         key_dst->src_port = key_src->src_port;
 269         key_dst->dst_port = key_src->dst_port;
 270
 271         /* non-INVALID_ARRAY_INDEX value indicates this key is valid */
 272         tbl->keys[key_idx].start_index = item_idx;
 273         tbl->key_num++;
 274
 275         return key_idx;
 276 }
 277
 278 static inline int
 279 is_same_key(struct tcp4_key k1, struct tcp4_key k2)
 280 {
 281         if (is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) == 0)
 282                 return 0;
 283
 284         if (is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) == 0)
 285                 return 0;
 286
 287         return ((k1.ip_src_addr == k2.ip_src_addr) &&
 288                         (k1.ip_dst_addr == k2.ip_dst_addr) &&
 289                         (k1.recv_ack == k2.recv_ack) &&
 290                         (k1.src_port == k2.src_port) &&
 291                         (k1.dst_port == k2.dst_port));
 292 }
 293
 294 /*
 295  * update packet length for the flushed packet.
 296  */
 297 static inline void
 298 update_header(struct gro_tcp4_item *item)
 299 {
 300         struct ipv4_hdr *ipv4_hdr;
 301         struct rte_mbuf *pkt = item->firstseg;
 302
 303         ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
 304                         pkt->l2_len);
 305         ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
 306                         pkt->l2_len);
 307 }
 308
 309 int32_t
 310 gro_tcp4_reassemble(struct rte_mbuf *pkt,
 311                 struct gro_tcp4_tbl *tbl,
 312                 uint64_t start_time)
 313 {
 314         struct ether_hdr *eth_hdr;
 315         struct ipv4_hdr *ipv4_hdr;
 316         struct tcp_hdr *tcp_hdr;
 317         uint32_t sent_seq;
 318         uint16_t tcp_dl, ip_id;
 319
 320         struct tcp4_key key;
 321         uint32_t cur_idx, prev_idx, item_idx;
 322         uint32_t i, max_key_num;
 323         int cmp;
 324
 325         eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
 326         ipv4_hdr = (struct ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
 327         tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
 328
 329         /*
 330          * if FIN, SYN, RST, PSH, URG, ECE or
 331          * CWR is set, return immediately.
 332          */
 333         if (tcp_hdr->tcp_flags != TCP_ACK_FLAG)
 334                 return -1;
 335         /* if payload length is 0, return immediately */
 336         tcp_dl = rte_be_to_cpu_16(ipv4_hdr->total_length) - pkt->l3_len -
 337                 pkt->l4_len;
 338         if (tcp_dl == 0)
 339                 return -1;
 340
 341         ip_id = rte_be_to_cpu_16(ipv4_hdr->packet_id);
 342         sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
 343
 344         ether_addr_copy(&(eth_hdr->s_addr), &(key.eth_saddr));
 345         ether_addr_copy(&(eth_hdr->d_addr), &(key.eth_daddr));
 346         key.ip_src_addr = ipv4_hdr->src_addr;
 347         key.ip_dst_addr = ipv4_hdr->dst_addr;
 348         key.src_port = tcp_hdr->src_port;
 349         key.dst_port = tcp_hdr->dst_port;
 350         key.recv_ack = tcp_hdr->recv_ack;
 351
 352         /* search for a key */
 353         max_key_num = tbl->max_key_num;
 354         for (i = 0; i < max_key_num; i++) {
 355                 if ((tbl->keys[i].start_index != INVALID_ARRAY_INDEX) &&
 356                                 is_same_key(tbl->keys[i].key, key))
 357                         break;
 358         }
 359
 360         /* can't find a key, so insert a new key and a new item. */
 361         if (i == tbl->max_key_num) {
 362                 item_idx = insert_new_item(tbl, pkt, ip_id, sent_seq,
 363                                 INVALID_ARRAY_INDEX, start_time);
 364                 if (item_idx == INVALID_ARRAY_INDEX)
 365                         return -1;
 366                 if (insert_new_key(tbl, &key, item_idx) ==
 367                                 INVALID_ARRAY_INDEX) {
 368                         /*
 369                          * fail to insert a new key, so
 370                          * delete the inserted item
 371                          */
 372                         delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
 373                         return -1;
 374                 }
 375                 return 0;
 376         }
 377
 378         /* traverse all packets in the item group to find one to merge */
 379         cur_idx = tbl->keys[i].start_index;
 380         prev_idx = cur_idx;
 381         do {
 382                 cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
 383                                 pkt->l4_len, tcp_dl, ip_id, sent_seq);
 384                 if (cmp) {
 385                         if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
 386                                                 pkt, ip_id,
 387                                                 sent_seq, cmp))
 388                                 return 1;
 389                         /*
 390                          * fail to merge two packets since the packet
 391                          * length will be greater than the max value.
 392                          * So insert the packet into the item group.
 393                          */
 394                         if (insert_new_item(tbl, pkt, ip_id, sent_seq,
 395                                                 prev_idx, start_time) ==
 396                                         INVALID_ARRAY_INDEX)
 397                                 return -1;
 398                         return 0;
 399                 }
 400                 prev_idx = cur_idx;
 401                 cur_idx = tbl->items[cur_idx].next_pkt_idx;
 402         } while (cur_idx != INVALID_ARRAY_INDEX);
 403
 404         /*
 405          * can't find a packet in the item group to merge,
 406          * so insert the packet into the item group.
 407          */
 408         if (insert_new_item(tbl, pkt, ip_id, sent_seq, prev_idx,
 409                                 start_time) == INVALID_ARRAY_INDEX)
 410                 return -1;
 411
 412         return 0;
 413 }
 414
 415 uint16_t
 416 gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
 417                 uint64_t flush_timestamp,
 418                 struct rte_mbuf **out,
 419                 uint16_t nb_out)
 420 {
 421         uint16_t k = 0;
 422         uint32_t i, j;
 423         uint32_t max_key_num = tbl->max_key_num;
 424
 425         for (i = 0; i < max_key_num; i++) {
 426                 /* all keys have been checked, return immediately */
 427                 if (tbl->key_num == 0)
 428                         return k;
 429
 430                 j = tbl->keys[i].start_index;
 431                 while (j != INVALID_ARRAY_INDEX) {
 432                         if (tbl->items[j].start_time <= flush_timestamp) {
 433                                 out[k++] = tbl->items[j].firstseg;
 434                                 if (tbl->items[j].nb_merged > 1)
 435                                         update_header(&(tbl->items[j]));
 436                                 /*
 437                                  * delete the item and get
 438                                  * the next packet index
 439                                  */
 440                                 j = delete_item(tbl, j,
 441                                                 INVALID_ARRAY_INDEX);
 442
 443                                 /*
 444                                  * delete the key as all of
 445                                  * packets are flushed
 446                                  */
 447                                 if (j == INVALID_ARRAY_INDEX) {
 448                                         tbl->keys[i].start_index =
 449                                                 INVALID_ARRAY_INDEX;
 450                                         tbl->key_num--;
 451                                 } else
 452                                         /* update start_index of the key */
 453                                         tbl->keys[i].start_index = j;
 454
 455                                 if (k == nb_out)
 456                                         return k;
 457                         } else
 458                                 /*
 459                                  * left packets of this key won't be
 460                                  * timeout, so go to check other keys.
 461                                  */
 462                                 break;
 463                 }
 464         }
 465         return k;
 466 }
 467
 468 uint32_t
 469 gro_tcp4_tbl_pkt_count(void *tbl)
 470 {
 471         struct gro_tcp4_tbl *gro_tbl = tbl;
 472
 473         if (gro_tbl)
 474                 return gro_tbl->item_num;
 475
 476         return 0;
 477 }