lib/librte_gro/gro_tcp4.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cycles.h>
   8 #include <rte_ethdev.h>
   9
  10 #include "gro_tcp4.h"
  11
  12 void *
  13 gro_tcp4_tbl_create(uint16_t socket_id,
  14                 uint16_t max_flow_num,
  15                 uint16_t max_item_per_flow)
  16 {
  17         struct gro_tcp4_tbl *tbl;
  18         size_t size;
  19         uint32_t entries_num, i;
  20
  21         entries_num = max_flow_num * max_item_per_flow;
  22         entries_num = RTE_MIN(entries_num, GRO_TCP4_TBL_MAX_ITEM_NUM);
  23
  24         if (entries_num == 0)
  25                 return NULL;
  26
  27         tbl = rte_zmalloc_socket(__func__,
  28                         sizeof(struct gro_tcp4_tbl),
  29                         RTE_CACHE_LINE_SIZE,
  30                         socket_id);
  31         if (tbl == NULL)
  32                 return NULL;
  33
  34         size = sizeof(struct gro_tcp4_item) * entries_num;
  35         tbl->items = rte_zmalloc_socket(__func__,
  36                         size,
  37                         RTE_CACHE_LINE_SIZE,
  38                         socket_id);
  39         if (tbl->items == NULL) {
  40                 rte_free(tbl);
  41                 return NULL;
  42         }
  43         tbl->max_item_num = entries_num;
  44
  45         size = sizeof(struct gro_tcp4_flow) * entries_num;
  46         tbl->flows = rte_zmalloc_socket(__func__,
  47                         size,
  48                         RTE_CACHE_LINE_SIZE,
  49                         socket_id);
  50         if (tbl->flows == NULL) {
  51                 rte_free(tbl->items);
  52                 rte_free(tbl);
  53                 return NULL;
  54         }
  55         /* INVALID_ARRAY_INDEX indicates an empty flow */
  56         for (i = 0; i < entries_num; i++)
  57                 tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
  58         tbl->max_flow_num = entries_num;
  59
  60         return tbl;
  61 }
  62
  63 void
  64 gro_tcp4_tbl_destroy(void *tbl)
  65 {
  66         struct gro_tcp4_tbl *tcp_tbl = tbl;
  67
  68         if (tcp_tbl) {
  69                 rte_free(tcp_tbl->items);
  70                 rte_free(tcp_tbl->flows);
  71         }
  72         rte_free(tcp_tbl);
  73 }
  74
  75 static inline uint32_t
  76 find_an_empty_item(struct gro_tcp4_tbl *tbl)
  77 {
  78         uint32_t i;
  79         uint32_t max_item_num = tbl->max_item_num;
  80
  81         for (i = 0; i < max_item_num; i++)
  82                 if (tbl->items[i].firstseg == NULL)
  83                         return i;
  84         return INVALID_ARRAY_INDEX;
  85 }
  86
  87 static inline uint32_t
  88 find_an_empty_flow(struct gro_tcp4_tbl *tbl)
  89 {
  90         uint32_t i;
  91         uint32_t max_flow_num = tbl->max_flow_num;
  92
  93         for (i = 0; i < max_flow_num; i++)
  94                 if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
  95                         return i;
  96         return INVALID_ARRAY_INDEX;
  97 }
  98
  99 static inline uint32_t
 100 insert_new_item(struct gro_tcp4_tbl *tbl,
 101                 struct rte_mbuf *pkt,
 102                 uint64_t start_time,
 103                 uint32_t prev_idx,
 104                 uint32_t sent_seq,
 105                 uint16_t ip_id,
 106                 uint8_t is_atomic)
 107 {
 108         uint32_t item_idx;
 109
 110         item_idx = find_an_empty_item(tbl);
 111         if (item_idx == INVALID_ARRAY_INDEX)
 112                 return INVALID_ARRAY_INDEX;
 113
 114         tbl->items[item_idx].firstseg = pkt;
 115         tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
 116         tbl->items[item_idx].start_time = start_time;
 117         tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
 118         tbl->items[item_idx].sent_seq = sent_seq;
 119         tbl->items[item_idx].ip_id = ip_id;
 120         tbl->items[item_idx].nb_merged = 1;
 121         tbl->items[item_idx].is_atomic = is_atomic;
 122         tbl->item_num++;
 123
 124         /* if the previous packet exists, chain them together. */
 125         if (prev_idx != INVALID_ARRAY_INDEX) {
 126                 tbl->items[item_idx].next_pkt_idx =
 127                         tbl->items[prev_idx].next_pkt_idx;
 128                 tbl->items[prev_idx].next_pkt_idx = item_idx;
 129         }
 130
 131         return item_idx;
 132 }
 133
 134 static inline uint32_t
 135 delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
 136                 uint32_t prev_item_idx)
 137 {
 138         uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
 139
 140         /* NULL indicates an empty item */
 141         tbl->items[item_idx].firstseg = NULL;
 142         tbl->item_num--;
 143         if (prev_item_idx != INVALID_ARRAY_INDEX)
 144                 tbl->items[prev_item_idx].next_pkt_idx = next_idx;
 145
 146         return next_idx;
 147 }
 148
 149 static inline uint32_t
 150 insert_new_flow(struct gro_tcp4_tbl *tbl,
 151                 struct tcp4_flow_key *src,
 152                 uint32_t item_idx)
 153 {
 154         struct tcp4_flow_key *dst;
 155         uint32_t flow_idx;
 156
 157         flow_idx = find_an_empty_flow(tbl);
 158         if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
 159                 return INVALID_ARRAY_INDEX;
 160
 161         dst = &(tbl->flows[flow_idx].key);
 162
 163         ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
 164         ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
 165         dst->ip_src_addr = src->ip_src_addr;
 166         dst->ip_dst_addr = src->ip_dst_addr;
 167         dst->recv_ack = src->recv_ack;
 168         dst->src_port = src->src_port;
 169         dst->dst_port = src->dst_port;
 170
 171         tbl->flows[flow_idx].start_index = item_idx;
 172         tbl->flow_num++;
 173
 174         return flow_idx;
 175 }
 176
 177 /*
 178  * update the packet length for the flushed packet.
 179  */
 180 static inline void
 181 update_header(struct gro_tcp4_item *item)
 182 {
 183         struct ipv4_hdr *ipv4_hdr;
 184         struct rte_mbuf *pkt = item->firstseg;
 185
 186         ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
 187                         pkt->l2_len);
 188         ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
 189                         pkt->l2_len);
 190 }
 191
 192 int32_t
 193 gro_tcp4_reassemble(struct rte_mbuf *pkt,
 194                 struct gro_tcp4_tbl *tbl,
 195                 uint64_t start_time)
 196 {
 197         struct ether_hdr *eth_hdr;
 198         struct ipv4_hdr *ipv4_hdr;
 199         struct tcp_hdr *tcp_hdr;
 200         uint32_t sent_seq;
 201         int32_t tcp_dl;
 202         uint16_t ip_id, hdr_len, frag_off;
 203         uint8_t is_atomic;
 204
 205         struct tcp4_flow_key key;
 206         uint32_t cur_idx, prev_idx, item_idx;
 207         uint32_t i, max_flow_num, remaining_flow_num;
 208         int cmp;
 209         uint8_t find;
 210
 211         eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
 212         ipv4_hdr = (struct ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
 213         tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
 214         hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
 215
 216         /*
 217          * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
 218          * or CWR set.
 219          */
 220         if (tcp_hdr->tcp_flags != TCP_ACK_FLAG)
 221                 return -1;
 222         /*
 223          * Don't process the packet whose payload length is less than or
 224          * equal to 0.
 225          */
 226         tcp_dl = pkt->pkt_len - hdr_len;
 227         if (tcp_dl <= 0)
 228                 return -1;
 229
 230         /*
 231          * Save IPv4 ID for the packet whose DF bit is 0. For the packet
 232          * whose DF bit is 1, IPv4 ID is ignored.
 233          */
 234         frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
 235         is_atomic = (frag_off & IPV4_HDR_DF_FLAG) == IPV4_HDR_DF_FLAG;
 236         ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
 237         sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
 238
 239         ether_addr_copy(&(eth_hdr->s_addr), &(key.eth_saddr));
 240         ether_addr_copy(&(eth_hdr->d_addr), &(key.eth_daddr));
 241         key.ip_src_addr = ipv4_hdr->src_addr;
 242         key.ip_dst_addr = ipv4_hdr->dst_addr;
 243         key.src_port = tcp_hdr->src_port;
 244         key.dst_port = tcp_hdr->dst_port;
 245         key.recv_ack = tcp_hdr->recv_ack;
 246
 247         /* Search for a matched flow. */
 248         max_flow_num = tbl->max_flow_num;
 249         remaining_flow_num = tbl->flow_num;
 250         find = 0;
 251         for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
 252                 if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
 253                         if (is_same_tcp4_flow(tbl->flows[i].key, key)) {
 254                                 find = 1;
 255                                 break;
 256                         }
 257                         remaining_flow_num--;
 258                 }
 259         }
 260
 261         /*
 262          * Fail to find a matched flow. Insert a new flow and store the
 263          * packet into the flow.
 264          */
 265         if (find == 0) {
 266                 item_idx = insert_new_item(tbl, pkt, start_time,
 267                                 INVALID_ARRAY_INDEX, sent_seq, ip_id,
 268                                 is_atomic);
 269                 if (item_idx == INVALID_ARRAY_INDEX)
 270                         return -1;
 271                 if (insert_new_flow(tbl, &key, item_idx) ==
 272                                 INVALID_ARRAY_INDEX) {
 273                         /*
 274                          * Fail to insert a new flow, so delete the
 275                          * stored packet.
 276                          */
 277                         delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
 278                         return -1;
 279                 }
 280                 return 0;
 281         }
 282
 283         /*
 284          * Check all packets in the flow and try to find a neighbor for
 285          * the input packet.
 286          */
 287         cur_idx = tbl->flows[i].start_index;
 288         prev_idx = cur_idx;
 289         do {
 290                 cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
 291                                 sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
 292                                 is_atomic);
 293                 if (cmp) {
 294                         if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
 295                                                 pkt, cmp, sent_seq, ip_id, 0))
 296                                 return 1;
 297                         /*
 298                          * Fail to merge the two packets, as the packet
 299                          * length is greater than the max value. Store
 300                          * the packet into the flow.
 301                          */
 302                         if (insert_new_item(tbl, pkt, start_time, prev_idx,
 303                                                 sent_seq, ip_id, is_atomic) ==
 304                                         INVALID_ARRAY_INDEX)
 305                                 return -1;
 306                         return 0;
 307                 }
 308                 prev_idx = cur_idx;
 309                 cur_idx = tbl->items[cur_idx].next_pkt_idx;
 310         } while (cur_idx != INVALID_ARRAY_INDEX);
 311
 312         /* Fail to find a neighbor, so store the packet into the flow. */
 313         if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
 314                                 ip_id, is_atomic) == INVALID_ARRAY_INDEX)
 315                 return -1;
 316
 317         return 0;
 318 }
 319
 320 uint16_t
 321 gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
 322                 uint64_t flush_timestamp,
 323                 struct rte_mbuf **out,
 324                 uint16_t nb_out)
 325 {
 326         uint16_t k = 0;
 327         uint32_t i, j;
 328         uint32_t max_flow_num = tbl->max_flow_num;
 329
 330         for (i = 0; i < max_flow_num; i++) {
 331                 if (unlikely(tbl->flow_num == 0))
 332                         return k;
 333
 334                 j = tbl->flows[i].start_index;
 335                 while (j != INVALID_ARRAY_INDEX) {
 336                         if (tbl->items[j].start_time <= flush_timestamp) {
 337                                 out[k++] = tbl->items[j].firstseg;
 338                                 if (tbl->items[j].nb_merged > 1)
 339                                         update_header(&(tbl->items[j]));
 340                                 /*
 341                                  * Delete the packet and get the next
 342                                  * packet in the flow.
 343                                  */
 344                                 j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
 345                                 tbl->flows[i].start_index = j;
 346                                 if (j == INVALID_ARRAY_INDEX)
 347                                         tbl->flow_num--;
 348
 349                                 if (unlikely(k == nb_out))
 350                                         return k;
 351                         } else
 352                                 /*
 353                                  * The left packets in this flow won't be
 354                                  * timeout. Go to check other flows.
 355                                  */
 356                                 break;
 357                 }
 358         }
 359         return k;
 360 }
 361
 362 uint32_t
 363 gro_tcp4_tbl_pkt_count(void *tbl)
 364 {
 365         struct gro_tcp4_tbl *gro_tbl = tbl;
 366
 367         if (gro_tbl)
 368                 return gro_tbl->item_num;
 369
 370         return 0;
 371 }