lib/librte_gro/gro_udp4.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Inspur Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cycles.h>
   8 #include <rte_ethdev.h>
   9
  10 #include "gro_udp4.h"
  11
  12 void *
  13 gro_udp4_tbl_create(uint16_t socket_id,
  14                 uint16_t max_flow_num,
  15                 uint16_t max_item_per_flow)
  16 {
  17         struct gro_udp4_tbl *tbl;
  18         size_t size;
  19         uint32_t entries_num, i;
  20
  21         entries_num = max_flow_num * max_item_per_flow;
  22         entries_num = RTE_MIN(entries_num, GRO_UDP4_TBL_MAX_ITEM_NUM);
  23
  24         if (entries_num == 0)
  25                 return NULL;
  26
  27         tbl = rte_zmalloc_socket(__func__,
  28                         sizeof(struct gro_udp4_tbl),
  29                         RTE_CACHE_LINE_SIZE,
  30                         socket_id);
  31         if (tbl == NULL)
  32                 return NULL;
  33
  34         size = sizeof(struct gro_udp4_item) * entries_num;
  35         tbl->items = rte_zmalloc_socket(__func__,
  36                         size,
  37                         RTE_CACHE_LINE_SIZE,
  38                         socket_id);
  39         if (tbl->items == NULL) {
  40                 rte_free(tbl);
  41                 return NULL;
  42         }
  43         tbl->max_item_num = entries_num;
  44
  45         size = sizeof(struct gro_udp4_flow) * entries_num;
  46         tbl->flows = rte_zmalloc_socket(__func__,
  47                         size,
  48                         RTE_CACHE_LINE_SIZE,
  49                         socket_id);
  50         if (tbl->flows == NULL) {
  51                 rte_free(tbl->items);
  52                 rte_free(tbl);
  53                 return NULL;
  54         }
  55         /* INVALID_ARRAY_INDEX indicates an empty flow */
  56         for (i = 0; i < entries_num; i++)
  57                 tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
  58         tbl->max_flow_num = entries_num;
  59
  60         return tbl;
  61 }
  62
  63 void
  64 gro_udp4_tbl_destroy(void *tbl)
  65 {
  66         struct gro_udp4_tbl *udp_tbl = tbl;
  67
  68         if (udp_tbl) {
  69                 rte_free(udp_tbl->items);
  70                 rte_free(udp_tbl->flows);
  71         }
  72         rte_free(udp_tbl);
  73 }
  74
  75 static inline uint32_t
  76 find_an_empty_item(struct gro_udp4_tbl *tbl)
  77 {
  78         uint32_t i;
  79         uint32_t max_item_num = tbl->max_item_num;
  80
  81         for (i = 0; i < max_item_num; i++)
  82                 if (tbl->items[i].firstseg == NULL)
  83                         return i;
  84         return INVALID_ARRAY_INDEX;
  85 }
  86
  87 static inline uint32_t
  88 find_an_empty_flow(struct gro_udp4_tbl *tbl)
  89 {
  90         uint32_t i;
  91         uint32_t max_flow_num = tbl->max_flow_num;
  92
  93         for (i = 0; i < max_flow_num; i++)
  94                 if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
  95                         return i;
  96         return INVALID_ARRAY_INDEX;
  97 }
  98
  99 static inline uint32_t
 100 insert_new_item(struct gro_udp4_tbl *tbl,
 101                 struct rte_mbuf *pkt,
 102                 uint64_t start_time,
 103                 uint32_t prev_idx,
 104                 uint16_t frag_offset,
 105                 uint8_t is_last_frag)
 106 {
 107         uint32_t item_idx;
 108
 109         item_idx = find_an_empty_item(tbl);
 110         if (unlikely(item_idx == INVALID_ARRAY_INDEX))
 111                 return INVALID_ARRAY_INDEX;
 112
 113         tbl->items[item_idx].firstseg = pkt;
 114         tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
 115         tbl->items[item_idx].start_time = start_time;
 116         tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
 117         tbl->items[item_idx].frag_offset = frag_offset;
 118         tbl->items[item_idx].is_last_frag = is_last_frag;
 119         tbl->items[item_idx].nb_merged = 1;
 120         tbl->item_num++;
 121
 122         /* if the previous packet exists, chain them together. */
 123         if (prev_idx != INVALID_ARRAY_INDEX) {
 124                 tbl->items[item_idx].next_pkt_idx =
 125                         tbl->items[prev_idx].next_pkt_idx;
 126                 tbl->items[prev_idx].next_pkt_idx = item_idx;
 127         }
 128
 129         return item_idx;
 130 }
 131
 132 static inline uint32_t
 133 delete_item(struct gro_udp4_tbl *tbl, uint32_t item_idx,
 134                 uint32_t prev_item_idx)
 135 {
 136         uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
 137
 138         /* NULL indicates an empty item */
 139         tbl->items[item_idx].firstseg = NULL;
 140         tbl->item_num--;
 141         if (prev_item_idx != INVALID_ARRAY_INDEX)
 142                 tbl->items[prev_item_idx].next_pkt_idx = next_idx;
 143
 144         return next_idx;
 145 }
 146
 147 static inline uint32_t
 148 insert_new_flow(struct gro_udp4_tbl *tbl,
 149                 struct udp4_flow_key *src,
 150                 uint32_t item_idx)
 151 {
 152         struct udp4_flow_key *dst;
 153         uint32_t flow_idx;
 154
 155         flow_idx = find_an_empty_flow(tbl);
 156         if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
 157                 return INVALID_ARRAY_INDEX;
 158
 159         dst = &(tbl->flows[flow_idx].key);
 160
 161         rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
 162         rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
 163         dst->ip_src_addr = src->ip_src_addr;
 164         dst->ip_dst_addr = src->ip_dst_addr;
 165         dst->ip_id = src->ip_id;
 166
 167         tbl->flows[flow_idx].start_index = item_idx;
 168         tbl->flow_num++;
 169
 170         return flow_idx;
 171 }
 172
 173 /*
 174  * update the packet length for the flushed packet.
 175  */
 176 static inline void
 177 update_header(struct gro_udp4_item *item)
 178 {
 179         struct rte_ipv4_hdr *ipv4_hdr;
 180         struct rte_mbuf *pkt = item->firstseg;
 181         uint16_t frag_offset;
 182
 183         ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
 184                         pkt->l2_len);
 185         ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
 186                         pkt->l2_len);
 187
 188         /* Clear MF bit if it is last fragment */
 189         if (item->is_last_frag) {
 190                 frag_offset = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
 191                 ipv4_hdr->fragment_offset =
 192                         rte_cpu_to_be_16(frag_offset & ~RTE_IPV4_HDR_MF_FLAG);
 193         }
 194 }
 195
 196 int32_t
 197 gro_udp4_reassemble(struct rte_mbuf *pkt,
 198                 struct gro_udp4_tbl *tbl,
 199                 uint64_t start_time)
 200 {
 201         struct rte_ether_hdr *eth_hdr;
 202         struct rte_ipv4_hdr *ipv4_hdr;
 203         uint16_t ip_dl;
 204         uint16_t ip_id, hdr_len;
 205         uint16_t frag_offset = 0;
 206         uint8_t is_last_frag;
 207
 208         struct udp4_flow_key key;
 209         uint32_t cur_idx, prev_idx, item_idx;
 210         uint32_t i, max_flow_num, remaining_flow_num;
 211         int cmp;
 212         uint8_t find;
 213
 214         eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
 215         ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
 216         hdr_len = pkt->l2_len + pkt->l3_len;
 217
 218         /*
 219          * Don't process non-fragment packet.
 220          */
 221         if (!is_ipv4_fragment(ipv4_hdr))
 222                 return -1;
 223
 224         /*
 225          * Don't process the packet whose payload length is less than or
 226          * equal to 0.
 227          */
 228         if (pkt->pkt_len <= hdr_len)
 229                 return -1;
 230
 231         ip_dl = rte_be_to_cpu_16(ipv4_hdr->total_length);
 232         if (ip_dl <= pkt->l3_len)
 233                 return -1;
 234
 235         ip_dl -= pkt->l3_len;
 236         ip_id = rte_be_to_cpu_16(ipv4_hdr->packet_id);
 237         frag_offset = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
 238         is_last_frag = ((frag_offset & RTE_IPV4_HDR_MF_FLAG) == 0) ? 1 : 0;
 239         frag_offset = (uint16_t)(frag_offset & RTE_IPV4_HDR_OFFSET_MASK) << 3;
 240
 241         rte_ether_addr_copy(&(eth_hdr->s_addr), &(key.eth_saddr));
 242         rte_ether_addr_copy(&(eth_hdr->d_addr), &(key.eth_daddr));
 243         key.ip_src_addr = ipv4_hdr->src_addr;
 244         key.ip_dst_addr = ipv4_hdr->dst_addr;
 245         key.ip_id = ip_id;
 246
 247         /* Search for a matched flow. */
 248         max_flow_num = tbl->max_flow_num;
 249         remaining_flow_num = tbl->flow_num;
 250         find = 0;
 251         for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
 252                 if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
 253                         if (is_same_udp4_flow(tbl->flows[i].key, key)) {
 254                                 find = 1;
 255                                 break;
 256                         }
 257                         remaining_flow_num--;
 258                 }
 259         }
 260
 261         /*
 262          * Fail to find a matched flow. Insert a new flow and store the
 263          * packet into the flow.
 264          */
 265         if (find == 0) {
 266                 item_idx = insert_new_item(tbl, pkt, start_time,
 267                                 INVALID_ARRAY_INDEX, frag_offset,
 268                                 is_last_frag);
 269                 if (unlikely(item_idx == INVALID_ARRAY_INDEX))
 270                         return -1;
 271                 if (insert_new_flow(tbl, &key, item_idx) ==
 272                                 INVALID_ARRAY_INDEX) {
 273                         /*
 274                          * Fail to insert a new flow, so delete the
 275                          * stored packet.
 276                          */
 277                         delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
 278                         return -1;
 279                 }
 280                 return 0;
 281         }
 282
 283         /*
 284          * Check all packets in the flow and try to find a neighbor for
 285          * the input packet.
 286          */
 287         cur_idx = tbl->flows[i].start_index;
 288         prev_idx = cur_idx;
 289         do {
 290                 cmp = udp4_check_neighbor(&(tbl->items[cur_idx]),
 291                                 frag_offset, ip_dl, 0);
 292                 if (cmp) {
 293                         if (merge_two_udp4_packets(&(tbl->items[cur_idx]),
 294                                                 pkt, cmp, frag_offset,
 295                                                 is_last_frag, 0))
 296                                 return 1;
 297                         /*
 298                          * Fail to merge the two packets, as the packet
 299                          * length is greater than the max value. Store
 300                          * the packet into the flow.
 301                          */
 302                         if (insert_new_item(tbl, pkt, start_time, prev_idx,
 303                                                 frag_offset, is_last_frag) ==
 304                                         INVALID_ARRAY_INDEX)
 305                                 return -1;
 306                         return 0;
 307                 }
 308
 309                 /* Ensure inserted items are ordered by frag_offset */
 310                 if (frag_offset
 311                         < tbl->items[cur_idx].frag_offset) {
 312                         break;
 313                 }
 314
 315                 prev_idx = cur_idx;
 316                 cur_idx = tbl->items[cur_idx].next_pkt_idx;
 317         } while (cur_idx != INVALID_ARRAY_INDEX);
 318
 319         /* Fail to find a neighbor, so store the packet into the flow. */
 320         if (cur_idx == tbl->flows[i].start_index) {
 321                 /* Insert it before the first packet of the flow */
 322                 item_idx = insert_new_item(tbl, pkt, start_time,
 323                                 INVALID_ARRAY_INDEX, frag_offset,
 324                                 is_last_frag);
 325                 if (unlikely(item_idx == INVALID_ARRAY_INDEX))
 326                         return -1;
 327                 tbl->items[item_idx].next_pkt_idx = cur_idx;
 328                 tbl->flows[i].start_index = item_idx;
 329         } else {
 330                 if (insert_new_item(tbl, pkt, start_time, prev_idx,
 331                                 frag_offset, is_last_frag)
 332                         == INVALID_ARRAY_INDEX)
 333                         return -1;
 334         }
 335
 336         return 0;
 337 }
 338
 339 static int
 340 gro_udp4_merge_items(struct gro_udp4_tbl *tbl,
 341                            uint32_t start_idx)
 342 {
 343         uint16_t frag_offset;
 344         uint8_t is_last_frag;
 345         int16_t ip_dl;
 346         struct rte_mbuf *pkt;
 347         int cmp;
 348         uint32_t item_idx;
 349         uint16_t hdr_len;
 350
 351         item_idx = tbl->items[start_idx].next_pkt_idx;
 352         while (item_idx != INVALID_ARRAY_INDEX) {
 353                 pkt = tbl->items[item_idx].firstseg;
 354                 hdr_len = pkt->l2_len + pkt->l3_len;
 355                 ip_dl = pkt->pkt_len - hdr_len;
 356                 frag_offset = tbl->items[item_idx].frag_offset;
 357                 is_last_frag = tbl->items[item_idx].is_last_frag;
 358                 cmp = udp4_check_neighbor(&(tbl->items[start_idx]),
 359                                         frag_offset, ip_dl, 0);
 360                 if (cmp) {
 361                         if (merge_two_udp4_packets(
 362                                         &(tbl->items[start_idx]),
 363                                         pkt, cmp, frag_offset,
 364                                         is_last_frag, 0)) {
 365                                 item_idx = delete_item(tbl, item_idx,
 366                                                         INVALID_ARRAY_INDEX);
 367                                 tbl->items[start_idx].next_pkt_idx
 368                                         = item_idx;
 369                         } else
 370                                 return 0;
 371                 } else
 372                         return 0;
 373         }
 374
 375         return 0;
 376 }
 377
 378 uint16_t
 379 gro_udp4_tbl_timeout_flush(struct gro_udp4_tbl *tbl,
 380                 uint64_t flush_timestamp,
 381                 struct rte_mbuf **out,
 382                 uint16_t nb_out)
 383 {
 384         uint16_t k = 0;
 385         uint32_t i, j;
 386         uint32_t max_flow_num = tbl->max_flow_num;
 387
 388         for (i = 0; i < max_flow_num; i++) {
 389                 if (unlikely(tbl->flow_num == 0))
 390                         return k;
 391
 392                 j = tbl->flows[i].start_index;
 393                 while (j != INVALID_ARRAY_INDEX) {
 394                         if (tbl->items[j].start_time <= flush_timestamp) {
 395                                 gro_udp4_merge_items(tbl, j);
 396                                 out[k++] = tbl->items[j].firstseg;
 397                                 if (tbl->items[j].nb_merged > 1)
 398                                         update_header(&(tbl->items[j]));
 399                                 /*
 400                                  * Delete the packet and get the next
 401                                  * packet in the flow.
 402                                  */
 403                                 j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
 404                                 tbl->flows[i].start_index = j;
 405                                 if (j == INVALID_ARRAY_INDEX)
 406                                         tbl->flow_num--;
 407
 408                                 if (unlikely(k == nb_out))
 409                                         return k;
 410                         } else
 411                                 /*
 412                                  * Flushing packets does not strictly follow
 413                                  * timestamp. It does not flush left packets of
 414                                  * the flow this time once it finds one item
 415                                  * whose start_time is greater than
 416                                  * flush_timestamp. So go to check other flows.
 417                                  */
 418                                 break;
 419                 }
 420         }
 421         return k;
 422 }
 423
 424 uint32_t
 425 gro_udp4_tbl_pkt_count(void *tbl)
 426 {
 427         struct gro_udp4_tbl *gro_tbl = tbl;
 428
 429         if (gro_tbl)
 430                 return gro_tbl->item_num;
 431
 432         return 0;
 433 }