node/ip4_rewrite.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2020 Marvell International Ltd.
   3  */
   4
   5 #include <rte_debug.h>
   6 #include <rte_ethdev.h>
   7 #include <rte_ether.h>
   8 #include <rte_graph.h>
   9 #include <rte_graph_worker.h>
  10 #include <rte_ip.h>
  11 #include <rte_malloc.h>
  12 #include <rte_mbuf.h>
  13 #include <rte_tcp.h>
  14 #include <rte_udp.h>
  15 #include <rte_vect.h>
  16
  17 #include "rte_node_ip4_api.h"
  18
  19 #include "ip4_rewrite_priv.h"
  20 #include "node_private.h"
  21
  22 struct ip4_rewrite_node_ctx {
  23         /* Dynamic offset to mbuf priv1 */
  24         int mbuf_priv1_off;
  25         /* Cached next index */
  26         uint16_t next_index;
  27 };
  28
  29 static struct ip4_rewrite_node_main *ip4_rewrite_nm;
  30
  31 #define IP4_REWRITE_NODE_LAST_NEXT(ctx) \
  32         (((struct ip4_rewrite_node_ctx *)ctx)->next_index)
  33
  34 #define IP4_REWRITE_NODE_PRIV1_OFF(ctx) \
  35         (((struct ip4_rewrite_node_ctx *)ctx)->mbuf_priv1_off)
  36
  37 static uint16_t
  38 ip4_rewrite_node_process(struct rte_graph *graph, struct rte_node *node,
  39                          void **objs, uint16_t nb_objs)
  40 {
  41         struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
  42         struct ip4_rewrite_nh_header *nh = ip4_rewrite_nm->nh;
  43         const int dyn = IP4_REWRITE_NODE_PRIV1_OFF(node->ctx);
  44         uint16_t next0, next1, next2, next3, next_index;
  45         struct rte_ipv4_hdr *ip0, *ip1, *ip2, *ip3;
  46         uint16_t n_left_from, held = 0, last_spec = 0;
  47         void *d0, *d1, *d2, *d3;
  48         void **to_next, **from;
  49         rte_xmm_t priv01;
  50         rte_xmm_t priv23;
  51         int i;
  52
  53         /* Speculative next as last next */
  54         next_index = IP4_REWRITE_NODE_LAST_NEXT(node->ctx);
  55         rte_prefetch0(nh);
  56
  57         pkts = (struct rte_mbuf **)objs;
  58         from = objs;
  59         n_left_from = nb_objs;
  60
  61         for (i = 0; i < 4 && i < n_left_from; i++)
  62                 rte_prefetch0(pkts[i]);
  63
  64         /* Get stream for the speculated next node */
  65         to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
  66         /* Update Ethernet header of pkts */
  67         while (n_left_from >= 4) {
  68                 if (likely(n_left_from > 7)) {
  69                         /* Prefetch only next-mbuf struct and priv area.
  70                          * Data need not be prefetched as we only write.
  71                          */
  72                         rte_prefetch0(pkts[4]);
  73                         rte_prefetch0(pkts[5]);
  74                         rte_prefetch0(pkts[6]);
  75                         rte_prefetch0(pkts[7]);
  76                 }
  77
  78                 mbuf0 = pkts[0];
  79                 mbuf1 = pkts[1];
  80                 mbuf2 = pkts[2];
  81                 mbuf3 = pkts[3];
  82
  83                 pkts += 4;
  84                 n_left_from -= 4;
  85                 priv01.u64[0] = node_mbuf_priv1(mbuf0, dyn)->u;
  86                 priv01.u64[1] = node_mbuf_priv1(mbuf1, dyn)->u;
  87                 priv23.u64[0] = node_mbuf_priv1(mbuf2, dyn)->u;
  88                 priv23.u64[1] = node_mbuf_priv1(mbuf3, dyn)->u;
  89
  90                 /* Increment checksum by one. */
  91                 priv01.u32[1] += rte_cpu_to_be_16(0x0100);
  92                 priv01.u32[3] += rte_cpu_to_be_16(0x0100);
  93                 priv23.u32[1] += rte_cpu_to_be_16(0x0100);
  94                 priv23.u32[3] += rte_cpu_to_be_16(0x0100);
  95
  96                 /* Update ttl,cksum rewrite ethernet hdr on mbuf0 */
  97                 d0 = rte_pktmbuf_mtod(mbuf0, void *);
  98                 rte_memcpy(d0, nh[priv01.u16[0]].rewrite_data,
  99                            nh[priv01.u16[0]].rewrite_len);
 100
 101                 next0 = nh[priv01.u16[0]].tx_node;
 102                 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 +
 103                                               sizeof(struct rte_ether_hdr));
 104                 ip0->time_to_live = priv01.u16[1] - 1;
 105                 ip0->hdr_checksum = priv01.u16[2] + priv01.u16[3];
 106
 107                 /* Update ttl,cksum rewrite ethernet hdr on mbuf1 */
 108                 d1 = rte_pktmbuf_mtod(mbuf1, void *);
 109                 rte_memcpy(d1, nh[priv01.u16[4]].rewrite_data,
 110                            nh[priv01.u16[4]].rewrite_len);
 111
 112                 next1 = nh[priv01.u16[4]].tx_node;
 113                 ip1 = (struct rte_ipv4_hdr *)((uint8_t *)d1 +
 114                                               sizeof(struct rte_ether_hdr));
 115                 ip1->time_to_live = priv01.u16[5] - 1;
 116                 ip1->hdr_checksum = priv01.u16[6] + priv01.u16[7];
 117
 118                 /* Update ttl,cksum rewrite ethernet hdr on mbuf2 */
 119                 d2 = rte_pktmbuf_mtod(mbuf2, void *);
 120                 rte_memcpy(d2, nh[priv23.u16[0]].rewrite_data,
 121                            nh[priv23.u16[0]].rewrite_len);
 122                 next2 = nh[priv23.u16[0]].tx_node;
 123                 ip2 = (struct rte_ipv4_hdr *)((uint8_t *)d2 +
 124                                               sizeof(struct rte_ether_hdr));
 125                 ip2->time_to_live = priv23.u16[1] - 1;
 126                 ip2->hdr_checksum = priv23.u16[2] + priv23.u16[3];
 127
 128                 /* Update ttl,cksum rewrite ethernet hdr on mbuf3 */
 129                 d3 = rte_pktmbuf_mtod(mbuf3, void *);
 130                 rte_memcpy(d3, nh[priv23.u16[4]].rewrite_data,
 131                            nh[priv23.u16[4]].rewrite_len);
 132
 133                 next3 = nh[priv23.u16[4]].tx_node;
 134                 ip3 = (struct rte_ipv4_hdr *)((uint8_t *)d3 +
 135                                               sizeof(struct rte_ether_hdr));
 136                 ip3->time_to_live = priv23.u16[5] - 1;
 137                 ip3->hdr_checksum = priv23.u16[6] + priv23.u16[7];
 138
 139                 /* Enqueue four to next node */
 140                 rte_edge_t fix_spec =
 141                         ((next_index == next0) && (next0 == next1) &&
 142                          (next1 == next2) && (next2 == next3));
 143
 144                 if (unlikely(fix_spec == 0)) {
 145                         /* Copy things successfully speculated till now */
 146                         rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
 147                         from += last_spec;
 148                         to_next += last_spec;
 149                         held += last_spec;
 150                         last_spec = 0;
 151
 152                         /* next0 */
 153                         if (next_index == next0) {
 154                                 to_next[0] = from[0];
 155                                 to_next++;
 156                                 held++;
 157                         } else {
 158                                 rte_node_enqueue_x1(graph, node, next0,
 159                                                     from[0]);
 160                         }
 161
 162                         /* next1 */
 163                         if (next_index == next1) {
 164                                 to_next[0] = from[1];
 165                                 to_next++;
 166                                 held++;
 167                         } else {
 168                                 rte_node_enqueue_x1(graph, node, next1,
 169                                                     from[1]);
 170                         }
 171
 172                         /* next2 */
 173                         if (next_index == next2) {
 174                                 to_next[0] = from[2];
 175                                 to_next++;
 176                                 held++;
 177                         } else {
 178                                 rte_node_enqueue_x1(graph, node, next2,
 179                                                     from[2]);
 180                         }
 181
 182                         /* next3 */
 183                         if (next_index == next3) {
 184                                 to_next[0] = from[3];
 185                                 to_next++;
 186                                 held++;
 187                         } else {
 188                                 rte_node_enqueue_x1(graph, node, next3,
 189                                                     from[3]);
 190                         }
 191
 192                         from += 4;
 193
 194                         /* Change speculation if last two are same */
 195                         if ((next_index != next3) && (next2 == next3)) {
 196                                 /* Put the current speculated node */
 197                                 rte_node_next_stream_put(graph, node,
 198                                                          next_index, held);
 199                                 held = 0;
 200
 201                                 /* Get next speculated stream */
 202                                 next_index = next3;
 203                                 to_next = rte_node_next_stream_get(
 204                                         graph, node, next_index, nb_objs);
 205                         }
 206                 } else {
 207                         last_spec += 4;
 208                 }
 209         }
 210
 211         while (n_left_from > 0) {
 212                 uint16_t chksum;
 213
 214                 mbuf0 = pkts[0];
 215
 216                 pkts += 1;
 217                 n_left_from -= 1;
 218
 219                 d0 = rte_pktmbuf_mtod(mbuf0, void *);
 220                 rte_memcpy(d0, nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_data,
 221                            nh[node_mbuf_priv1(mbuf0, dyn)->nh].rewrite_len);
 222
 223                 next0 = nh[node_mbuf_priv1(mbuf0, dyn)->nh].tx_node;
 224                 ip0 = (struct rte_ipv4_hdr *)((uint8_t *)d0 +
 225                                               sizeof(struct rte_ether_hdr));
 226                 chksum = node_mbuf_priv1(mbuf0, dyn)->cksum +
 227                          rte_cpu_to_be_16(0x0100);
 228                 chksum += chksum >= 0xffff;
 229                 ip0->hdr_checksum = chksum;
 230                 ip0->time_to_live = node_mbuf_priv1(mbuf0, dyn)->ttl - 1;
 231
 232                 if (unlikely(next_index ^ next0)) {
 233                         /* Copy things successfully speculated till now */
 234                         rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
 235                         from += last_spec;
 236                         to_next += last_spec;
 237                         held += last_spec;
 238                         last_spec = 0;
 239
 240                         rte_node_enqueue_x1(graph, node, next0, from[0]);
 241                         from += 1;
 242                 } else {
 243                         last_spec += 1;
 244                 }
 245         }
 246
 247         /* !!! Home run !!! */
 248         if (likely(last_spec == nb_objs)) {
 249                 rte_node_next_stream_move(graph, node, next_index);
 250                 return nb_objs;
 251         }
 252
 253         held += last_spec;
 254         rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
 255         rte_node_next_stream_put(graph, node, next_index, held);
 256         /* Save the last next used */
 257         IP4_REWRITE_NODE_LAST_NEXT(node->ctx) = next_index;
 258
 259         return nb_objs;
 260 }
 261
 262 static int
 263 ip4_rewrite_node_init(const struct rte_graph *graph, struct rte_node *node)
 264 {
 265         static bool init_once;
 266
 267         RTE_SET_USED(graph);
 268         RTE_BUILD_BUG_ON(sizeof(struct ip4_rewrite_node_ctx) > RTE_NODE_CTX_SZ);
 269
 270         if (!init_once) {
 271                 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register(
 272                                 &node_mbuf_priv1_dynfield_desc);
 273                 if (node_mbuf_priv1_dynfield_offset < 0)
 274                         return -rte_errno;
 275                 init_once = true;
 276         }
 277         IP4_REWRITE_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset;
 278
 279         node_dbg("ip4_rewrite", "Initialized ip4_rewrite node initialized");
 280
 281         return 0;
 282 }
 283
 284 int
 285 ip4_rewrite_set_next(uint16_t port_id, uint16_t next_index)
 286 {
 287         if (ip4_rewrite_nm == NULL) {
 288                 ip4_rewrite_nm = rte_zmalloc(
 289                         "ip4_rewrite", sizeof(struct ip4_rewrite_node_main),
 290                         RTE_CACHE_LINE_SIZE);
 291                 if (ip4_rewrite_nm == NULL)
 292                         return -ENOMEM;
 293         }
 294         ip4_rewrite_nm->next_index[port_id] = next_index;
 295
 296         return 0;
 297 }
 298
 299 int
 300 rte_node_ip4_rewrite_add(uint16_t next_hop, uint8_t *rewrite_data,
 301                          uint8_t rewrite_len, uint16_t dst_port)
 302 {
 303         struct ip4_rewrite_nh_header *nh;
 304
 305         if (next_hop >= RTE_GRAPH_IP4_REWRITE_MAX_NH)
 306                 return -EINVAL;
 307
 308         if (rewrite_len > RTE_GRAPH_IP4_REWRITE_MAX_LEN)
 309                 return -EINVAL;
 310
 311         if (ip4_rewrite_nm == NULL) {
 312                 ip4_rewrite_nm = rte_zmalloc(
 313                         "ip4_rewrite", sizeof(struct ip4_rewrite_node_main),
 314                         RTE_CACHE_LINE_SIZE);
 315                 if (ip4_rewrite_nm == NULL)
 316                         return -ENOMEM;
 317         }
 318
 319         /* Check if dst port doesn't exist as edge */
 320         if (!ip4_rewrite_nm->next_index[dst_port])
 321                 return -EINVAL;
 322
 323         /* Update next hop */
 324         nh = &ip4_rewrite_nm->nh[next_hop];
 325
 326         memcpy(nh->rewrite_data, rewrite_data, rewrite_len);
 327         nh->tx_node = ip4_rewrite_nm->next_index[dst_port];
 328         nh->rewrite_len = rewrite_len;
 329         nh->enabled = true;
 330
 331         return 0;
 332 }
 333
 334 static struct rte_node_register ip4_rewrite_node = {
 335         .process = ip4_rewrite_node_process,
 336         .name = "ip4_rewrite",
 337         /* Default edge i.e '0' is pkt drop */
 338         .nb_edges = 1,
 339         .next_nodes = {
 340                 [0] = "pkt_drop",
 341         },
 342         .init = ip4_rewrite_node_init,
 343 };
 344
 345 struct rte_node_register *
 346 ip4_rewrite_node_get(void)
 347 {
 348         return &ip4_rewrite_node;
 349 }
 350
 351 RTE_NODE_REGISTER(ip4_rewrite_node);