node/ip4_lookup.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2020 Marvell International Ltd.
   3  */
   4
   5 #include <arpa/inet.h>
   6 #include <sys/socket.h>
   7
   8 #include <rte_debug.h>
   9 #include <rte_ethdev.h>
  10 #include <rte_ether.h>
  11 #include <rte_graph.h>
  12 #include <rte_graph_worker.h>
  13 #include <rte_ip.h>
  14 #include <rte_lpm.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_tcp.h>
  17 #include <rte_udp.h>
  18 #include <rte_vect.h>
  19
  20 #include "rte_node_ip4_api.h"
  21
  22 #include "node_private.h"
  23
  24 #define IPV4_L3FWD_LPM_MAX_RULES 1024
  25 #define IPV4_L3FWD_LPM_NUMBER_TBL8S (1 << 8)
  26
  27 /* IP4 Lookup global data struct */
  28 struct ip4_lookup_node_main {
  29         struct rte_lpm *lpm_tbl[RTE_MAX_NUMA_NODES];
  30 };
  31
  32 struct ip4_lookup_node_ctx {
  33         /* Socket's LPM table */
  34         struct rte_lpm *lpm;
  35         /* Dynamic offset to mbuf priv1 */
  36         int mbuf_priv1_off;
  37 };
  38
  39 int node_mbuf_priv1_dynfield_offset = -1;
  40
  41 static struct ip4_lookup_node_main ip4_lookup_nm;
  42
  43 #define IP4_LOOKUP_NODE_LPM(ctx) \
  44         (((struct ip4_lookup_node_ctx *)ctx)->lpm)
  45
  46 #define IP4_LOOKUP_NODE_PRIV1_OFF(ctx) \
  47         (((struct ip4_lookup_node_ctx *)ctx)->mbuf_priv1_off)
  48
  49 #if defined(__ARM_NEON)
  50 #include "ip4_lookup_neon.h"
  51 #elif defined(RTE_ARCH_X86)
  52 #include "ip4_lookup_sse.h"
  53 #endif
  54
  55 static uint16_t
  56 ip4_lookup_node_process_scalar(struct rte_graph *graph, struct rte_node *node,
  57                         void **objs, uint16_t nb_objs)
  58 {
  59         struct rte_lpm *lpm = IP4_LOOKUP_NODE_LPM(node->ctx);
  60         const int dyn = IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx);
  61         struct rte_ipv4_hdr *ipv4_hdr;
  62         void **to_next, **from;
  63         uint16_t last_spec = 0;
  64         struct rte_mbuf *mbuf;
  65         rte_edge_t next_index;
  66         uint16_t held = 0;
  67         uint32_t drop_nh;
  68         int i, rc;
  69
  70         /* Speculative next */
  71         next_index = RTE_NODE_IP4_LOOKUP_NEXT_REWRITE;
  72         /* Drop node */
  73         drop_nh = ((uint32_t)RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP) << 16;
  74         from = objs;
  75
  76         /* Get stream for the speculated next node */
  77         to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
  78         for (i = 0; i < nb_objs; i++) {
  79                 uint32_t next_hop;
  80                 uint16_t next;
  81
  82                 mbuf = (struct rte_mbuf *)objs[i];
  83
  84                 /* Extract DIP of mbuf0 */
  85                 ipv4_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *,
  86                                 sizeof(struct rte_ether_hdr));
  87                 /* Extract cksum, ttl as ipv4 hdr is in cache */
  88                 node_mbuf_priv1(mbuf, dyn)->cksum = ipv4_hdr->hdr_checksum;
  89                 node_mbuf_priv1(mbuf, dyn)->ttl = ipv4_hdr->time_to_live;
  90
  91                 rc = rte_lpm_lookup(lpm, rte_be_to_cpu_32(ipv4_hdr->dst_addr),
  92                                     &next_hop);
  93                 next_hop = (rc == 0) ? next_hop : drop_nh;
  94
  95                 node_mbuf_priv1(mbuf, dyn)->nh = (uint16_t)next_hop;
  96                 next_hop = next_hop >> 16;
  97                 next = (uint16_t)next_hop;
  98
  99                 if (unlikely(next_index != next)) {
 100                         /* Copy things successfully speculated till now */
 101                         rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
 102                         from += last_spec;
 103                         to_next += last_spec;
 104                         held += last_spec;
 105                         last_spec = 0;
 106
 107                         rte_node_enqueue_x1(graph, node, next, from[0]);
 108                         from += 1;
 109                 } else {
 110                         last_spec += 1;
 111                 }
 112         }
 113
 114         /* !!! Home run !!! */
 115         if (likely(last_spec == nb_objs)) {
 116                 rte_node_next_stream_move(graph, node, next_index);
 117                 return nb_objs;
 118         }
 119         held += last_spec;
 120         rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
 121         rte_node_next_stream_put(graph, node, next_index, held);
 122
 123         return nb_objs;
 124 }
 125
 126 int
 127 rte_node_ip4_route_add(uint32_t ip, uint8_t depth, uint16_t next_hop,
 128                        enum rte_node_ip4_lookup_next next_node)
 129 {
 130         char abuf[INET6_ADDRSTRLEN];
 131         struct in_addr in;
 132         uint8_t socket;
 133         uint32_t val;
 134         int ret;
 135
 136         in.s_addr = htonl(ip);
 137         inet_ntop(AF_INET, &in, abuf, sizeof(abuf));
 138         /* Embedded next node id into 24 bit next hop */
 139         val = ((next_node << 16) | next_hop) & ((1ull << 24) - 1);
 140         node_dbg("ip4_lookup", "LPM: Adding route %s / %d nh (0x%x)", abuf,
 141                  depth, val);
 142
 143         for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
 144                 if (!ip4_lookup_nm.lpm_tbl[socket])
 145                         continue;
 146
 147                 ret = rte_lpm_add(ip4_lookup_nm.lpm_tbl[socket],
 148                                   ip, depth, val);
 149                 if (ret < 0) {
 150                         node_err("ip4_lookup",
 151                                  "Unable to add entry %s / %d nh (%x) to LPM table on sock %d, rc=%d\n",
 152                                  abuf, depth, val, socket, ret);
 153                         return ret;
 154                 }
 155         }
 156
 157         return 0;
 158 }
 159
 160 static int
 161 setup_lpm(struct ip4_lookup_node_main *nm, int socket)
 162 {
 163         struct rte_lpm_config config_ipv4;
 164         char s[RTE_LPM_NAMESIZE];
 165
 166         /* One LPM table per socket */
 167         if (nm->lpm_tbl[socket])
 168                 return 0;
 169
 170         /* create the LPM table */
 171         config_ipv4.max_rules = IPV4_L3FWD_LPM_MAX_RULES;
 172         config_ipv4.number_tbl8s = IPV4_L3FWD_LPM_NUMBER_TBL8S;
 173         config_ipv4.flags = 0;
 174         snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socket);
 175         nm->lpm_tbl[socket] = rte_lpm_create(s, socket, &config_ipv4);
 176         if (nm->lpm_tbl[socket] == NULL)
 177                 return -rte_errno;
 178
 179         return 0;
 180 }
 181
 182 static int
 183 ip4_lookup_node_init(const struct rte_graph *graph, struct rte_node *node)
 184 {
 185         uint16_t socket, lcore_id;
 186         static uint8_t init_once;
 187         int rc;
 188
 189         RTE_SET_USED(graph);
 190         RTE_BUILD_BUG_ON(sizeof(struct ip4_lookup_node_ctx) > RTE_NODE_CTX_SZ);
 191
 192         if (!init_once) {
 193                 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register(
 194                                 &node_mbuf_priv1_dynfield_desc);
 195                 if (node_mbuf_priv1_dynfield_offset < 0)
 196                         return -rte_errno;
 197
 198                 /* Setup LPM tables for all sockets */
 199                 RTE_LCORE_FOREACH(lcore_id)
 200                 {
 201                         socket = rte_lcore_to_socket_id(lcore_id);
 202                         rc = setup_lpm(&ip4_lookup_nm, socket);
 203                         if (rc) {
 204                                 node_err("ip4_lookup",
 205                                          "Failed to setup lpm tbl for sock %u, rc=%d",
 206                                          socket, rc);
 207                                 return rc;
 208                         }
 209                 }
 210                 init_once = 1;
 211         }
 212
 213         /* Update socket's LPM and mbuf dyn priv1 offset in node ctx */
 214         IP4_LOOKUP_NODE_LPM(node->ctx) = ip4_lookup_nm.lpm_tbl[graph->socket];
 215         IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset;
 216
 217 #if defined(__ARM_NEON) || defined(RTE_ARCH_X86)
 218         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
 219                 node->process = ip4_lookup_node_process_vec;
 220 #endif
 221
 222         node_dbg("ip4_lookup", "Initialized ip4_lookup node");
 223
 224         return 0;
 225 }
 226
 227 static struct rte_node_register ip4_lookup_node = {
 228         .process = ip4_lookup_node_process_scalar,
 229         .name = "ip4_lookup",
 230
 231         .init = ip4_lookup_node_init,
 232
 233         .nb_edges = RTE_NODE_IP4_LOOKUP_NEXT_MAX,
 234         .next_nodes = {
 235                 [RTE_NODE_IP4_LOOKUP_NEXT_REWRITE] = "ip4_rewrite",
 236                 [RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP] = "pkt_drop",
 237         },
 238 };
 239
 240 RTE_NODE_REGISTER(ip4_lookup_node);