lib/librte_node/pkt_cls.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2020 Marvell.
   3  */
   4
   5 #include <rte_debug.h>
   6 #include <rte_ether.h>
   7 #include <rte_ethdev.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_graph.h>
  10 #include <rte_graph_worker.h>
  11
  12 #include "pkt_cls_priv.h"
  13 #include "node_private.h"
  14
  15 /* Next node for each ptype, default is '0' is "pkt_drop" */
  16 static const uint8_t p_nxt[256] __rte_cache_aligned = {
  17         [RTE_PTYPE_L3_IPV4] = PKT_CLS_NEXT_IP4_LOOKUP,
  18
  19         [RTE_PTYPE_L3_IPV4_EXT] = PKT_CLS_NEXT_IP4_LOOKUP,
  20
  21         [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN] = PKT_CLS_NEXT_IP4_LOOKUP,
  22
  23         [RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER] =
  24                 PKT_CLS_NEXT_IP4_LOOKUP,
  25
  26         [RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L2_ETHER] =
  27                 PKT_CLS_NEXT_IP4_LOOKUP,
  28
  29         [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER] =
  30                 PKT_CLS_NEXT_IP4_LOOKUP,
  31 };
  32
  33 static uint16_t
  34 pkt_cls_node_process(struct rte_graph *graph, struct rte_node *node,
  35                      void **objs, uint16_t nb_objs)
  36 {
  37         struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
  38         uint8_t l0, l1, l2, l3, last_type;
  39         uint16_t next_index, n_left_from;
  40         uint16_t held = 0, last_spec = 0;
  41         struct pkt_cls_node_ctx *ctx;
  42         void **to_next, **from;
  43         uint32_t i;
  44
  45         pkts = (struct rte_mbuf **)objs;
  46         from = objs;
  47         n_left_from = nb_objs;
  48
  49         for (i = OBJS_PER_CLINE; i < RTE_GRAPH_BURST_SIZE; i += OBJS_PER_CLINE)
  50                 rte_prefetch0(&objs[i]);
  51
  52 #if RTE_GRAPH_BURST_SIZE > 64
  53         for (i = 0; i < 4 && i < n_left_from; i++)
  54                 rte_prefetch0(pkts[i]);
  55 #endif
  56
  57         ctx = (struct pkt_cls_node_ctx *)node->ctx;
  58         last_type = ctx->l2l3_type;
  59         next_index = p_nxt[last_type];
  60
  61         /* Get stream for the speculated next node */
  62         to_next = rte_node_next_stream_get(graph, node,
  63                                            next_index, nb_objs);
  64         while (n_left_from >= 4) {
  65 #if RTE_GRAPH_BURST_SIZE > 64
  66                 if (likely(n_left_from > 7)) {
  67                         rte_prefetch0(pkts[4]);
  68                         rte_prefetch0(pkts[5]);
  69                         rte_prefetch0(pkts[6]);
  70                         rte_prefetch0(pkts[7]);
  71                 }
  72 #endif
  73
  74                 mbuf0 = pkts[0];
  75                 mbuf1 = pkts[1];
  76                 mbuf2 = pkts[2];
  77                 mbuf3 = pkts[3];
  78                 pkts += 4;
  79                 n_left_from -= 4;
  80
  81                 l0 = mbuf0->packet_type &
  82                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
  83                 l1 = mbuf1->packet_type &
  84                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
  85                 l2 = mbuf2->packet_type &
  86                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
  87                 l3 = mbuf3->packet_type &
  88                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
  89
  90                 /* Check if they are destined to same
  91                  * next node based on l2l3 packet type.
  92                  */
  93                 uint8_t fix_spec = (last_type ^ l0) | (last_type ^ l1) |
  94                         (last_type ^ l2) | (last_type ^ l3);
  95
  96                 if (unlikely(fix_spec)) {
  97                         /* Copy things successfully speculated till now */
  98                         rte_memcpy(to_next, from,
  99                                    last_spec * sizeof(from[0]));
 100                         from += last_spec;
 101                         to_next += last_spec;
 102                         held += last_spec;
 103                         last_spec = 0;
 104
 105                         /* l0 */
 106                         if (p_nxt[l0] == next_index) {
 107                                 to_next[0] = from[0];
 108                                 to_next++;
 109                                 held++;
 110                         } else {
 111                                 rte_node_enqueue_x1(graph, node,
 112                                                     p_nxt[l0], from[0]);
 113                         }
 114
 115                         /* l1 */
 116                         if (p_nxt[l1] == next_index) {
 117                                 to_next[0] = from[1];
 118                                 to_next++;
 119                                 held++;
 120                         } else {
 121                                 rte_node_enqueue_x1(graph, node,
 122                                                     p_nxt[l1], from[1]);
 123                         }
 124
 125                         /* l2 */
 126                         if (p_nxt[l2] == next_index) {
 127                                 to_next[0] = from[2];
 128                                 to_next++;
 129                                 held++;
 130                         } else {
 131                                 rte_node_enqueue_x1(graph, node,
 132                                                     p_nxt[l2], from[2]);
 133                         }
 134
 135                         /* l3 */
 136                         if (p_nxt[l3] == next_index) {
 137                                 to_next[0] = from[3];
 138                                 to_next++;
 139                                 held++;
 140                         } else {
 141                                 rte_node_enqueue_x1(graph, node,
 142                                                     p_nxt[l3], from[3]);
 143                         }
 144
 145                         /* Update speculated ptype */
 146                         if ((last_type != l3) && (l2 == l3) &&
 147                             (next_index != p_nxt[l3])) {
 148                                 /* Put the current stream for
 149                                  * speculated ltype.
 150                                  */
 151                                 rte_node_next_stream_put(graph, node,
 152                                                          next_index, held);
 153
 154                                 held = 0;
 155
 156                                 /* Get next stream for new ltype */
 157                                 next_index = p_nxt[l3];
 158                                 last_type = l3;
 159                                 to_next = rte_node_next_stream_get(graph, node,
 160                                                                    next_index,
 161                                                                    nb_objs);
 162                         } else if (next_index == p_nxt[l3]) {
 163                                 last_type = l3;
 164                         }
 165
 166                         from += 4;
 167                 } else {
 168                         last_spec += 4;
 169                 }
 170         }
 171
 172         while (n_left_from > 0) {
 173                 mbuf0 = pkts[0];
 174
 175                 pkts += 1;
 176                 n_left_from -= 1;
 177
 178                 l0 = mbuf0->packet_type &
 179                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
 180                 if (unlikely((l0 != last_type) &&
 181                              (p_nxt[l0] != next_index))) {
 182                         /* Copy things successfully speculated till now */
 183                         rte_memcpy(to_next, from,
 184                                    last_spec * sizeof(from[0]));
 185                         from += last_spec;
 186                         to_next += last_spec;
 187                         held += last_spec;
 188                         last_spec = 0;
 189
 190                         rte_node_enqueue_x1(graph, node,
 191                                             p_nxt[l0], from[0]);
 192                         from += 1;
 193                 } else {
 194                         last_spec += 1;
 195                 }
 196         }
 197
 198         /* !!! Home run !!! */
 199         if (likely(last_spec == nb_objs)) {
 200                 rte_node_next_stream_move(graph, node, next_index);
 201                 return nb_objs;
 202         }
 203
 204         held += last_spec;
 205         /* Copy things successfully speculated till now */
 206         rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
 207         rte_node_next_stream_put(graph, node, next_index, held);
 208
 209         ctx->l2l3_type = last_type;
 210         return nb_objs;
 211 }
 212
 213 /* Packet Classification Node */
 214 struct rte_node_register pkt_cls_node = {
 215         .process = pkt_cls_node_process,
 216         .name = "pkt_cls",
 217
 218         .nb_edges = PKT_CLS_NEXT_MAX,
 219         .next_nodes = {
 220                 /* Pkt drop node starts at '0' */
 221                 [PKT_CLS_NEXT_PKT_DROP] = "pkt_drop",
 222                 [PKT_CLS_NEXT_IP4_LOOKUP] = "ip4_lookup",
 223         },
 224 };
 225 RTE_NODE_REGISTER(pkt_cls_node);