lib/node/pkt_cls.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2020 Marvell.
   3  */
   4
   5 #include <rte_graph.h>
   6 #include <rte_graph_worker.h>
   7
   8 #include "pkt_cls_priv.h"
   9 #include "node_private.h"
  10
  11 /* Next node for each ptype, default is '0' is "pkt_drop" */
  12 static const uint8_t p_nxt[256] __rte_cache_aligned = {
  13         [RTE_PTYPE_L3_IPV4] = PKT_CLS_NEXT_IP4_LOOKUP,
  14
  15         [RTE_PTYPE_L3_IPV4_EXT] = PKT_CLS_NEXT_IP4_LOOKUP,
  16
  17         [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN] = PKT_CLS_NEXT_IP4_LOOKUP,
  18
  19         [RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER] =
  20                 PKT_CLS_NEXT_IP4_LOOKUP,
  21
  22         [RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L2_ETHER] =
  23                 PKT_CLS_NEXT_IP4_LOOKUP,
  24
  25         [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER] =
  26                 PKT_CLS_NEXT_IP4_LOOKUP,
  27 };
  28
  29 static uint16_t
  30 pkt_cls_node_process(struct rte_graph *graph, struct rte_node *node,
  31                      void **objs, uint16_t nb_objs)
  32 {
  33         struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
  34         uint8_t l0, l1, l2, l3, last_type;
  35         uint16_t next_index, n_left_from;
  36         uint16_t held = 0, last_spec = 0;
  37         struct pkt_cls_node_ctx *ctx;
  38         void **to_next, **from;
  39         uint32_t i;
  40
  41         pkts = (struct rte_mbuf **)objs;
  42         from = objs;
  43         n_left_from = nb_objs;
  44
  45         for (i = OBJS_PER_CLINE; i < RTE_GRAPH_BURST_SIZE; i += OBJS_PER_CLINE)
  46                 rte_prefetch0(&objs[i]);
  47
  48 #if RTE_GRAPH_BURST_SIZE > 64
  49         for (i = 0; i < 4 && i < n_left_from; i++)
  50                 rte_prefetch0(pkts[i]);
  51 #endif
  52
  53         ctx = (struct pkt_cls_node_ctx *)node->ctx;
  54         last_type = ctx->l2l3_type;
  55         next_index = p_nxt[last_type];
  56
  57         /* Get stream for the speculated next node */
  58         to_next = rte_node_next_stream_get(graph, node,
  59                                            next_index, nb_objs);
  60         while (n_left_from >= 4) {
  61 #if RTE_GRAPH_BURST_SIZE > 64
  62                 if (likely(n_left_from > 7)) {
  63                         rte_prefetch0(pkts[4]);
  64                         rte_prefetch0(pkts[5]);
  65                         rte_prefetch0(pkts[6]);
  66                         rte_prefetch0(pkts[7]);
  67                 }
  68 #endif
  69
  70                 mbuf0 = pkts[0];
  71                 mbuf1 = pkts[1];
  72                 mbuf2 = pkts[2];
  73                 mbuf3 = pkts[3];
  74                 pkts += 4;
  75                 n_left_from -= 4;
  76
  77                 l0 = mbuf0->packet_type &
  78                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
  79                 l1 = mbuf1->packet_type &
  80                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
  81                 l2 = mbuf2->packet_type &
  82                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
  83                 l3 = mbuf3->packet_type &
  84                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
  85
  86                 /* Check if they are destined to same
  87                  * next node based on l2l3 packet type.
  88                  */
  89                 uint8_t fix_spec = (last_type ^ l0) | (last_type ^ l1) |
  90                         (last_type ^ l2) | (last_type ^ l3);
  91
  92                 if (unlikely(fix_spec)) {
  93                         /* Copy things successfully speculated till now */
  94                         rte_memcpy(to_next, from,
  95                                    last_spec * sizeof(from[0]));
  96                         from += last_spec;
  97                         to_next += last_spec;
  98                         held += last_spec;
  99                         last_spec = 0;
 100
 101                         /* l0 */
 102                         if (p_nxt[l0] == next_index) {
 103                                 to_next[0] = from[0];
 104                                 to_next++;
 105                                 held++;
 106                         } else {
 107                                 rte_node_enqueue_x1(graph, node,
 108                                                     p_nxt[l0], from[0]);
 109                         }
 110
 111                         /* l1 */
 112                         if (p_nxt[l1] == next_index) {
 113                                 to_next[0] = from[1];
 114                                 to_next++;
 115                                 held++;
 116                         } else {
 117                                 rte_node_enqueue_x1(graph, node,
 118                                                     p_nxt[l1], from[1]);
 119                         }
 120
 121                         /* l2 */
 122                         if (p_nxt[l2] == next_index) {
 123                                 to_next[0] = from[2];
 124                                 to_next++;
 125                                 held++;
 126                         } else {
 127                                 rte_node_enqueue_x1(graph, node,
 128                                                     p_nxt[l2], from[2]);
 129                         }
 130
 131                         /* l3 */
 132                         if (p_nxt[l3] == next_index) {
 133                                 to_next[0] = from[3];
 134                                 to_next++;
 135                                 held++;
 136                         } else {
 137                                 rte_node_enqueue_x1(graph, node,
 138                                                     p_nxt[l3], from[3]);
 139                         }
 140
 141                         /* Update speculated ptype */
 142                         if ((last_type != l3) && (l2 == l3) &&
 143                             (next_index != p_nxt[l3])) {
 144                                 /* Put the current stream for
 145                                  * speculated ltype.
 146                                  */
 147                                 rte_node_next_stream_put(graph, node,
 148                                                          next_index, held);
 149
 150                                 held = 0;
 151
 152                                 /* Get next stream for new ltype */
 153                                 next_index = p_nxt[l3];
 154                                 last_type = l3;
 155                                 to_next = rte_node_next_stream_get(graph, node,
 156                                                                    next_index,
 157                                                                    nb_objs);
 158                         } else if (next_index == p_nxt[l3]) {
 159                                 last_type = l3;
 160                         }
 161
 162                         from += 4;
 163                 } else {
 164                         last_spec += 4;
 165                 }
 166         }
 167
 168         while (n_left_from > 0) {
 169                 mbuf0 = pkts[0];
 170
 171                 pkts += 1;
 172                 n_left_from -= 1;
 173
 174                 l0 = mbuf0->packet_type &
 175                         (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
 176                 if (unlikely((l0 != last_type) &&
 177                              (p_nxt[l0] != next_index))) {
 178                         /* Copy things successfully speculated till now */
 179                         rte_memcpy(to_next, from,
 180                                    last_spec * sizeof(from[0]));
 181                         from += last_spec;
 182                         to_next += last_spec;
 183                         held += last_spec;
 184                         last_spec = 0;
 185
 186                         rte_node_enqueue_x1(graph, node,
 187                                             p_nxt[l0], from[0]);
 188                         from += 1;
 189                 } else {
 190                         last_spec += 1;
 191                 }
 192         }
 193
 194         /* !!! Home run !!! */
 195         if (likely(last_spec == nb_objs)) {
 196                 rte_node_next_stream_move(graph, node, next_index);
 197                 return nb_objs;
 198         }
 199
 200         held += last_spec;
 201         /* Copy things successfully speculated till now */
 202         rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
 203         rte_node_next_stream_put(graph, node, next_index, held);
 204
 205         ctx->l2l3_type = last_type;
 206         return nb_objs;
 207 }
 208
 209 /* Packet Classification Node */
 210 struct rte_node_register pkt_cls_node = {
 211         .process = pkt_cls_node_process,
 212         .name = "pkt_cls",
 213
 214         .nb_edges = PKT_CLS_NEXT_MAX,
 215         .next_nodes = {
 216                 /* Pkt drop node starts at '0' */
 217                 [PKT_CLS_NEXT_PKT_DROP] = "pkt_drop",
 218                 [PKT_CLS_NEXT_IP4_LOOKUP] = "ip4_lookup",
 219         },
 220 };
 221 RTE_NODE_REGISTER(pkt_cls_node);