static __rte_always_inline uint16_t
ethdev_rx_node_process_inline(struct rte_graph *graph, struct rte_node *node,
- uint16_t port, uint16_t queue)
+ ethdev_rx_node_ctx_t *ctx)
{
- uint16_t count, next_index = ETHDEV_RX_NEXT_IP4_LOOKUP;
+ uint16_t count, next_index;
+ uint16_t port, queue;
+
+ port = ctx->port_id;
+ queue = ctx->queue_id;
+ next_index = ctx->cls_next;
/* Get pkts from port */
count = rte_eth_rx_burst(port, queue, (struct rte_mbuf **)node->objs,
RTE_SET_USED(objs);
RTE_SET_USED(cnt);
- n_pkts = ethdev_rx_node_process_inline(graph, node, ctx->port_id,
- ctx->queue_id);
+ n_pkts = ethdev_rx_node_process_inline(graph, node, ctx);
return n_pkts;
}
RTE_VERIFY(elem != NULL);
+ ctx->cls_next = ETHDEV_RX_NEXT_PKT_CLS;
+
/* Check and setup ptype */
return ethdev_ptype_setup(ctx->port_id, ctx->queue_id);
}
.init = ethdev_rx_node_init,
.nb_edges = ETHDEV_RX_NEXT_MAX,
- .next_nodes = {[ETHDEV_RX_NEXT_IP4_LOOKUP] = "ip4_lookup"},
+ .next_nodes = {
+ /* Default pkt classification node */
+ [ETHDEV_RX_NEXT_PKT_CLS] = "pkt_cls",
+ [ETHDEV_RX_NEXT_IP4_LOOKUP] = "ip4_lookup",
+ },
};
struct rte_node_register *
--- /dev/null
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C) 2020 Marvell.
+ */
+
+#include <rte_debug.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_mbuf.h>
+#include <rte_graph.h>
+#include <rte_graph_worker.h>
+
+#include "pkt_cls_priv.h"
+#include "node_private.h"
+
+/* Next node for each ptype, default is '0' is "pkt_drop" */
+static const uint8_t p_nxt[256] __rte_cache_aligned = {
+ [RTE_PTYPE_L3_IPV4] = PKT_CLS_NEXT_IP4_LOOKUP,
+
+ [RTE_PTYPE_L3_IPV4_EXT] = PKT_CLS_NEXT_IP4_LOOKUP,
+
+ [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN] = PKT_CLS_NEXT_IP4_LOOKUP,
+
+ [RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER] =
+ PKT_CLS_NEXT_IP4_LOOKUP,
+
+ [RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L2_ETHER] =
+ PKT_CLS_NEXT_IP4_LOOKUP,
+
+ [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER] =
+ PKT_CLS_NEXT_IP4_LOOKUP,
+};
+
+static uint16_t
+pkt_cls_node_process(struct rte_graph *graph, struct rte_node *node,
+ void **objs, uint16_t nb_objs)
+{
+ struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
+ uint8_t l0, l1, l2, l3, last_type;
+ uint16_t next_index, n_left_from;
+ uint16_t held = 0, last_spec = 0;
+ struct pkt_cls_node_ctx *ctx;
+ void **to_next, **from;
+ uint32_t i;
+
+ pkts = (struct rte_mbuf **)objs;
+ from = objs;
+ n_left_from = nb_objs;
+
+ for (i = OBJS_PER_CLINE; i < RTE_GRAPH_BURST_SIZE; i += OBJS_PER_CLINE)
+ rte_prefetch0(&objs[i]);
+
+#if RTE_GRAPH_BURST_SIZE > 64
+ for (i = 0; i < 4 && i < n_left_from; i++)
+ rte_prefetch0(pkts[i]);
+#endif
+
+ ctx = (struct pkt_cls_node_ctx *)node->ctx;
+ last_type = ctx->l2l3_type;
+ next_index = p_nxt[last_type];
+
+ /* Get stream for the speculated next node */
+ to_next = rte_node_next_stream_get(graph, node,
+ next_index, nb_objs);
+ while (n_left_from >= 4) {
+#if RTE_GRAPH_BURST_SIZE > 64
+ if (likely(n_left_from > 7)) {
+ rte_prefetch0(pkts[4]);
+ rte_prefetch0(pkts[5]);
+ rte_prefetch0(pkts[6]);
+ rte_prefetch0(pkts[7]);
+ }
+#endif
+
+ mbuf0 = pkts[0];
+ mbuf1 = pkts[1];
+ mbuf2 = pkts[2];
+ mbuf3 = pkts[3];
+ pkts += 4;
+ n_left_from -= 4;
+
+ l0 = mbuf0->packet_type &
+ (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
+ l1 = mbuf1->packet_type &
+ (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
+ l2 = mbuf2->packet_type &
+ (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
+ l3 = mbuf3->packet_type &
+ (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
+
+ /* Check if they are destined to same
+ * next node based on l2l3 packet type.
+ */
+ uint8_t fix_spec = (last_type ^ l0) | (last_type ^ l1) |
+ (last_type ^ l2) | (last_type ^ l3);
+
+ if (unlikely(fix_spec)) {
+ /* Copy things successfully speculated till now */
+ rte_memcpy(to_next, from,
+ last_spec * sizeof(from[0]));
+ from += last_spec;
+ to_next += last_spec;
+ held += last_spec;
+ last_spec = 0;
+
+ /* l0 */
+ if (p_nxt[l0] == next_index) {
+ to_next[0] = from[0];
+ to_next++;
+ held++;
+ } else {
+ rte_node_enqueue_x1(graph, node,
+ p_nxt[l0], from[0]);
+ }
+
+ /* l1 */
+ if (p_nxt[l1] == next_index) {
+ to_next[0] = from[1];
+ to_next++;
+ held++;
+ } else {
+ rte_node_enqueue_x1(graph, node,
+ p_nxt[l1], from[1]);
+ }
+
+ /* l2 */
+ if (p_nxt[l2] == next_index) {
+ to_next[0] = from[2];
+ to_next++;
+ held++;
+ } else {
+ rte_node_enqueue_x1(graph, node,
+ p_nxt[l2], from[2]);
+ }
+
+ /* l3 */
+ if (p_nxt[l3] == next_index) {
+ to_next[0] = from[3];
+ to_next++;
+ held++;
+ } else {
+ rte_node_enqueue_x1(graph, node,
+ p_nxt[l3], from[3]);
+ }
+
+ /* Update speculated ptype */
+ if ((last_type != l3) && (l2 == l3) &&
+ (next_index != p_nxt[l3])) {
+ /* Put the current stream for
+ * speculated ltype.
+ */
+ rte_node_next_stream_put(graph, node,
+ next_index, held);
+
+ held = 0;
+
+ /* Get next stream for new ltype */
+ next_index = p_nxt[l3];
+ last_type = l3;
+ to_next = rte_node_next_stream_get(graph, node,
+ next_index,
+ nb_objs);
+ } else if (next_index == p_nxt[l3]) {
+ last_type = l3;
+ }
+
+ from += 4;
+ } else {
+ last_spec += 4;
+ }
+ }
+
+ while (n_left_from > 0) {
+ mbuf0 = pkts[0];
+
+ pkts += 1;
+ n_left_from -= 1;
+
+ l0 = mbuf0->packet_type &
+ (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
+ if (unlikely((l0 != last_type) &&
+ (p_nxt[l0] != next_index))) {
+ /* Copy things successfully speculated till now */
+ rte_memcpy(to_next, from,
+ last_spec * sizeof(from[0]));
+ from += last_spec;
+ to_next += last_spec;
+ held += last_spec;
+ last_spec = 0;
+
+ rte_node_enqueue_x1(graph, node,
+ p_nxt[l0], from[0]);
+ from += 1;
+ } else {
+ last_spec += 1;
+ }
+ }
+
+ /* !!! Home run !!! */
+ if (likely(last_spec == nb_objs)) {
+ rte_node_next_stream_move(graph, node, next_index);
+ return nb_objs;
+ }
+
+ held += last_spec;
+ /* Copy things successfully speculated till now */
+ rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
+ rte_node_next_stream_put(graph, node, next_index, held);
+
+ ctx->l2l3_type = last_type;
+ return nb_objs;
+}
+
+/* Packet Classification Node */
+struct rte_node_register pkt_cls_node = {
+ .process = pkt_cls_node_process,
+ .name = "pkt_cls",
+
+ .nb_edges = PKT_CLS_NEXT_MAX,
+ .next_nodes = {
+ /* Pkt drop node starts at '0' */
+ [PKT_CLS_NEXT_PKT_DROP] = "pkt_drop",
+ [PKT_CLS_NEXT_IP4_LOOKUP] = "ip4_lookup",
+ },
+};
+RTE_NODE_REGISTER(pkt_cls_node);
git.droids-corp.org / dpdk.git / commitdiff