From: Jerin Jacob Date: Sat, 11 Apr 2020 14:14:05 +0000 (+0530) Subject: graph: populate fastpath memory for graph reel X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=d6bba625cd5097199bde4a766a03e61bb2349778;p=dpdk.git graph: populate fastpath memory for graph reel Adding support to create and populate the memory for graph reel. This includes reserving the memory in the memzone, populating the nodes, Allocating memory for node-specific streams to hold objects. Once it is populated the reel memory contains the following sections. +---------------------+ | Graph Header | +---------------------+ | Fence | +---------------------+ | Circular buffer | +---------------------+ | Fence | +---------------------+ | Node Object 0 | +------------------- -+ | Node Object 1 | +------------------- -+ | Node Object 2 | +------------------- -+ | Node Object n | +------------------- -+ Signed-off-by: Jerin Jacob Signed-off-by: Kiran Kumar K Signed-off-by: Pavan Nikhilesh Signed-off-by: Nithin Dabilpuram --- diff --git a/lib/librte_graph/Makefile b/lib/librte_graph/Makefile index 9cb8066e75..fee756e89f 100644 --- a/lib/librte_graph/Makefile +++ b/lib/librte_graph/Makefile @@ -18,8 +18,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += node.c SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph.c SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph_ops.c SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph_debug.c +SRCS-$(CONFIG_RTE_LIBRTE_GRAPH) += graph_populate.c # install header files SYMLINK-$(CONFIG_RTE_LIBRTE_GRAPH)-include += rte_graph.h +SYMLINK-$(CONFIG_RTE_LIBRTE_GRAPH)-include += rte_graph_worker.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_graph/graph.c b/lib/librte_graph/graph.c index 21c32cfeda..96004830ca 100644 --- a/lib/librte_graph/graph.c +++ b/lib/librte_graph/graph.c @@ -2,6 +2,7 @@ * Copyright(C) 2020 Marvell International Ltd. */ +#include #include #include "graph_private.h" @@ -20,3 +21,18 @@ graph_spinlock_unlock(void) { rte_spinlock_unlock(&graph_lock); } + +void __rte_noinline +__rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node) +{ + uint16_t size = node->size; + + RTE_VERIFY(size != UINT16_MAX); + /* Allocate double amount of size to avoid immediate realloc */ + size = RTE_MIN(UINT16_MAX, RTE_MAX(RTE_GRAPH_BURST_SIZE, size * 2)); + node->objs = rte_realloc_socket(node->objs, size * sizeof(void *), + RTE_CACHE_LINE_SIZE, graph->socket); + RTE_VERIFY(node->objs); + node->size = size; + node->realloc_count++; +} diff --git a/lib/librte_graph/graph_populate.c b/lib/librte_graph/graph_populate.c new file mode 100644 index 0000000000..093512efab --- /dev/null +++ b/lib/librte_graph/graph_populate.c @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2020 Marvell International Ltd. + */ + +#include +#include + +#include +#include +#include +#include + +#include "graph_private.h" + +static size_t +graph_fp_mem_calc_size(struct graph *graph) +{ + struct graph_node *graph_node; + rte_node_t val; + size_t sz; + + /* Graph header */ + sz = sizeof(struct rte_graph); + /* Source nodes list */ + sz += sizeof(rte_graph_off_t) * graph->src_node_count; + /* Circular buffer for pending streams of size number of nodes */ + val = rte_align32pow2(graph->node_count * sizeof(rte_graph_off_t)); + sz = RTE_ALIGN(sz, val); + graph->cir_start = sz; + graph->cir_mask = rte_align32pow2(graph->node_count) - 1; + sz += val; + /* Fence */ + sz += sizeof(RTE_GRAPH_FENCE); + sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); + graph->nodes_start = sz; + /* For 0..N node objects with fence */ + STAILQ_FOREACH(graph_node, &graph->node_list, next) { + sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); + sz += sizeof(struct rte_node); + /* Pointer to next nodes(edges) */ + sz += sizeof(struct rte_node *) * graph_node->node->nb_edges; + } + + graph->mem_sz = sz; + return sz; +} + +static void +graph_header_popluate(struct graph *_graph) +{ + struct rte_graph *graph = _graph->graph; + + graph->tail = 0; + graph->head = (int32_t)-_graph->src_node_count; + graph->cir_mask = _graph->cir_mask; + graph->nb_nodes = _graph->node_count; + graph->cir_start = RTE_PTR_ADD(graph, _graph->cir_start); + graph->nodes_start = _graph->nodes_start; + graph->socket = _graph->socket; + graph->id = _graph->id; + memcpy(graph->name, _graph->name, RTE_GRAPH_NAMESIZE); + graph->fence = RTE_GRAPH_FENCE; +} + +static void +graph_nodes_populate(struct graph *_graph) +{ + rte_graph_off_t off = _graph->nodes_start; + struct rte_graph *graph = _graph->graph; + struct graph_node *graph_node; + rte_edge_t count, nb_edges; + const char *parent; + rte_node_t pid; + + STAILQ_FOREACH(graph_node, &_graph->node_list, next) { + struct rte_node *node = RTE_PTR_ADD(graph, off); + memset(node, 0, sizeof(*node)); + node->fence = RTE_GRAPH_FENCE; + node->off = off; + node->process = graph_node->node->process; + memcpy(node->name, graph_node->node->name, RTE_GRAPH_NAMESIZE); + pid = graph_node->node->parent_id; + if (pid != RTE_NODE_ID_INVALID) { /* Cloned node */ + parent = rte_node_id_to_name(pid); + memcpy(node->parent, parent, RTE_GRAPH_NAMESIZE); + } + node->id = graph_node->node->id; + node->parent_id = pid; + nb_edges = graph_node->node->nb_edges; + node->nb_edges = nb_edges; + off += sizeof(struct rte_node); + /* Copy the name in first pass to replace with rte_node* later*/ + for (count = 0; count < nb_edges; count++) + node->nodes[count] = (struct rte_node *)&graph_node + ->adjacency_list[count] + ->node->name[0]; + + off += sizeof(struct rte_node *) * nb_edges; + off = RTE_ALIGN(off, RTE_CACHE_LINE_SIZE); + node->next = off; + __rte_node_stream_alloc(graph, node); + } +} + +struct rte_node * +graph_node_id_to_ptr(const struct rte_graph *graph, rte_node_t id) +{ + rte_node_t count; + rte_graph_off_t off; + struct rte_node *node; + + rte_graph_foreach_node(count, off, graph, node) + if (unlikely(node->id == id)) + return node; + + return NULL; +} + +struct rte_node * +graph_node_name_to_ptr(const struct rte_graph *graph, const char *name) +{ + rte_node_t count; + rte_graph_off_t off; + struct rte_node *node; + + rte_graph_foreach_node(count, off, graph, node) + if (strncmp(name, node->name, RTE_NODE_NAMESIZE) == 0) + return node; + + return NULL; +} + +static int +graph_node_nexts_populate(struct graph *_graph) +{ + rte_node_t count, val; + rte_graph_off_t off; + struct rte_node *node; + const struct rte_graph *graph = _graph->graph; + const char *name; + + rte_graph_foreach_node(count, off, graph, node) { + for (val = 0; val < node->nb_edges; val++) { + name = (const char *)node->nodes[val]; + node->nodes[val] = graph_node_name_to_ptr(graph, name); + if (node->nodes[val] == NULL) + SET_ERR_JMP(EINVAL, fail, "%s not found", name); + } + } + + return 0; +fail: + return -rte_errno; +} + +static int +graph_src_nodes_populate(struct graph *_graph) +{ + struct rte_graph *graph = _graph->graph; + struct graph_node *graph_node; + struct rte_node *node; + int32_t head = -1; + const char *name; + + STAILQ_FOREACH(graph_node, &_graph->node_list, next) { + if (graph_node->node->flags & RTE_NODE_SOURCE_F) { + name = graph_node->node->name; + node = graph_node_name_to_ptr(graph, name); + if (node == NULL) + SET_ERR_JMP(EINVAL, fail, "%s not found", name); + + __rte_node_stream_alloc(graph, node); + graph->cir_start[head--] = node->off; + } + } + + return 0; +fail: + return -rte_errno; +} + +static int +graph_fp_mem_populate(struct graph *graph) +{ + int rc; + + graph_header_popluate(graph); + graph_nodes_populate(graph); + rc = graph_node_nexts_populate(graph); + rc |= graph_src_nodes_populate(graph); + + return rc; +} + +int +graph_fp_mem_create(struct graph *graph) +{ + const struct rte_memzone *mz; + size_t sz; + + sz = graph_fp_mem_calc_size(graph); + mz = rte_memzone_reserve(graph->name, sz, graph->socket, 0); + if (mz == NULL) + SET_ERR_JMP(ENOMEM, fail, "Memzone %s reserve failed", + graph->name); + + graph->graph = mz->addr; + graph->mz = mz; + + return graph_fp_mem_populate(graph); +fail: + return -rte_errno; +} + +static void +graph_nodes_mem_destroy(struct rte_graph *graph) +{ + rte_node_t count; + rte_graph_off_t off; + struct rte_node *node; + + if (graph == NULL) + return; + + rte_graph_foreach_node(count, off, graph, node) + rte_free(node->objs); +} + +int +graph_fp_mem_destroy(struct graph *graph) +{ + graph_nodes_mem_destroy(graph->graph); + return rte_memzone_free(graph->mz); +} diff --git a/lib/librte_graph/graph_private.h b/lib/librte_graph/graph_private.h index 220a35e2a9..7fce52e00f 100644 --- a/lib/librte_graph/graph_private.h +++ b/lib/librte_graph/graph_private.h @@ -12,6 +12,7 @@ #include #include "rte_graph.h" +#include "rte_graph_worker.h" extern int rte_graph_logtype; @@ -254,6 +255,69 @@ rte_node_t graph_nodes_count(struct graph *graph); */ void graph_mark_nodes_as_not_visited(struct graph *graph); +/* Fast path graph memory populate unctions */ + +/** + * @internal + * + * Create fast-path memory for the graph and nodes. + * + * @param graph + * Pointer to the internal graph object. + * + * @return + * - 0: Success. + * - -ENOMEM: Not enough for graph and nodes. + * - -EINVAL: Graph nodes not found. + */ +int graph_fp_mem_create(struct graph *graph); + +/** + * @internal + * + * Free fast-path memory used by graph and nodes. + * + * @param graph + * Pointer to the internal graph object. + * + * @return + * - 0: Success. + * - <0: Graph memzone related error. + */ +int graph_fp_mem_destroy(struct graph *graph); + +/* Lookup functions */ +/** + * @internal + * + * Get graph node object from node id. + * + * @param graph + * Pointer to rte_graph object. + * @param id + * Node Identifier. + * + * @return + * Pointer to rte_node if identifier is valid else NULL. + */ +struct rte_node *graph_node_id_to_ptr(const struct rte_graph *graph, + rte_node_t id); + +/** + * @internal + * + * Get graph node object from node name. + * + * @param graph + * Pointer to rte_graph object. + * @param node_name + * Pointer to character string holding the node name. + * + * @return + * Pointer to rte_node if identifier is valid else NULL. + */ +struct rte_node *graph_node_name_to_ptr(const struct rte_graph *graph, + const char *node_name); /** * @internal diff --git a/lib/librte_graph/meson.build b/lib/librte_graph/meson.build index af2695d86c..ef118c92eb 100644 --- a/lib/librte_graph/meson.build +++ b/lib/librte_graph/meson.build @@ -3,7 +3,7 @@ name = 'graph' -sources = files('node.c', 'graph.c', 'graph_ops.c', 'graph_debug.c') -headers = files('rte_graph.h') +sources = files('node.c', 'graph.c', 'graph_ops.c', 'graph_debug.c', 'graph_populate.c') +headers = files('rte_graph.h', 'rte_graph_worker.h') deps += ['eal'] diff --git a/lib/librte_graph/node.c b/lib/librte_graph/node.c index 0652d40c24..873c9ab16d 100644 --- a/lib/librte_graph/node.c +++ b/lib/librte_graph/node.c @@ -61,6 +61,11 @@ __rte_node_register(const struct rte_node_register *reg) rte_edge_t i; size_t sz; + /* Limit Node specific metadata to one cacheline on 64B CL machine */ + RTE_BUILD_BUG_ON((offsetof(struct rte_node, nodes) - + offsetof(struct rte_node, ctx)) != + RTE_CACHE_LINE_MIN_SIZE); + graph_spinlock_lock(); /* Check sanity */ diff --git a/lib/librte_graph/rte_graph_version.map b/lib/librte_graph/rte_graph_version.map index f2c2139c5a..a9fe1b6108 100644 --- a/lib/librte_graph/rte_graph_version.map +++ b/lib/librte_graph/rte_graph_version.map @@ -2,6 +2,7 @@ EXPERIMENTAL { global: __rte_node_register; + __rte_node_stream_alloc; rte_node_clone; rte_node_dump; diff --git a/lib/librte_graph/rte_graph_worker.h b/lib/librte_graph/rte_graph_worker.h new file mode 100644 index 0000000000..42be8fd13f --- /dev/null +++ b/lib/librte_graph/rte_graph_worker.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2020 Marvell International Ltd. + */ + +#ifndef _RTE_GRAPH_WORKER_H_ +#define _RTE_GRAPH_WORKER_H_ + +/** + * @file rte_graph_worker.h + * + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * This API allows a worker thread to walk over a graph and nodes to create, + * process, enqueue and move streams of objects to the next nodes. + */ + +#include +#include +#include +#include +#include + +#include "rte_graph.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @internal + * + * Data structure to hold graph data. + */ +struct rte_graph { + uint32_t tail; /**< Tail of circular buffer. */ + uint32_t head; /**< Head of circular buffer. */ + uint32_t cir_mask; /**< Circular buffer wrap around mask. */ + rte_node_t nb_nodes; /**< Number of nodes in the graph. */ + rte_graph_off_t *cir_start; /**< Pointer to circular buffer. */ + rte_graph_off_t nodes_start; /**< Offset at which node memory starts. */ + rte_graph_t id; /**< Graph identifier. */ + int socket; /**< Socket ID where memory is allocated. */ + char name[RTE_GRAPH_NAMESIZE]; /**< Name of the graph. */ + uint64_t fence; /**< Fence. */ +} __rte_cache_aligned; + +/** + * @internal + * + * Data structure to hold node data. + */ +struct rte_node { + /* Slow path area */ + uint64_t fence; /**< Fence. */ + rte_graph_off_t next; /**< Index to next node. */ + rte_node_t id; /**< Node identifier. */ + rte_node_t parent_id; /**< Parent Node identifier. */ + rte_edge_t nb_edges; /**< Number of edges from this node. */ + uint32_t realloc_count; /**< Number of times realloced. */ + + char parent[RTE_NODE_NAMESIZE]; /**< Parent node name. */ + char name[RTE_NODE_NAMESIZE]; /**< Name of the node. */ + + /* Fast path area */ +#define RTE_NODE_CTX_SZ 16 + uint8_t ctx[RTE_NODE_CTX_SZ] __rte_cache_aligned; /**< Node Context. */ + uint16_t size; /**< Total number of objects available. */ + uint16_t idx; /**< Number of objects used. */ + rte_graph_off_t off; /**< Offset of node in the graph reel. */ + uint64_t total_cycles; /**< Cycles spent in this node. */ + uint64_t total_calls; /**< Calls done to this node. */ + uint64_t total_objs; /**< Objects processed by this node. */ + RTE_STD_C11 + union { + void **objs; /**< Array of object pointers. */ + uint64_t objs_u64; + }; + RTE_STD_C11 + union { + rte_node_process_t process; /**< Process function. */ + uint64_t process_u64; + }; + struct rte_node *nodes[] __rte_cache_min_aligned; /**< Next nodes. */ +} __rte_cache_aligned; + +/** + * @internal + * + * Allocate a stream of objects. + * + * If stream already exists then re-allocate it to a larger size. + * + * @param graph + * Pointer to the graph object. + * @param node + * Pointer to the node object. + */ +__rte_experimental +void __rte_node_stream_alloc(struct rte_graph *graph, struct rte_node *node); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_GRAPH_WORKER_H_ */