4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <rte_common.h>
38 #include <rte_memory.h>
39 #include <rte_memzone.h>
40 #include <rte_cycles.h>
41 #include <rte_prefetch.h>
42 #include <rte_branch_prediction.h>
44 #include <rte_malloc.h>
45 #include <rte_string_fns.h>
47 #include "rte_pipeline.h"
49 #define RTE_TABLE_INVALID UINT32_MAX
51 #ifdef RTE_PIPELINE_STATS_COLLECT
52 #define RTE_PIPELINE_STATS_ADD(counter, val) \
53 ({ (counter) += (val); })
55 #define RTE_PIPELINE_STATS_ADD_M(counter, mask) \
56 ({ (counter) += __builtin_popcountll(mask); })
58 #define RTE_PIPELINE_STATS_ADD(counter, val)
59 #define RTE_PIPELINE_STATS_ADD_M(counter, mask)
63 /* Input parameters */
64 struct rte_port_in_ops ops;
65 rte_pipeline_port_in_action_handler f_action;
69 /* The table to which this port is connected */
72 /* Handle to low-level port */
75 /* List of enabled ports */
76 struct rte_port_in *next;
78 uint64_t n_pkts_dropped_by_ah;
82 /* Input parameters */
83 struct rte_port_out_ops ops;
84 rte_pipeline_port_out_action_handler f_action;
85 rte_pipeline_port_out_action_handler_bulk f_action_bulk;
88 /* Handle to low-level port */
91 uint64_t n_pkts_dropped_by_ah;
95 /* Input parameters */
96 struct rte_table_ops ops;
97 rte_pipeline_table_action_handler_hit f_action_hit;
98 rte_pipeline_table_action_handler_miss f_action_miss;
100 struct rte_pipeline_table_entry *default_entry;
103 uint32_t table_next_id;
104 uint32_t table_next_id_valid;
106 /* Handle to the low-level table object */
109 /* Stats for this table. */
110 uint64_t n_pkts_dropped_by_lkp_hit_ah;
111 uint64_t n_pkts_dropped_by_lkp_miss_ah;
112 uint64_t n_pkts_dropped_lkp_hit;
113 uint64_t n_pkts_dropped_lkp_miss;
116 #define RTE_PIPELINE_MAX_NAME_SZ 124
118 struct rte_pipeline {
119 /* Input parameters */
120 char name[RTE_PIPELINE_MAX_NAME_SZ];
122 uint32_t offset_port_id;
124 /* Internal tables */
125 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
126 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
127 struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
129 /* Occupancy of internal tables */
130 uint32_t num_ports_in;
131 uint32_t num_ports_out;
134 /* List of enabled ports */
135 uint64_t enabled_port_in_mask;
136 struct rte_port_in *port_in_first;
138 /* Pipeline run structures */
139 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
140 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
141 uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
142 uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
143 } __rte_cache_aligned;
145 static inline uint32_t
146 rte_mask_get_next(uint64_t mask, uint32_t pos)
148 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
149 (mask >> ((pos + 1) & 0x3F));
150 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
153 static inline uint32_t
154 rte_mask_get_prev(uint64_t mask, uint32_t pos)
156 uint64_t mask_rot = (mask >> (pos & 0x3F)) |
157 (mask << ((64 - pos) & 0x3F));
158 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
162 rte_pipeline_table_free(struct rte_table *table);
165 rte_pipeline_port_in_free(struct rte_port_in *port);
168 rte_pipeline_port_out_free(struct rte_port_out *port);
175 rte_pipeline_check_params(struct rte_pipeline_params *params)
177 if (params == NULL) {
178 RTE_LOG(ERR, PIPELINE,
179 "%s: Incorrect value for parameter params\n", __func__);
184 if (params->name == NULL) {
185 RTE_LOG(ERR, PIPELINE,
186 "%s: Incorrect value for parameter name\n", __func__);
191 if ((params->socket_id < 0) ||
192 (params->socket_id >= RTE_MAX_NUMA_NODES)) {
193 RTE_LOG(ERR, PIPELINE,
194 "%s: Incorrect value for parameter socket_id\n",
202 struct rte_pipeline *
203 rte_pipeline_create(struct rte_pipeline_params *params)
205 struct rte_pipeline *p;
208 /* Check input parameters */
209 status = rte_pipeline_check_params(params);
211 RTE_LOG(ERR, PIPELINE,
212 "%s: Pipeline params check failed (%d)\n",
217 /* Allocate memory for the pipeline on requested socket */
218 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
219 RTE_CACHE_LINE_SIZE, params->socket_id);
222 RTE_LOG(ERR, PIPELINE,
223 "%s: Pipeline memory allocation failed\n", __func__);
227 /* Save input parameters */
228 snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
229 p->socket_id = params->socket_id;
230 p->offset_port_id = params->offset_port_id;
232 /* Initialize pipeline internal data structure */
234 p->num_ports_out = 0;
236 p->enabled_port_in_mask = 0;
237 p->port_in_first = NULL;
243 rte_pipeline_free(struct rte_pipeline *p)
247 /* Check input parameters */
249 RTE_LOG(ERR, PIPELINE,
250 "%s: rte_pipeline parameter is NULL\n", __func__);
254 /* Free input ports */
255 for (i = 0; i < p->num_ports_in; i++) {
256 struct rte_port_in *port = &p->ports_in[i];
258 rte_pipeline_port_in_free(port);
262 for (i = 0; i < p->num_tables; i++) {
263 struct rte_table *table = &p->tables[i];
265 rte_pipeline_table_free(table);
268 /* Free output ports */
269 for (i = 0; i < p->num_ports_out; i++) {
270 struct rte_port_out *port = &p->ports_out[i];
272 rte_pipeline_port_out_free(port);
275 /* Free pipeline memory */
286 rte_table_check_params(struct rte_pipeline *p,
287 struct rte_pipeline_table_params *params,
291 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
295 if (params == NULL) {
296 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
300 if (table_id == NULL) {
301 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
307 if (params->ops == NULL) {
308 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
313 if (params->ops->f_create == NULL) {
314 RTE_LOG(ERR, PIPELINE,
315 "%s: f_create function pointer is NULL\n", __func__);
319 if (params->ops->f_lookup == NULL) {
320 RTE_LOG(ERR, PIPELINE,
321 "%s: f_lookup function pointer is NULL\n", __func__);
325 /* De we have room for one more table? */
326 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
327 RTE_LOG(ERR, PIPELINE,
328 "%s: Incorrect value for num_tables parameter\n",
337 rte_pipeline_table_create(struct rte_pipeline *p,
338 struct rte_pipeline_table_params *params,
341 struct rte_table *table;
342 struct rte_pipeline_table_entry *default_entry;
344 uint32_t entry_size, id;
347 /* Check input arguments */
348 status = rte_table_check_params(p, params, table_id);
353 table = &p->tables[id];
355 /* Allocate space for the default table entry */
356 entry_size = sizeof(struct rte_pipeline_table_entry) +
357 params->action_data_size;
358 default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
359 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
360 if (default_entry == NULL) {
361 RTE_LOG(ERR, PIPELINE,
362 "%s: Failed to allocate default entry\n", __func__);
366 /* Create the table */
367 h_table = params->ops->f_create(params->arg_create, p->socket_id,
369 if (h_table == NULL) {
370 rte_free(default_entry);
371 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
375 /* Commit current table to the pipeline */
379 /* Save input parameters */
380 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
381 table->f_action_hit = params->f_action_hit;
382 table->f_action_miss = params->f_action_miss;
383 table->arg_ah = params->arg_ah;
384 table->entry_size = entry_size;
386 /* Clear the lookup miss actions (to be set later through API) */
387 table->default_entry = default_entry;
388 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
390 /* Initialize table internal data structure */
391 table->h_table = h_table;
392 table->table_next_id = 0;
393 table->table_next_id_valid = 0;
399 rte_pipeline_table_free(struct rte_table *table)
401 if (table->ops.f_free != NULL)
402 table->ops.f_free(table->h_table);
404 rte_free(table->default_entry);
408 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
410 struct rte_pipeline_table_entry *default_entry,
411 struct rte_pipeline_table_entry **default_entry_ptr)
413 struct rte_table *table;
415 /* Check input arguments */
417 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
422 if (default_entry == NULL) {
423 RTE_LOG(ERR, PIPELINE,
424 "%s: default_entry parameter is NULL\n", __func__);
428 if (table_id >= p->num_tables) {
429 RTE_LOG(ERR, PIPELINE,
430 "%s: table_id %d out of range\n", __func__, table_id);
434 table = &p->tables[table_id];
436 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
437 table->table_next_id_valid &&
438 (default_entry->table_id != table->table_next_id)) {
439 RTE_LOG(ERR, PIPELINE,
440 "%s: Tree-like topologies not allowed\n", __func__);
444 /* Set the lookup miss actions */
445 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
446 (table->table_next_id_valid == 0)) {
447 table->table_next_id = default_entry->table_id;
448 table->table_next_id_valid = 1;
451 memcpy(table->default_entry, default_entry, table->entry_size);
453 *default_entry_ptr = table->default_entry;
458 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
460 struct rte_pipeline_table_entry *entry)
462 struct rte_table *table;
464 /* Check input arguments */
466 RTE_LOG(ERR, PIPELINE,
467 "%s: pipeline parameter is NULL\n", __func__);
471 if (table_id >= p->num_tables) {
472 RTE_LOG(ERR, PIPELINE,
473 "%s: table_id %d out of range\n", __func__, table_id);
477 table = &p->tables[table_id];
479 /* Save the current contents of the default entry */
481 memcpy(entry, table->default_entry, table->entry_size);
483 /* Clear the lookup miss actions */
484 memset(table->default_entry, 0, table->entry_size);
485 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
491 rte_pipeline_table_entry_add(struct rte_pipeline *p,
494 struct rte_pipeline_table_entry *entry,
496 struct rte_pipeline_table_entry **entry_ptr)
498 struct rte_table *table;
500 /* Check input arguments */
502 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
508 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
513 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
518 if (table_id >= p->num_tables) {
519 RTE_LOG(ERR, PIPELINE,
520 "%s: table_id %d out of range\n", __func__, table_id);
524 table = &p->tables[table_id];
526 if (table->ops.f_add == NULL) {
527 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
532 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
533 table->table_next_id_valid &&
534 (entry->table_id != table->table_next_id)) {
535 RTE_LOG(ERR, PIPELINE,
536 "%s: Tree-like topologies not allowed\n", __func__);
541 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
542 (table->table_next_id_valid == 0)) {
543 table->table_next_id = entry->table_id;
544 table->table_next_id_valid = 1;
547 return (table->ops.f_add)(table->h_table, key, (void *) entry,
548 key_found, (void **) entry_ptr);
552 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
556 struct rte_pipeline_table_entry *entry)
558 struct rte_table *table;
560 /* Check input arguments */
562 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
568 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
573 if (table_id >= p->num_tables) {
574 RTE_LOG(ERR, PIPELINE,
575 "%s: table_id %d out of range\n", __func__, table_id);
579 table = &p->tables[table_id];
581 if (table->ops.f_delete == NULL) {
582 RTE_LOG(ERR, PIPELINE,
583 "%s: f_delete function pointer NULL\n", __func__);
587 return (table->ops.f_delete)(table->h_table, key, key_found, entry);
590 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
593 struct rte_pipeline_table_entry **entries,
596 struct rte_pipeline_table_entry **entries_ptr)
598 struct rte_table *table;
601 /* Check input arguments */
603 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
609 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
613 if (entries == NULL) {
614 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
619 if (table_id >= p->num_tables) {
620 RTE_LOG(ERR, PIPELINE,
621 "%s: table_id %d out of range\n", __func__, table_id);
625 table = &p->tables[table_id];
627 if (table->ops.f_add_bulk == NULL) {
628 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
633 for (i = 0; i < n_keys; i++) {
634 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
635 table->table_next_id_valid &&
636 (entries[i]->table_id != table->table_next_id)) {
637 RTE_LOG(ERR, PIPELINE,
638 "%s: Tree-like topologies not allowed\n", __func__);
644 for (i = 0; i < n_keys; i++) {
645 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
646 (table->table_next_id_valid == 0)) {
647 table->table_next_id = entries[i]->table_id;
648 table->table_next_id_valid = 1;
652 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
653 n_keys, key_found, (void **) entries_ptr);
656 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
661 struct rte_pipeline_table_entry **entries)
663 struct rte_table *table;
665 /* Check input arguments */
667 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
673 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
678 if (table_id >= p->num_tables) {
679 RTE_LOG(ERR, PIPELINE,
680 "%s: table_id %d out of range\n", __func__, table_id);
684 table = &p->tables[table_id];
686 if (table->ops.f_delete_bulk == NULL) {
687 RTE_LOG(ERR, PIPELINE,
688 "%s: f_delete function pointer NULL\n", __func__);
692 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
701 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
702 struct rte_pipeline_port_in_params *params,
706 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
710 if (params == NULL) {
711 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
714 if (port_id == NULL) {
715 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
721 if (params->ops == NULL) {
722 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
727 if (params->ops->f_create == NULL) {
728 RTE_LOG(ERR, PIPELINE,
729 "%s: f_create function pointer NULL\n", __func__);
733 if (params->ops->f_rx == NULL) {
734 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
740 if ((params->burst_size == 0) ||
741 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
742 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
747 /* Do we have room for one more port? */
748 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
749 RTE_LOG(ERR, PIPELINE,
750 "%s: invalid value for num_ports_in\n", __func__);
758 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
759 struct rte_pipeline_port_out_params *params,
762 rte_pipeline_port_out_action_handler f_ah;
763 rte_pipeline_port_out_action_handler_bulk f_ah_bulk;
766 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
771 if (params == NULL) {
772 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
776 if (port_id == NULL) {
777 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
783 if (params->ops == NULL) {
784 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
789 if (params->ops->f_create == NULL) {
790 RTE_LOG(ERR, PIPELINE,
791 "%s: f_create function pointer NULL\n", __func__);
795 if (params->ops->f_tx == NULL) {
796 RTE_LOG(ERR, PIPELINE,
797 "%s: f_tx function pointer NULL\n", __func__);
801 if (params->ops->f_tx_bulk == NULL) {
802 RTE_LOG(ERR, PIPELINE,
803 "%s: f_tx_bulk function pointer NULL\n", __func__);
807 f_ah = params->f_action;
808 f_ah_bulk = params->f_action_bulk;
809 if (((f_ah != NULL) && (f_ah_bulk == NULL)) ||
810 ((f_ah == NULL) && (f_ah_bulk != NULL))) {
811 RTE_LOG(ERR, PIPELINE, "%s: Action handlers have to be either"
812 "both enabled or both disabled\n", __func__);
816 /* Do we have room for one more port? */
817 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
818 RTE_LOG(ERR, PIPELINE,
819 "%s: invalid value for num_ports_out\n", __func__);
827 rte_pipeline_port_in_create(struct rte_pipeline *p,
828 struct rte_pipeline_port_in_params *params,
831 struct rte_port_in *port;
836 /* Check input arguments */
837 status = rte_pipeline_port_in_check_params(p, params, port_id);
841 id = p->num_ports_in;
842 port = &p->ports_in[id];
844 /* Create the port */
845 h_port = params->ops->f_create(params->arg_create, p->socket_id);
846 if (h_port == NULL) {
847 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
851 /* Commit current table to the pipeline */
855 /* Save input parameters */
856 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
857 port->f_action = params->f_action;
858 port->arg_ah = params->arg_ah;
859 port->burst_size = params->burst_size;
861 /* Initialize port internal data structure */
862 port->table_id = RTE_TABLE_INVALID;
863 port->h_port = h_port;
870 rte_pipeline_port_in_free(struct rte_port_in *port)
872 if (port->ops.f_free != NULL)
873 port->ops.f_free(port->h_port);
877 rte_pipeline_port_out_create(struct rte_pipeline *p,
878 struct rte_pipeline_port_out_params *params,
881 struct rte_port_out *port;
886 /* Check input arguments */
887 status = rte_pipeline_port_out_check_params(p, params, port_id);
891 id = p->num_ports_out;
892 port = &p->ports_out[id];
894 /* Create the port */
895 h_port = params->ops->f_create(params->arg_create, p->socket_id);
896 if (h_port == NULL) {
897 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
901 /* Commit current table to the pipeline */
905 /* Save input parameters */
906 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
907 port->f_action = params->f_action;
908 port->f_action_bulk = params->f_action_bulk;
909 port->arg_ah = params->arg_ah;
911 /* Initialize port internal data structure */
912 port->h_port = h_port;
918 rte_pipeline_port_out_free(struct rte_port_out *port)
920 if (port->ops.f_free != NULL)
921 port->ops.f_free(port->h_port);
925 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
929 struct rte_port_in *port;
931 /* Check input arguments */
933 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
938 if (port_id >= p->num_ports_in) {
939 RTE_LOG(ERR, PIPELINE,
940 "%s: port IN ID %u is out of range\n",
945 if (table_id >= p->num_tables) {
946 RTE_LOG(ERR, PIPELINE,
947 "%s: Table ID %u is out of range\n",
952 port = &p->ports_in[port_id];
953 port->table_id = table_id;
959 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
961 struct rte_port_in *port, *port_prev, *port_next;
962 struct rte_port_in *port_first, *port_last;
964 uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
966 /* Check input arguments */
968 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
973 if (port_id >= p->num_ports_in) {
974 RTE_LOG(ERR, PIPELINE,
975 "%s: port IN ID %u is out of range\n",
980 /* Return if current input port is already enabled */
981 port_mask = 1LLU << port_id;
982 if (p->enabled_port_in_mask & port_mask)
985 p->enabled_port_in_mask |= port_mask;
987 /* Add current input port to the pipeline chain of enabled ports */
988 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
989 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
991 port_prev = &p->ports_in[port_prev_id];
992 port_next = &p->ports_in[port_next_id];
993 port = &p->ports_in[port_id];
995 port_prev->next = port;
996 port->next = port_next;
998 /* Update the first and last input ports in the chain */
999 port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
1000 port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
1002 port_first = &p->ports_in[port_first_id];
1003 port_last = &p->ports_in[port_last_id];
1005 p->port_in_first = port_first;
1006 port_last->next = NULL;
1012 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
1014 struct rte_port_in *port_prev, *port_next, *port_first, *port_last;
1016 uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
1018 /* Check input arguments */
1020 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1025 if (port_id >= p->num_ports_in) {
1026 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
1031 /* Return if current input port is already disabled */
1032 port_mask = 1LLU << port_id;
1033 if ((p->enabled_port_in_mask & port_mask) == 0)
1036 /* Return if no other enabled ports */
1037 if (__builtin_popcountll(p->enabled_port_in_mask) == 1) {
1038 p->enabled_port_in_mask &= ~port_mask;
1039 p->port_in_first = NULL;
1044 /* Add current input port to the pipeline chain of enabled ports */
1045 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1046 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1048 port_prev = &p->ports_in[port_prev_id];
1049 port_next = &p->ports_in[port_next_id];
1051 port_prev->next = port_next;
1052 p->enabled_port_in_mask &= ~port_mask;
1054 /* Update the first and last input ports in the chain */
1055 port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
1056 port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
1058 port_first = &p->ports_in[port_first_id];
1059 port_last = &p->ports_in[port_last_id];
1061 p->port_in_first = port_first;
1062 port_last->next = NULL;
1072 rte_pipeline_check(struct rte_pipeline *p)
1074 uint32_t port_in_id;
1076 /* Check input arguments */
1078 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1083 /* Check that pipeline has at least one input port, one table and one
1085 if (p->num_ports_in == 0) {
1086 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1090 if (p->num_tables == 0) {
1091 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1095 if (p->num_ports_out == 0) {
1096 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1101 /* Check that all input ports are connected */
1102 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1103 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1105 if (port_in->table_id == RTE_TABLE_INVALID) {
1106 RTE_LOG(ERR, PIPELINE,
1107 "%s: Port IN ID %u is not connected\n",
1108 __func__, port_in_id);
1117 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1119 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1120 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1121 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1122 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1124 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1125 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1128 for (i = 0; i < n_pkts; i++) {
1129 uint64_t pkt_mask = 1LLU << i;
1130 uint32_t pos = p->entries[i]->action;
1132 p->action_mask1[pos] |= pkt_mask;
1137 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1138 uint64_t pkt_mask = 1LLU << i;
1141 if ((pkt_mask & pkts_mask) == 0)
1144 pos = p->entries[i]->action;
1145 p->action_mask1[pos] |= pkt_mask;
1151 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1152 uint64_t pkts_mask, uint32_t port_id)
1154 struct rte_port_out *port_out = &p->ports_out[port_id];
1156 /* Output port user actions */
1157 if (port_out->f_action_bulk != NULL) {
1158 uint64_t mask = pkts_mask;
1160 port_out->f_action_bulk(p->pkts, &pkts_mask, port_out->arg_ah);
1161 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask ^ mask;
1162 RTE_PIPELINE_STATS_ADD_M(port_out->n_pkts_dropped_by_ah,
1166 /* Output port TX */
1168 port_out->ops.f_tx_bulk(port_out->h_port, p->pkts, pkts_mask);
1172 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1174 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1175 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1178 for (i = 0; i < n_pkts; i++) {
1179 struct rte_mbuf *pkt = p->pkts[i];
1180 uint32_t port_out_id = p->entries[i]->port_id;
1181 struct rte_port_out *port_out =
1182 &p->ports_out[port_out_id];
1184 /* Output port user actions */
1185 if (port_out->f_action == NULL) /* Output port TX */
1186 port_out->ops.f_tx(port_out->h_port, pkt);
1188 uint64_t pkt_mask = 1LLU;
1190 port_out->f_action(pkt, &pkt_mask,
1192 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1193 (pkt_mask ^ 1LLU) << i;
1195 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1198 /* Output port TX */
1200 port_out->ops.f_tx(port_out->h_port,
1207 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1208 uint64_t pkt_mask = 1LLU << i;
1209 struct rte_mbuf *pkt;
1210 struct rte_port_out *port_out;
1211 uint32_t port_out_id;
1213 if ((pkt_mask & pkts_mask) == 0)
1217 port_out_id = p->entries[i]->port_id;
1218 port_out = &p->ports_out[port_out_id];
1220 /* Output port user actions */
1221 if (port_out->f_action == NULL) /* Output port TX */
1222 port_out->ops.f_tx(port_out->h_port, pkt);
1226 port_out->f_action(pkt, &pkt_mask,
1228 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1229 (pkt_mask ^ 1LLU) << i;
1231 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1234 /* Output port TX */
1236 port_out->ops.f_tx(port_out->h_port,
1244 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1247 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1248 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1251 for (i = 0; i < n_pkts; i++) {
1252 struct rte_mbuf *pkt = p->pkts[i];
1253 uint32_t port_out_id =
1254 RTE_MBUF_METADATA_UINT32(pkt,
1256 struct rte_port_out *port_out = &p->ports_out[
1259 /* Output port user actions */
1260 if (port_out->f_action == NULL) /* Output port TX */
1261 port_out->ops.f_tx(port_out->h_port, pkt);
1263 uint64_t pkt_mask = 1LLU;
1265 port_out->f_action(pkt, &pkt_mask,
1267 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1268 (pkt_mask ^ 1LLU) << i;
1270 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1273 /* Output port TX */
1275 port_out->ops.f_tx(port_out->h_port,
1282 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1283 uint64_t pkt_mask = 1LLU << i;
1284 struct rte_mbuf *pkt;
1285 struct rte_port_out *port_out;
1286 uint32_t port_out_id;
1288 if ((pkt_mask & pkts_mask) == 0)
1292 port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1294 port_out = &p->ports_out[port_out_id];
1296 /* Output port user actions */
1297 if (port_out->f_action == NULL) /* Output port TX */
1298 port_out->ops.f_tx(port_out->h_port, pkt);
1302 port_out->f_action(pkt, &pkt_mask,
1304 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1305 (pkt_mask ^ 1LLU) << i;
1307 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1310 /* Output port TX */
1312 port_out->ops.f_tx(port_out->h_port,
1320 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1322 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1323 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1326 for (i = 0; i < n_pkts; i++)
1327 rte_pktmbuf_free(p->pkts[i]);
1331 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1332 uint64_t pkt_mask = 1LLU << i;
1334 if ((pkt_mask & pkts_mask) == 0)
1337 rte_pktmbuf_free(p->pkts[i]);
1343 rte_pipeline_run(struct rte_pipeline *p)
1345 struct rte_port_in *port_in;
1347 for (port_in = p->port_in_first; port_in != NULL;
1348 port_in = port_in->next) {
1350 uint32_t n_pkts, table_id;
1353 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1354 port_in->burst_size);
1358 pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1359 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1360 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1361 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1362 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1364 /* Input port user actions */
1365 if (port_in->f_action != NULL) {
1366 uint64_t mask = pkts_mask;
1368 port_in->f_action(p->pkts, n_pkts, &pkts_mask, port_in->arg_ah);
1370 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1371 RTE_PIPELINE_STATS_ADD_M(port_in->n_pkts_dropped_by_ah, mask);
1375 for (table_id = port_in->table_id; pkts_mask != 0; ) {
1376 struct rte_table *table;
1377 uint64_t lookup_hit_mask, lookup_miss_mask;
1380 table = &p->tables[table_id];
1381 table->ops.f_lookup(table->h_table, p->pkts, pkts_mask,
1382 &lookup_hit_mask, (void **) p->entries);
1383 lookup_miss_mask = pkts_mask & (~lookup_hit_mask);
1386 if (lookup_miss_mask != 0) {
1387 struct rte_pipeline_table_entry *default_entry =
1388 table->default_entry;
1390 /* Table user actions */
1391 if (table->f_action_miss != NULL) {
1392 uint64_t mask = lookup_miss_mask;
1394 table->f_action_miss(p->pkts,
1396 default_entry, table->arg_ah);
1397 mask ^= lookup_miss_mask;
1398 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1399 RTE_PIPELINE_STATS_ADD_M(
1400 table->n_pkts_dropped_by_lkp_miss_ah, mask);
1403 /* Table reserved actions */
1404 if ((default_entry->action ==
1405 RTE_PIPELINE_ACTION_PORT) &&
1406 (lookup_miss_mask != 0))
1407 rte_pipeline_action_handler_port_bulk(p,
1409 default_entry->port_id);
1411 uint32_t pos = default_entry->action;
1413 p->action_mask0[pos] = lookup_miss_mask;
1414 if (pos == RTE_PIPELINE_ACTION_DROP) {
1415 RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_miss,
1422 if (lookup_hit_mask != 0) {
1423 /* Table user actions */
1424 if (table->f_action_hit != NULL) {
1425 uint64_t mask = lookup_hit_mask;
1427 table->f_action_hit(p->pkts,
1429 p->entries, table->arg_ah);
1430 mask ^= lookup_hit_mask;
1431 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1432 RTE_PIPELINE_STATS_ADD_M(
1433 table->n_pkts_dropped_by_lkp_hit_ah, mask);
1436 /* Table reserved actions */
1437 rte_pipeline_compute_masks(p, lookup_hit_mask);
1438 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1440 RTE_PIPELINE_ACTION_DROP];
1441 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1443 RTE_PIPELINE_ACTION_PORT];
1444 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1446 RTE_PIPELINE_ACTION_PORT_META];
1447 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1449 RTE_PIPELINE_ACTION_TABLE];
1451 RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_hit,
1452 p->action_mask1[RTE_PIPELINE_ACTION_DROP]);
1455 /* Prepare for next iteration */
1456 pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1457 table_id = table->table_next_id;
1458 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1461 /* Table reserved action PORT */
1462 rte_pipeline_action_handler_port(p,
1463 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1465 /* Table reserved action PORT META */
1466 rte_pipeline_action_handler_port_meta(p,
1467 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1469 /* Table reserved action DROP */
1470 rte_pipeline_action_handler_drop(p,
1471 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1478 rte_pipeline_flush(struct rte_pipeline *p)
1482 /* Check input arguments */
1484 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1489 for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1490 struct rte_port_out *port = &p->ports_out[port_id];
1492 if (port->ops.f_flush != NULL)
1493 port->ops.f_flush(port->h_port);
1500 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1501 uint32_t port_id, struct rte_mbuf *pkt)
1503 struct rte_port_out *port_out = &p->ports_out[port_id];
1505 /* Output port user actions */
1506 if (port_out->f_action == NULL)
1507 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1509 uint64_t pkt_mask = 1LLU;
1511 port_out->f_action(pkt, &pkt_mask, port_out->arg_ah);
1513 if (pkt_mask != 0) /* Output port TX */
1514 port_out->ops.f_tx(port_out->h_port, pkt);
1516 rte_pktmbuf_free(pkt);
1517 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, 1);
1524 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1525 struct rte_pipeline_port_in_stats *stats, int clear)
1527 struct rte_port_in *port;
1531 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1536 if (port_id >= p->num_ports_in) {
1537 RTE_LOG(ERR, PIPELINE,
1538 "%s: port IN ID %u is out of range\n",
1543 port = &p->ports_in[port_id];
1545 if (port->ops.f_stats != NULL) {
1546 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1549 } else if (stats != NULL)
1550 memset(&stats->stats, 0, sizeof(stats->stats));
1553 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1556 port->n_pkts_dropped_by_ah = 0;
1561 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1562 struct rte_pipeline_port_out_stats *stats, int clear)
1564 struct rte_port_out *port;
1568 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1572 if (port_id >= p->num_ports_out) {
1573 RTE_LOG(ERR, PIPELINE,
1574 "%s: port OUT ID %u is out of range\n", __func__, port_id);
1578 port = &p->ports_out[port_id];
1579 if (port->ops.f_stats != NULL) {
1580 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1583 } else if (stats != NULL)
1584 memset(&stats->stats, 0, sizeof(stats->stats));
1587 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1590 port->n_pkts_dropped_by_ah = 0;
1595 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1596 struct rte_pipeline_table_stats *stats, int clear)
1598 struct rte_table *table;
1602 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1607 if (table_id >= p->num_tables) {
1608 RTE_LOG(ERR, PIPELINE,
1609 "%s: table %u is out of range\n", __func__, table_id);
1613 table = &p->tables[table_id];
1614 if (table->ops.f_stats != NULL) {
1615 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1618 } else if (stats != NULL)
1619 memset(&stats->stats, 0, sizeof(stats->stats));
1621 if (stats != NULL) {
1622 stats->n_pkts_dropped_by_lkp_hit_ah =
1623 table->n_pkts_dropped_by_lkp_hit_ah;
1624 stats->n_pkts_dropped_by_lkp_miss_ah =
1625 table->n_pkts_dropped_by_lkp_miss_ah;
1626 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1627 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1631 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1632 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1633 table->n_pkts_dropped_lkp_hit = 0;
1634 table->n_pkts_dropped_lkp_miss = 0;