4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <rte_common.h>
38 #include <rte_memory.h>
39 #include <rte_memzone.h>
40 #include <rte_cycles.h>
41 #include <rte_prefetch.h>
42 #include <rte_branch_prediction.h>
44 #include <rte_malloc.h>
45 #include <rte_string_fns.h>
47 #include "rte_pipeline.h"
49 #define RTE_TABLE_INVALID UINT32_MAX
51 #ifdef RTE_PIPELINE_STATS_COLLECT
52 #define RTE_PIPELINE_STATS_ADD(counter, val) \
53 ({ (counter) += (val); })
55 #define RTE_PIPELINE_STATS_ADD_M(counter, mask) \
56 ({ (counter) += __builtin_popcountll(mask); })
58 #define RTE_PIPELINE_STATS_ADD(counter, val)
59 #define RTE_PIPELINE_STATS_ADD_M(counter, mask)
63 /* Input parameters */
64 struct rte_port_in_ops ops;
65 rte_pipeline_port_in_action_handler f_action;
69 /* The table to which this port is connected */
72 /* Handle to low-level port */
75 /* List of enabled ports */
76 struct rte_port_in *next;
78 uint64_t n_pkts_dropped_by_ah;
82 /* Input parameters */
83 struct rte_port_out_ops ops;
84 rte_pipeline_port_out_action_handler f_action;
85 rte_pipeline_port_out_action_handler_bulk f_action_bulk;
88 /* Handle to low-level port */
91 uint64_t n_pkts_dropped_by_ah;
95 /* Input parameters */
96 struct rte_table_ops ops;
97 rte_pipeline_table_action_handler_hit f_action_hit;
98 rte_pipeline_table_action_handler_miss f_action_miss;
100 struct rte_pipeline_table_entry *default_entry;
103 uint32_t table_next_id;
104 uint32_t table_next_id_valid;
106 /* Handle to the low-level table object */
109 /* Stats for this table. */
110 uint64_t n_pkts_dropped_by_lkp_hit_ah;
111 uint64_t n_pkts_dropped_by_lkp_miss_ah;
112 uint64_t n_pkts_dropped_lkp_hit;
113 uint64_t n_pkts_dropped_lkp_miss;
116 #define RTE_PIPELINE_MAX_NAME_SZ 124
118 struct rte_pipeline {
119 /* Input parameters */
120 char name[RTE_PIPELINE_MAX_NAME_SZ];
122 uint32_t offset_port_id;
124 /* Internal tables */
125 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
126 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
127 struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
129 /* Occupancy of internal tables */
130 uint32_t num_ports_in;
131 uint32_t num_ports_out;
134 /* List of enabled ports */
135 uint64_t enabled_port_in_mask;
136 struct rte_port_in *port_in_first;
138 /* Pipeline run structures */
139 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
140 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
141 uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
142 uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
143 } __rte_cache_aligned;
145 static inline uint32_t
146 rte_mask_get_next(uint64_t mask, uint32_t pos)
148 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
149 (mask >> ((pos + 1) & 0x3F));
150 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
153 static inline uint32_t
154 rte_mask_get_prev(uint64_t mask, uint32_t pos)
156 uint64_t mask_rot = (mask >> (pos & 0x3F)) |
157 (mask << ((64 - pos) & 0x3F));
158 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
162 rte_pipeline_table_free(struct rte_table *table);
165 rte_pipeline_port_in_free(struct rte_port_in *port);
168 rte_pipeline_port_out_free(struct rte_port_out *port);
175 rte_pipeline_check_params(struct rte_pipeline_params *params)
177 if (params == NULL) {
178 RTE_LOG(ERR, PIPELINE,
179 "%s: Incorrect value for parameter params\n", __func__);
184 if (params->name == NULL) {
185 RTE_LOG(ERR, PIPELINE,
186 "%s: Incorrect value for parameter name\n", __func__);
191 if ((params->socket_id < 0) ||
192 (params->socket_id >= RTE_MAX_NUMA_NODES)) {
193 RTE_LOG(ERR, PIPELINE,
194 "%s: Incorrect value for parameter socket_id\n",
202 struct rte_pipeline *
203 rte_pipeline_create(struct rte_pipeline_params *params)
205 struct rte_pipeline *p;
208 /* Check input parameters */
209 status = rte_pipeline_check_params(params);
211 RTE_LOG(ERR, PIPELINE,
212 "%s: Pipeline params check failed (%d)\n",
217 /* Allocate memory for the pipeline on requested socket */
218 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
219 RTE_CACHE_LINE_SIZE, params->socket_id);
222 RTE_LOG(ERR, PIPELINE,
223 "%s: Pipeline memory allocation failed\n", __func__);
227 /* Save input parameters */
228 snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
229 p->socket_id = params->socket_id;
230 p->offset_port_id = params->offset_port_id;
232 /* Initialize pipeline internal data structure */
234 p->num_ports_out = 0;
236 p->enabled_port_in_mask = 0;
237 p->port_in_first = NULL;
243 rte_pipeline_free(struct rte_pipeline *p)
247 /* Check input parameters */
249 RTE_LOG(ERR, PIPELINE,
250 "%s: rte_pipeline parameter is NULL\n", __func__);
254 /* Free input ports */
255 for (i = 0; i < p->num_ports_in; i++) {
256 struct rte_port_in *port = &p->ports_in[i];
258 rte_pipeline_port_in_free(port);
262 for (i = 0; i < p->num_tables; i++) {
263 struct rte_table *table = &p->tables[i];
265 rte_pipeline_table_free(table);
268 /* Free output ports */
269 for (i = 0; i < p->num_ports_out; i++) {
270 struct rte_port_out *port = &p->ports_out[i];
272 rte_pipeline_port_out_free(port);
275 /* Free pipeline memory */
286 rte_table_check_params(struct rte_pipeline *p,
287 struct rte_pipeline_table_params *params,
291 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
295 if (params == NULL) {
296 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
300 if (table_id == NULL) {
301 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
307 if (params->ops == NULL) {
308 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
313 if (params->ops->f_create == NULL) {
314 RTE_LOG(ERR, PIPELINE,
315 "%s: f_create function pointer is NULL\n", __func__);
319 if (params->ops->f_lookup == NULL) {
320 RTE_LOG(ERR, PIPELINE,
321 "%s: f_lookup function pointer is NULL\n", __func__);
325 /* De we have room for one more table? */
326 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
327 RTE_LOG(ERR, PIPELINE,
328 "%s: Incorrect value for num_tables parameter\n",
337 rte_pipeline_table_create(struct rte_pipeline *p,
338 struct rte_pipeline_table_params *params,
341 struct rte_table *table;
342 struct rte_pipeline_table_entry *default_entry;
344 uint32_t entry_size, id;
347 /* Check input arguments */
348 status = rte_table_check_params(p, params, table_id);
353 table = &p->tables[id];
355 /* Allocate space for the default table entry */
356 entry_size = sizeof(struct rte_pipeline_table_entry) +
357 params->action_data_size;
358 default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
359 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
360 if (default_entry == NULL) {
361 RTE_LOG(ERR, PIPELINE,
362 "%s: Failed to allocate default entry\n", __func__);
366 /* Create the table */
367 h_table = params->ops->f_create(params->arg_create, p->socket_id,
369 if (h_table == NULL) {
370 rte_free(default_entry);
371 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
375 /* Commit current table to the pipeline */
379 /* Save input parameters */
380 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
381 table->f_action_hit = params->f_action_hit;
382 table->f_action_miss = params->f_action_miss;
383 table->arg_ah = params->arg_ah;
384 table->entry_size = entry_size;
386 /* Clear the lookup miss actions (to be set later through API) */
387 table->default_entry = default_entry;
388 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
390 /* Initialize table internal data structure */
391 table->h_table = h_table;
392 table->table_next_id = 0;
393 table->table_next_id_valid = 0;
399 rte_pipeline_table_free(struct rte_table *table)
401 if (table->ops.f_free != NULL)
402 table->ops.f_free(table->h_table);
404 rte_free(table->default_entry);
408 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
410 struct rte_pipeline_table_entry *default_entry,
411 struct rte_pipeline_table_entry **default_entry_ptr)
413 struct rte_table *table;
415 /* Check input arguments */
417 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
422 if (default_entry == NULL) {
423 RTE_LOG(ERR, PIPELINE,
424 "%s: default_entry parameter is NULL\n", __func__);
428 if (table_id >= p->num_tables) {
429 RTE_LOG(ERR, PIPELINE,
430 "%s: table_id %d out of range\n", __func__, table_id);
434 table = &p->tables[table_id];
436 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
437 table->table_next_id_valid &&
438 (default_entry->table_id != table->table_next_id)) {
439 RTE_LOG(ERR, PIPELINE,
440 "%s: Tree-like topologies not allowed\n", __func__);
444 /* Set the lookup miss actions */
445 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
446 (table->table_next_id_valid == 0)) {
447 table->table_next_id = default_entry->table_id;
448 table->table_next_id_valid = 1;
451 memcpy(table->default_entry, default_entry, table->entry_size);
453 *default_entry_ptr = table->default_entry;
458 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
460 struct rte_pipeline_table_entry *entry)
462 struct rte_table *table;
464 /* Check input arguments */
466 RTE_LOG(ERR, PIPELINE,
467 "%s: pipeline parameter is NULL\n", __func__);
471 if (table_id >= p->num_tables) {
472 RTE_LOG(ERR, PIPELINE,
473 "%s: table_id %d out of range\n", __func__, table_id);
477 table = &p->tables[table_id];
479 /* Save the current contents of the default entry */
481 memcpy(entry, table->default_entry, table->entry_size);
483 /* Clear the lookup miss actions */
484 memset(table->default_entry, 0, table->entry_size);
485 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
491 rte_pipeline_table_entry_add(struct rte_pipeline *p,
494 struct rte_pipeline_table_entry *entry,
496 struct rte_pipeline_table_entry **entry_ptr)
498 struct rte_table *table;
500 /* Check input arguments */
502 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
508 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
513 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
518 if (table_id >= p->num_tables) {
519 RTE_LOG(ERR, PIPELINE,
520 "%s: table_id %d out of range\n", __func__, table_id);
524 table = &p->tables[table_id];
526 if (table->ops.f_add == NULL) {
527 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
532 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
533 table->table_next_id_valid &&
534 (entry->table_id != table->table_next_id)) {
535 RTE_LOG(ERR, PIPELINE,
536 "%s: Tree-like topologies not allowed\n", __func__);
541 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
542 (table->table_next_id_valid == 0)) {
543 table->table_next_id = entry->table_id;
544 table->table_next_id_valid = 1;
547 return (table->ops.f_add)(table->h_table, key, (void *) entry,
548 key_found, (void **) entry_ptr);
552 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
556 struct rte_pipeline_table_entry *entry)
558 struct rte_table *table;
560 /* Check input arguments */
562 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
568 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
573 if (table_id >= p->num_tables) {
574 RTE_LOG(ERR, PIPELINE,
575 "%s: table_id %d out of range\n", __func__, table_id);
579 table = &p->tables[table_id];
581 if (table->ops.f_delete == NULL) {
582 RTE_LOG(ERR, PIPELINE,
583 "%s: f_delete function pointer NULL\n", __func__);
587 return (table->ops.f_delete)(table->h_table, key, key_found, entry);
595 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
596 struct rte_pipeline_port_in_params *params,
600 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
604 if (params == NULL) {
605 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
608 if (port_id == NULL) {
609 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
615 if (params->ops == NULL) {
616 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
621 if (params->ops->f_create == NULL) {
622 RTE_LOG(ERR, PIPELINE,
623 "%s: f_create function pointer NULL\n", __func__);
627 if (params->ops->f_rx == NULL) {
628 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
634 if ((params->burst_size == 0) ||
635 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
636 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
641 /* Do we have room for one more port? */
642 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
643 RTE_LOG(ERR, PIPELINE,
644 "%s: invalid value for num_ports_in\n", __func__);
652 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
653 struct rte_pipeline_port_out_params *params,
656 rte_pipeline_port_out_action_handler f_ah;
657 rte_pipeline_port_out_action_handler_bulk f_ah_bulk;
660 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
665 if (params == NULL) {
666 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
670 if (port_id == NULL) {
671 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
677 if (params->ops == NULL) {
678 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
683 if (params->ops->f_create == NULL) {
684 RTE_LOG(ERR, PIPELINE,
685 "%s: f_create function pointer NULL\n", __func__);
689 if (params->ops->f_tx == NULL) {
690 RTE_LOG(ERR, PIPELINE,
691 "%s: f_tx function pointer NULL\n", __func__);
695 if (params->ops->f_tx_bulk == NULL) {
696 RTE_LOG(ERR, PIPELINE,
697 "%s: f_tx_bulk function pointer NULL\n", __func__);
701 f_ah = params->f_action;
702 f_ah_bulk = params->f_action_bulk;
703 if (((f_ah != NULL) && (f_ah_bulk == NULL)) ||
704 ((f_ah == NULL) && (f_ah_bulk != NULL))) {
705 RTE_LOG(ERR, PIPELINE, "%s: Action handlers have to be either"
706 "both enabled or both disabled\n", __func__);
710 /* Do we have room for one more port? */
711 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
712 RTE_LOG(ERR, PIPELINE,
713 "%s: invalid value for num_ports_out\n", __func__);
721 rte_pipeline_port_in_create(struct rte_pipeline *p,
722 struct rte_pipeline_port_in_params *params,
725 struct rte_port_in *port;
730 /* Check input arguments */
731 status = rte_pipeline_port_in_check_params(p, params, port_id);
735 id = p->num_ports_in;
736 port = &p->ports_in[id];
738 /* Create the port */
739 h_port = params->ops->f_create(params->arg_create, p->socket_id);
740 if (h_port == NULL) {
741 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
745 /* Commit current table to the pipeline */
749 /* Save input parameters */
750 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
751 port->f_action = params->f_action;
752 port->arg_ah = params->arg_ah;
753 port->burst_size = params->burst_size;
755 /* Initialize port internal data structure */
756 port->table_id = RTE_TABLE_INVALID;
757 port->h_port = h_port;
764 rte_pipeline_port_in_free(struct rte_port_in *port)
766 if (port->ops.f_free != NULL)
767 port->ops.f_free(port->h_port);
771 rte_pipeline_port_out_create(struct rte_pipeline *p,
772 struct rte_pipeline_port_out_params *params,
775 struct rte_port_out *port;
780 /* Check input arguments */
781 status = rte_pipeline_port_out_check_params(p, params, port_id);
785 id = p->num_ports_out;
786 port = &p->ports_out[id];
788 /* Create the port */
789 h_port = params->ops->f_create(params->arg_create, p->socket_id);
790 if (h_port == NULL) {
791 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
795 /* Commit current table to the pipeline */
799 /* Save input parameters */
800 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
801 port->f_action = params->f_action;
802 port->f_action_bulk = params->f_action_bulk;
803 port->arg_ah = params->arg_ah;
805 /* Initialize port internal data structure */
806 port->h_port = h_port;
812 rte_pipeline_port_out_free(struct rte_port_out *port)
814 if (port->ops.f_free != NULL)
815 port->ops.f_free(port->h_port);
819 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
823 struct rte_port_in *port;
825 /* Check input arguments */
827 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
832 if (port_id >= p->num_ports_in) {
833 RTE_LOG(ERR, PIPELINE,
834 "%s: port IN ID %u is out of range\n",
839 if (table_id >= p->num_tables) {
840 RTE_LOG(ERR, PIPELINE,
841 "%s: Table ID %u is out of range\n",
846 port = &p->ports_in[port_id];
847 port->table_id = table_id;
853 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
855 struct rte_port_in *port, *port_prev, *port_next;
856 struct rte_port_in *port_first, *port_last;
858 uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
860 /* Check input arguments */
862 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
867 if (port_id >= p->num_ports_in) {
868 RTE_LOG(ERR, PIPELINE,
869 "%s: port IN ID %u is out of range\n",
874 /* Return if current input port is already enabled */
875 port_mask = 1LLU << port_id;
876 if (p->enabled_port_in_mask & port_mask)
879 p->enabled_port_in_mask |= port_mask;
881 /* Add current input port to the pipeline chain of enabled ports */
882 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
883 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
885 port_prev = &p->ports_in[port_prev_id];
886 port_next = &p->ports_in[port_next_id];
887 port = &p->ports_in[port_id];
889 port_prev->next = port;
890 port->next = port_next;
892 /* Update the first and last input ports in the chain */
893 port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
894 port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
896 port_first = &p->ports_in[port_first_id];
897 port_last = &p->ports_in[port_last_id];
899 p->port_in_first = port_first;
900 port_last->next = NULL;
906 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
908 struct rte_port_in *port_prev, *port_next, *port_first, *port_last;
910 uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
912 /* Check input arguments */
914 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
919 if (port_id >= p->num_ports_in) {
920 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
925 /* Return if current input port is already disabled */
926 port_mask = 1LLU << port_id;
927 if ((p->enabled_port_in_mask & port_mask) == 0)
930 /* Return if no other enabled ports */
931 if (__builtin_popcountll(p->enabled_port_in_mask) == 1) {
932 p->enabled_port_in_mask &= ~port_mask;
933 p->port_in_first = NULL;
938 /* Add current input port to the pipeline chain of enabled ports */
939 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
940 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
942 port_prev = &p->ports_in[port_prev_id];
943 port_next = &p->ports_in[port_next_id];
945 port_prev->next = port_next;
946 p->enabled_port_in_mask &= ~port_mask;
948 /* Update the first and last input ports in the chain */
949 port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
950 port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
952 port_first = &p->ports_in[port_first_id];
953 port_last = &p->ports_in[port_last_id];
955 p->port_in_first = port_first;
956 port_last->next = NULL;
966 rte_pipeline_check(struct rte_pipeline *p)
970 /* Check input arguments */
972 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
977 /* Check that pipeline has at least one input port, one table and one
979 if (p->num_ports_in == 0) {
980 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
984 if (p->num_tables == 0) {
985 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
989 if (p->num_ports_out == 0) {
990 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
995 /* Check that all input ports are connected */
996 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
997 struct rte_port_in *port_in = &p->ports_in[port_in_id];
999 if (port_in->table_id == RTE_TABLE_INVALID) {
1000 RTE_LOG(ERR, PIPELINE,
1001 "%s: Port IN ID %u is not connected\n",
1002 __func__, port_in_id);
1011 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1013 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1014 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1015 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1016 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1018 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1019 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1022 for (i = 0; i < n_pkts; i++) {
1023 uint64_t pkt_mask = 1LLU << i;
1024 uint32_t pos = p->entries[i]->action;
1026 p->action_mask1[pos] |= pkt_mask;
1031 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1032 uint64_t pkt_mask = 1LLU << i;
1035 if ((pkt_mask & pkts_mask) == 0)
1038 pos = p->entries[i]->action;
1039 p->action_mask1[pos] |= pkt_mask;
1045 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1046 uint64_t pkts_mask, uint32_t port_id)
1048 struct rte_port_out *port_out = &p->ports_out[port_id];
1050 /* Output port user actions */
1051 if (port_out->f_action_bulk != NULL) {
1052 uint64_t mask = pkts_mask;
1054 port_out->f_action_bulk(p->pkts, &pkts_mask, port_out->arg_ah);
1055 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask ^ mask;
1056 RTE_PIPELINE_STATS_ADD_M(port_out->n_pkts_dropped_by_ah,
1060 /* Output port TX */
1062 port_out->ops.f_tx_bulk(port_out->h_port, p->pkts, pkts_mask);
1066 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1068 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1069 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1072 for (i = 0; i < n_pkts; i++) {
1073 struct rte_mbuf *pkt = p->pkts[i];
1074 uint32_t port_out_id = p->entries[i]->port_id;
1075 struct rte_port_out *port_out =
1076 &p->ports_out[port_out_id];
1078 /* Output port user actions */
1079 if (port_out->f_action == NULL) /* Output port TX */
1080 port_out->ops.f_tx(port_out->h_port, pkt);
1082 uint64_t pkt_mask = 1LLU;
1084 port_out->f_action(pkt, &pkt_mask,
1086 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1087 (pkt_mask ^ 1LLU) << i;
1089 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1092 /* Output port TX */
1094 port_out->ops.f_tx(port_out->h_port,
1101 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1102 uint64_t pkt_mask = 1LLU << i;
1103 struct rte_mbuf *pkt;
1104 struct rte_port_out *port_out;
1105 uint32_t port_out_id;
1107 if ((pkt_mask & pkts_mask) == 0)
1111 port_out_id = p->entries[i]->port_id;
1112 port_out = &p->ports_out[port_out_id];
1114 /* Output port user actions */
1115 if (port_out->f_action == NULL) /* Output port TX */
1116 port_out->ops.f_tx(port_out->h_port, pkt);
1120 port_out->f_action(pkt, &pkt_mask,
1122 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1123 (pkt_mask ^ 1LLU) << i;
1125 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1128 /* Output port TX */
1130 port_out->ops.f_tx(port_out->h_port,
1138 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1141 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1142 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1145 for (i = 0; i < n_pkts; i++) {
1146 struct rte_mbuf *pkt = p->pkts[i];
1147 uint32_t port_out_id =
1148 RTE_MBUF_METADATA_UINT32(pkt,
1150 struct rte_port_out *port_out = &p->ports_out[
1153 /* Output port user actions */
1154 if (port_out->f_action == NULL) /* Output port TX */
1155 port_out->ops.f_tx(port_out->h_port, pkt);
1157 uint64_t pkt_mask = 1LLU;
1159 port_out->f_action(pkt, &pkt_mask,
1161 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1162 (pkt_mask ^ 1LLU) << i;
1164 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1167 /* Output port TX */
1169 port_out->ops.f_tx(port_out->h_port,
1176 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1177 uint64_t pkt_mask = 1LLU << i;
1178 struct rte_mbuf *pkt;
1179 struct rte_port_out *port_out;
1180 uint32_t port_out_id;
1182 if ((pkt_mask & pkts_mask) == 0)
1186 port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1188 port_out = &p->ports_out[port_out_id];
1190 /* Output port user actions */
1191 if (port_out->f_action == NULL) /* Output port TX */
1192 port_out->ops.f_tx(port_out->h_port, pkt);
1196 port_out->f_action(pkt, &pkt_mask,
1198 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1199 (pkt_mask ^ 1LLU) << i;
1201 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1204 /* Output port TX */
1206 port_out->ops.f_tx(port_out->h_port,
1214 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1216 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1217 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1220 for (i = 0; i < n_pkts; i++)
1221 rte_pktmbuf_free(p->pkts[i]);
1225 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1226 uint64_t pkt_mask = 1LLU << i;
1228 if ((pkt_mask & pkts_mask) == 0)
1231 rte_pktmbuf_free(p->pkts[i]);
1237 rte_pipeline_run(struct rte_pipeline *p)
1239 struct rte_port_in *port_in;
1241 for (port_in = p->port_in_first; port_in != NULL;
1242 port_in = port_in->next) {
1244 uint32_t n_pkts, table_id;
1247 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1248 port_in->burst_size);
1252 pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1253 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1254 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1255 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1256 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1258 /* Input port user actions */
1259 if (port_in->f_action != NULL) {
1260 uint64_t mask = pkts_mask;
1262 port_in->f_action(p->pkts, n_pkts, &pkts_mask, port_in->arg_ah);
1264 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1265 RTE_PIPELINE_STATS_ADD_M(port_in->n_pkts_dropped_by_ah, mask);
1269 for (table_id = port_in->table_id; pkts_mask != 0; ) {
1270 struct rte_table *table;
1271 uint64_t lookup_hit_mask, lookup_miss_mask;
1274 table = &p->tables[table_id];
1275 table->ops.f_lookup(table->h_table, p->pkts, pkts_mask,
1276 &lookup_hit_mask, (void **) p->entries);
1277 lookup_miss_mask = pkts_mask & (~lookup_hit_mask);
1280 if (lookup_miss_mask != 0) {
1281 struct rte_pipeline_table_entry *default_entry =
1282 table->default_entry;
1284 /* Table user actions */
1285 if (table->f_action_miss != NULL) {
1286 uint64_t mask = lookup_miss_mask;
1288 table->f_action_miss(p->pkts,
1290 default_entry, table->arg_ah);
1291 mask ^= lookup_miss_mask;
1292 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1293 RTE_PIPELINE_STATS_ADD_M(
1294 table->n_pkts_dropped_by_lkp_miss_ah, mask);
1297 /* Table reserved actions */
1298 if ((default_entry->action ==
1299 RTE_PIPELINE_ACTION_PORT) &&
1300 (lookup_miss_mask != 0))
1301 rte_pipeline_action_handler_port_bulk(p,
1303 default_entry->port_id);
1305 uint32_t pos = default_entry->action;
1307 p->action_mask0[pos] = lookup_miss_mask;
1308 if (pos == RTE_PIPELINE_ACTION_DROP) {
1309 RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_miss,
1316 if (lookup_hit_mask != 0) {
1317 /* Table user actions */
1318 if (table->f_action_hit != NULL) {
1319 uint64_t mask = lookup_hit_mask;
1321 table->f_action_hit(p->pkts,
1323 p->entries, table->arg_ah);
1324 mask ^= lookup_hit_mask;
1325 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1326 RTE_PIPELINE_STATS_ADD_M(
1327 table->n_pkts_dropped_by_lkp_hit_ah, mask);
1330 /* Table reserved actions */
1331 rte_pipeline_compute_masks(p, lookup_hit_mask);
1332 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1334 RTE_PIPELINE_ACTION_DROP];
1335 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1337 RTE_PIPELINE_ACTION_PORT];
1338 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1340 RTE_PIPELINE_ACTION_PORT_META];
1341 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1343 RTE_PIPELINE_ACTION_TABLE];
1345 RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_hit,
1346 p->action_mask1[RTE_PIPELINE_ACTION_DROP]);
1349 /* Prepare for next iteration */
1350 pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1351 table_id = table->table_next_id;
1352 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1355 /* Table reserved action PORT */
1356 rte_pipeline_action_handler_port(p,
1357 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1359 /* Table reserved action PORT META */
1360 rte_pipeline_action_handler_port_meta(p,
1361 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1363 /* Table reserved action DROP */
1364 rte_pipeline_action_handler_drop(p,
1365 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1372 rte_pipeline_flush(struct rte_pipeline *p)
1376 /* Check input arguments */
1378 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1383 for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1384 struct rte_port_out *port = &p->ports_out[port_id];
1386 if (port->ops.f_flush != NULL)
1387 port->ops.f_flush(port->h_port);
1394 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1395 uint32_t port_id, struct rte_mbuf *pkt)
1397 struct rte_port_out *port_out = &p->ports_out[port_id];
1399 /* Output port user actions */
1400 if (port_out->f_action == NULL)
1401 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1403 uint64_t pkt_mask = 1LLU;
1405 port_out->f_action(pkt, &pkt_mask, port_out->arg_ah);
1407 if (pkt_mask != 0) /* Output port TX */
1408 port_out->ops.f_tx(port_out->h_port, pkt);
1410 rte_pktmbuf_free(pkt);
1411 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, 1);
1418 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1419 struct rte_pipeline_port_in_stats *stats, int clear)
1421 struct rte_port_in *port;
1425 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1430 if (port_id >= p->num_ports_in) {
1431 RTE_LOG(ERR, PIPELINE,
1432 "%s: port IN ID %u is out of range\n",
1437 port = &p->ports_in[port_id];
1439 if (port->ops.f_stats != NULL) {
1440 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1443 } else if (stats != NULL)
1444 memset(&stats->stats, 0, sizeof(stats->stats));
1447 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1450 port->n_pkts_dropped_by_ah = 0;
1455 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1456 struct rte_pipeline_port_out_stats *stats, int clear)
1458 struct rte_port_out *port;
1462 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1466 if (port_id >= p->num_ports_out) {
1467 RTE_LOG(ERR, PIPELINE,
1468 "%s: port OUT ID %u is out of range\n", __func__, port_id);
1472 port = &p->ports_out[port_id];
1473 if (port->ops.f_stats != NULL) {
1474 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1477 } else if (stats != NULL)
1478 memset(&stats->stats, 0, sizeof(stats->stats));
1481 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1484 port->n_pkts_dropped_by_ah = 0;
1489 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1490 struct rte_pipeline_table_stats *stats, int clear)
1492 struct rte_table *table;
1496 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1501 if (table_id >= p->num_tables) {
1502 RTE_LOG(ERR, PIPELINE,
1503 "%s: table %u is out of range\n", __func__, table_id);
1507 table = &p->tables[table_id];
1508 if (table->ops.f_stats != NULL) {
1509 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1512 } else if (stats != NULL)
1513 memset(&stats->stats, 0, sizeof(stats->stats));
1515 if (stats != NULL) {
1516 stats->n_pkts_dropped_by_lkp_hit_ah =
1517 table->n_pkts_dropped_by_lkp_hit_ah;
1518 stats->n_pkts_dropped_by_lkp_miss_ah =
1519 table->n_pkts_dropped_by_lkp_miss_ah;
1520 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1521 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1525 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1526 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1527 table->n_pkts_dropped_lkp_hit = 0;
1528 table->n_pkts_dropped_lkp_miss = 0;