1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
8 #include <rte_common.h>
9 #include <rte_memory.h>
10 #include <rte_cycles.h>
11 #include <rte_prefetch.h>
12 #include <rte_branch_prediction.h>
14 #include <rte_malloc.h>
15 #include <rte_string_fns.h>
17 #include "rte_pipeline.h"
19 #define RTE_TABLE_INVALID UINT32_MAX
21 #ifdef RTE_PIPELINE_STATS_COLLECT
23 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \
24 ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
26 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \
27 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
29 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \
30 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
32 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \
34 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \
35 mask ^= (p)->pkts_drop_mask; \
36 (counter) += __builtin_popcountll(mask); \
41 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
42 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
43 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
44 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
49 /* Input parameters */
50 struct rte_port_in_ops ops;
51 rte_pipeline_port_in_action_handler f_action;
55 /* The table to which this port is connected */
58 /* Handle to low-level port */
61 /* List of enabled ports */
62 struct rte_port_in *next;
65 uint64_t n_pkts_dropped_by_ah;
69 /* Input parameters */
70 struct rte_port_out_ops ops;
71 rte_pipeline_port_out_action_handler f_action;
74 /* Handle to low-level port */
78 uint64_t n_pkts_dropped_by_ah;
82 /* Input parameters */
83 struct rte_table_ops ops;
84 rte_pipeline_table_action_handler_hit f_action_hit;
85 rte_pipeline_table_action_handler_miss f_action_miss;
87 struct rte_pipeline_table_entry *default_entry;
90 uint32_t table_next_id;
91 uint32_t table_next_id_valid;
93 /* Handle to the low-level table object */
97 uint64_t n_pkts_dropped_by_lkp_hit_ah;
98 uint64_t n_pkts_dropped_by_lkp_miss_ah;
99 uint64_t n_pkts_dropped_lkp_hit;
100 uint64_t n_pkts_dropped_lkp_miss;
103 #define RTE_PIPELINE_MAX_NAME_SZ 124
105 struct rte_pipeline {
106 /* Input parameters */
107 char name[RTE_PIPELINE_MAX_NAME_SZ];
109 uint32_t offset_port_id;
111 /* Internal tables */
112 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
113 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
114 struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
116 /* Occupancy of internal tables */
117 uint32_t num_ports_in;
118 uint32_t num_ports_out;
121 /* List of enabled ports */
122 uint64_t enabled_port_in_mask;
123 struct rte_port_in *port_in_next;
125 /* Pipeline run structures */
126 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
127 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
128 uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
129 uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
131 uint64_t n_pkts_ah_drop;
132 uint64_t pkts_drop_mask;
133 } __rte_cache_aligned;
135 static inline uint32_t
136 rte_mask_get_next(uint64_t mask, uint32_t pos)
138 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
139 (mask >> ((pos + 1) & 0x3F));
140 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
143 static inline uint32_t
144 rte_mask_get_prev(uint64_t mask, uint32_t pos)
146 uint64_t mask_rot = (mask >> (pos & 0x3F)) |
147 (mask << ((64 - pos) & 0x3F));
148 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
152 rte_pipeline_table_free(struct rte_table *table);
155 rte_pipeline_port_in_free(struct rte_port_in *port);
158 rte_pipeline_port_out_free(struct rte_port_out *port);
165 rte_pipeline_check_params(struct rte_pipeline_params *params)
167 if (params == NULL) {
168 RTE_LOG(ERR, PIPELINE,
169 "%s: Incorrect value for parameter params\n", __func__);
174 if (params->name == NULL) {
175 RTE_LOG(ERR, PIPELINE,
176 "%s: Incorrect value for parameter name\n", __func__);
181 if (params->socket_id < 0) {
182 RTE_LOG(ERR, PIPELINE,
183 "%s: Incorrect value for parameter socket_id\n",
191 struct rte_pipeline *
192 rte_pipeline_create(struct rte_pipeline_params *params)
194 struct rte_pipeline *p;
197 /* Check input parameters */
198 status = rte_pipeline_check_params(params);
200 RTE_LOG(ERR, PIPELINE,
201 "%s: Pipeline params check failed (%d)\n",
206 /* Allocate memory for the pipeline on requested socket */
207 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
208 RTE_CACHE_LINE_SIZE, params->socket_id);
211 RTE_LOG(ERR, PIPELINE,
212 "%s: Pipeline memory allocation failed\n", __func__);
216 /* Save input parameters */
217 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ);
218 p->socket_id = params->socket_id;
219 p->offset_port_id = params->offset_port_id;
221 /* Initialize pipeline internal data structure */
223 p->num_ports_out = 0;
225 p->enabled_port_in_mask = 0;
226 p->port_in_next = NULL;
228 p->n_pkts_ah_drop = 0;
234 rte_pipeline_free(struct rte_pipeline *p)
238 /* Check input parameters */
240 RTE_LOG(ERR, PIPELINE,
241 "%s: rte_pipeline parameter is NULL\n", __func__);
245 /* Free input ports */
246 for (i = 0; i < p->num_ports_in; i++) {
247 struct rte_port_in *port = &p->ports_in[i];
249 rte_pipeline_port_in_free(port);
253 for (i = 0; i < p->num_tables; i++) {
254 struct rte_table *table = &p->tables[i];
256 rte_pipeline_table_free(table);
259 /* Free output ports */
260 for (i = 0; i < p->num_ports_out; i++) {
261 struct rte_port_out *port = &p->ports_out[i];
263 rte_pipeline_port_out_free(port);
266 /* Free pipeline memory */
277 rte_table_check_params(struct rte_pipeline *p,
278 struct rte_pipeline_table_params *params,
282 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
286 if (params == NULL) {
287 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
291 if (table_id == NULL) {
292 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
298 if (params->ops == NULL) {
299 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
304 if (params->ops->f_create == NULL) {
305 RTE_LOG(ERR, PIPELINE,
306 "%s: f_create function pointer is NULL\n", __func__);
310 if (params->ops->f_lookup == NULL) {
311 RTE_LOG(ERR, PIPELINE,
312 "%s: f_lookup function pointer is NULL\n", __func__);
316 /* De we have room for one more table? */
317 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
318 RTE_LOG(ERR, PIPELINE,
319 "%s: Incorrect value for num_tables parameter\n",
328 rte_pipeline_table_create(struct rte_pipeline *p,
329 struct rte_pipeline_table_params *params,
332 struct rte_table *table;
333 struct rte_pipeline_table_entry *default_entry;
335 uint32_t entry_size, id;
338 /* Check input arguments */
339 status = rte_table_check_params(p, params, table_id);
344 table = &p->tables[id];
346 /* Allocate space for the default table entry */
347 entry_size = sizeof(struct rte_pipeline_table_entry) +
348 params->action_data_size;
349 default_entry = rte_zmalloc_socket(
350 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
351 if (default_entry == NULL) {
352 RTE_LOG(ERR, PIPELINE,
353 "%s: Failed to allocate default entry\n", __func__);
357 /* Create the table */
358 h_table = params->ops->f_create(params->arg_create, p->socket_id,
360 if (h_table == NULL) {
361 rte_free(default_entry);
362 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
366 /* Commit current table to the pipeline */
370 /* Save input parameters */
371 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
372 table->f_action_hit = params->f_action_hit;
373 table->f_action_miss = params->f_action_miss;
374 table->arg_ah = params->arg_ah;
375 table->entry_size = entry_size;
377 /* Clear the lookup miss actions (to be set later through API) */
378 table->default_entry = default_entry;
379 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
381 /* Initialize table internal data structure */
382 table->h_table = h_table;
383 table->table_next_id = 0;
384 table->table_next_id_valid = 0;
390 rte_pipeline_table_free(struct rte_table *table)
392 if (table->ops.f_free != NULL)
393 table->ops.f_free(table->h_table);
395 rte_free(table->default_entry);
399 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
401 struct rte_pipeline_table_entry *default_entry,
402 struct rte_pipeline_table_entry **default_entry_ptr)
404 struct rte_table *table;
406 /* Check input arguments */
408 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
413 if (default_entry == NULL) {
414 RTE_LOG(ERR, PIPELINE,
415 "%s: default_entry parameter is NULL\n", __func__);
419 if (table_id >= p->num_tables) {
420 RTE_LOG(ERR, PIPELINE,
421 "%s: table_id %d out of range\n", __func__, table_id);
425 table = &p->tables[table_id];
427 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
428 table->table_next_id_valid &&
429 (default_entry->table_id != table->table_next_id)) {
430 RTE_LOG(ERR, PIPELINE,
431 "%s: Tree-like topologies not allowed\n", __func__);
435 /* Set the lookup miss actions */
436 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
437 (table->table_next_id_valid == 0)) {
438 table->table_next_id = default_entry->table_id;
439 table->table_next_id_valid = 1;
442 memcpy(table->default_entry, default_entry, table->entry_size);
444 *default_entry_ptr = table->default_entry;
449 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
451 struct rte_pipeline_table_entry *entry)
453 struct rte_table *table;
455 /* Check input arguments */
457 RTE_LOG(ERR, PIPELINE,
458 "%s: pipeline parameter is NULL\n", __func__);
462 if (table_id >= p->num_tables) {
463 RTE_LOG(ERR, PIPELINE,
464 "%s: table_id %d out of range\n", __func__, table_id);
468 table = &p->tables[table_id];
470 /* Save the current contents of the default entry */
472 memcpy(entry, table->default_entry, table->entry_size);
474 /* Clear the lookup miss actions */
475 memset(table->default_entry, 0, table->entry_size);
476 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
482 rte_pipeline_table_entry_add(struct rte_pipeline *p,
485 struct rte_pipeline_table_entry *entry,
487 struct rte_pipeline_table_entry **entry_ptr)
489 struct rte_table *table;
491 /* Check input arguments */
493 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
499 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
504 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
509 if (table_id >= p->num_tables) {
510 RTE_LOG(ERR, PIPELINE,
511 "%s: table_id %d out of range\n", __func__, table_id);
515 table = &p->tables[table_id];
517 if (table->ops.f_add == NULL) {
518 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
523 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
524 table->table_next_id_valid &&
525 (entry->table_id != table->table_next_id)) {
526 RTE_LOG(ERR, PIPELINE,
527 "%s: Tree-like topologies not allowed\n", __func__);
532 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
533 (table->table_next_id_valid == 0)) {
534 table->table_next_id = entry->table_id;
535 table->table_next_id_valid = 1;
538 return (table->ops.f_add)(table->h_table, key, (void *) entry,
539 key_found, (void **) entry_ptr);
543 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
547 struct rte_pipeline_table_entry *entry)
549 struct rte_table *table;
551 /* Check input arguments */
553 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
559 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
564 if (table_id >= p->num_tables) {
565 RTE_LOG(ERR, PIPELINE,
566 "%s: table_id %d out of range\n", __func__, table_id);
570 table = &p->tables[table_id];
572 if (table->ops.f_delete == NULL) {
573 RTE_LOG(ERR, PIPELINE,
574 "%s: f_delete function pointer NULL\n", __func__);
578 return (table->ops.f_delete)(table->h_table, key, key_found, entry);
581 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
584 struct rte_pipeline_table_entry **entries,
587 struct rte_pipeline_table_entry **entries_ptr)
589 struct rte_table *table;
592 /* Check input arguments */
594 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
600 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
604 if (entries == NULL) {
605 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
610 if (table_id >= p->num_tables) {
611 RTE_LOG(ERR, PIPELINE,
612 "%s: table_id %d out of range\n", __func__, table_id);
616 table = &p->tables[table_id];
618 if (table->ops.f_add_bulk == NULL) {
619 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
624 for (i = 0; i < n_keys; i++) {
625 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
626 table->table_next_id_valid &&
627 (entries[i]->table_id != table->table_next_id)) {
628 RTE_LOG(ERR, PIPELINE,
629 "%s: Tree-like topologies not allowed\n", __func__);
635 for (i = 0; i < n_keys; i++) {
636 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
637 (table->table_next_id_valid == 0)) {
638 table->table_next_id = entries[i]->table_id;
639 table->table_next_id_valid = 1;
643 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
644 n_keys, key_found, (void **) entries_ptr);
647 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
652 struct rte_pipeline_table_entry **entries)
654 struct rte_table *table;
656 /* Check input arguments */
658 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
664 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
669 if (table_id >= p->num_tables) {
670 RTE_LOG(ERR, PIPELINE,
671 "%s: table_id %d out of range\n", __func__, table_id);
675 table = &p->tables[table_id];
677 if (table->ops.f_delete_bulk == NULL) {
678 RTE_LOG(ERR, PIPELINE,
679 "%s: f_delete function pointer NULL\n", __func__);
683 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
692 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
693 struct rte_pipeline_port_in_params *params,
697 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
701 if (params == NULL) {
702 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
705 if (port_id == NULL) {
706 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
712 if (params->ops == NULL) {
713 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
718 if (params->ops->f_create == NULL) {
719 RTE_LOG(ERR, PIPELINE,
720 "%s: f_create function pointer NULL\n", __func__);
724 if (params->ops->f_rx == NULL) {
725 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
731 if ((params->burst_size == 0) ||
732 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
733 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
738 /* Do we have room for one more port? */
739 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
740 RTE_LOG(ERR, PIPELINE,
741 "%s: invalid value for num_ports_in\n", __func__);
749 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
750 struct rte_pipeline_port_out_params *params,
754 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
759 if (params == NULL) {
760 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
764 if (port_id == NULL) {
765 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
771 if (params->ops == NULL) {
772 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
777 if (params->ops->f_create == NULL) {
778 RTE_LOG(ERR, PIPELINE,
779 "%s: f_create function pointer NULL\n", __func__);
783 if (params->ops->f_tx == NULL) {
784 RTE_LOG(ERR, PIPELINE,
785 "%s: f_tx function pointer NULL\n", __func__);
789 if (params->ops->f_tx_bulk == NULL) {
790 RTE_LOG(ERR, PIPELINE,
791 "%s: f_tx_bulk function pointer NULL\n", __func__);
795 /* Do we have room for one more port? */
796 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
797 RTE_LOG(ERR, PIPELINE,
798 "%s: invalid value for num_ports_out\n", __func__);
806 rte_pipeline_port_in_create(struct rte_pipeline *p,
807 struct rte_pipeline_port_in_params *params,
810 struct rte_port_in *port;
815 /* Check input arguments */
816 status = rte_pipeline_port_in_check_params(p, params, port_id);
820 id = p->num_ports_in;
821 port = &p->ports_in[id];
823 /* Create the port */
824 h_port = params->ops->f_create(params->arg_create, p->socket_id);
825 if (h_port == NULL) {
826 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
830 /* Commit current table to the pipeline */
834 /* Save input parameters */
835 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
836 port->f_action = params->f_action;
837 port->arg_ah = params->arg_ah;
838 port->burst_size = params->burst_size;
840 /* Initialize port internal data structure */
841 port->table_id = RTE_TABLE_INVALID;
842 port->h_port = h_port;
849 rte_pipeline_port_in_free(struct rte_port_in *port)
851 if (port->ops.f_free != NULL)
852 port->ops.f_free(port->h_port);
856 rte_pipeline_port_out_create(struct rte_pipeline *p,
857 struct rte_pipeline_port_out_params *params,
860 struct rte_port_out *port;
865 /* Check input arguments */
866 status = rte_pipeline_port_out_check_params(p, params, port_id);
870 id = p->num_ports_out;
871 port = &p->ports_out[id];
873 /* Create the port */
874 h_port = params->ops->f_create(params->arg_create, p->socket_id);
875 if (h_port == NULL) {
876 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
880 /* Commit current table to the pipeline */
884 /* Save input parameters */
885 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
886 port->f_action = params->f_action;
887 port->arg_ah = params->arg_ah;
889 /* Initialize port internal data structure */
890 port->h_port = h_port;
896 rte_pipeline_port_out_free(struct rte_port_out *port)
898 if (port->ops.f_free != NULL)
899 port->ops.f_free(port->h_port);
903 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
907 struct rte_port_in *port;
909 /* Check input arguments */
911 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
916 if (port_id >= p->num_ports_in) {
917 RTE_LOG(ERR, PIPELINE,
918 "%s: port IN ID %u is out of range\n",
923 if (table_id >= p->num_tables) {
924 RTE_LOG(ERR, PIPELINE,
925 "%s: Table ID %u is out of range\n",
930 port = &p->ports_in[port_id];
931 port->table_id = table_id;
937 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
939 struct rte_port_in *port, *port_prev, *port_next;
941 uint32_t port_prev_id, port_next_id;
943 /* Check input arguments */
945 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
950 if (port_id >= p->num_ports_in) {
951 RTE_LOG(ERR, PIPELINE,
952 "%s: port IN ID %u is out of range\n",
957 port = &p->ports_in[port_id];
959 /* Return if current input port is already enabled */
960 port_mask = 1LLU << port_id;
961 if (p->enabled_port_in_mask & port_mask)
964 p->enabled_port_in_mask |= port_mask;
966 /* Add current input port to the pipeline chain of enabled ports */
967 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
968 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
970 port_prev = &p->ports_in[port_prev_id];
971 port_next = &p->ports_in[port_next_id];
973 port_prev->next = port;
974 port->next = port_next;
976 /* Check if list of enabled ports was previously empty */
977 if (p->enabled_port_in_mask == port_mask)
978 p->port_in_next = port;
984 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
986 struct rte_port_in *port, *port_prev, *port_next;
988 uint32_t port_prev_id, port_next_id;
990 /* Check input arguments */
992 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
997 if (port_id >= p->num_ports_in) {
998 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
1003 port = &p->ports_in[port_id];
1005 /* Return if current input port is already disabled */
1006 port_mask = 1LLU << port_id;
1007 if ((p->enabled_port_in_mask & port_mask) == 0)
1010 p->enabled_port_in_mask &= ~port_mask;
1012 /* Return if no other enabled ports */
1013 if (p->enabled_port_in_mask == 0) {
1014 p->port_in_next = NULL;
1019 /* Add current input port to the pipeline chain of enabled ports */
1020 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1021 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1023 port_prev = &p->ports_in[port_prev_id];
1024 port_next = &p->ports_in[port_next_id];
1026 port_prev->next = port_next;
1028 /* Check if the port which has just been disabled is next to serve */
1029 if (port == p->port_in_next)
1030 p->port_in_next = port_next;
1040 rte_pipeline_check(struct rte_pipeline *p)
1042 uint32_t port_in_id;
1044 /* Check input arguments */
1046 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1051 /* Check that pipeline has at least one input port, one table and one
1053 if (p->num_ports_in == 0) {
1054 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1058 if (p->num_tables == 0) {
1059 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1063 if (p->num_ports_out == 0) {
1064 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1069 /* Check that all input ports are connected */
1070 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1071 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1073 if (port_in->table_id == RTE_TABLE_INVALID) {
1074 RTE_LOG(ERR, PIPELINE,
1075 "%s: Port IN ID %u is not connected\n",
1076 __func__, port_in_id);
1085 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1087 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1088 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1089 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1090 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1092 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1093 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1096 for (i = 0; i < n_pkts; i++) {
1097 uint64_t pkt_mask = 1LLU << i;
1098 uint32_t pos = p->entries[i]->action;
1100 p->action_mask1[pos] |= pkt_mask;
1105 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1106 uint64_t pkt_mask = 1LLU << i;
1109 if ((pkt_mask & pkts_mask) == 0)
1112 pos = p->entries[i]->action;
1113 p->action_mask1[pos] |= pkt_mask;
1119 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1120 uint64_t pkts_mask, uint32_t port_id)
1122 struct rte_port_out *port_out = &p->ports_out[port_id];
1124 p->pkts_mask = pkts_mask;
1126 /* Output port user actions */
1127 if (port_out->f_action != NULL) {
1128 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1130 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1131 port_out->n_pkts_dropped_by_ah);
1134 /* Output port TX */
1135 if (p->pkts_mask != 0)
1136 port_out->ops.f_tx_bulk(port_out->h_port,
1142 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1144 p->pkts_mask = pkts_mask;
1146 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1147 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1150 for (i = 0; i < n_pkts; i++) {
1151 struct rte_mbuf *pkt = p->pkts[i];
1152 uint32_t port_out_id = p->entries[i]->port_id;
1153 struct rte_port_out *port_out =
1154 &p->ports_out[port_out_id];
1156 /* Output port user actions */
1157 if (port_out->f_action == NULL) /* Output port TX */
1158 port_out->ops.f_tx(port_out->h_port, pkt);
1160 uint64_t pkt_mask = 1LLU << i;
1162 port_out->f_action(p,
1167 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1168 port_out->n_pkts_dropped_by_ah);
1170 /* Output port TX */
1171 if (pkt_mask & p->pkts_mask)
1172 port_out->ops.f_tx(port_out->h_port,
1179 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1180 uint64_t pkt_mask = 1LLU << i;
1181 struct rte_mbuf *pkt;
1182 struct rte_port_out *port_out;
1183 uint32_t port_out_id;
1185 if ((pkt_mask & pkts_mask) == 0)
1189 port_out_id = p->entries[i]->port_id;
1190 port_out = &p->ports_out[port_out_id];
1192 /* Output port user actions */
1193 if (port_out->f_action == NULL) /* Output port TX */
1194 port_out->ops.f_tx(port_out->h_port, pkt);
1196 port_out->f_action(p,
1201 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1202 port_out->n_pkts_dropped_by_ah);
1204 /* Output port TX */
1205 if (pkt_mask & p->pkts_mask)
1206 port_out->ops.f_tx(port_out->h_port,
1214 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1217 p->pkts_mask = pkts_mask;
1219 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1220 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1223 for (i = 0; i < n_pkts; i++) {
1224 struct rte_mbuf *pkt = p->pkts[i];
1225 uint32_t port_out_id =
1226 RTE_MBUF_METADATA_UINT32(pkt,
1228 struct rte_port_out *port_out = &p->ports_out[
1231 /* Output port user actions */
1232 if (port_out->f_action == NULL) /* Output port TX */
1233 port_out->ops.f_tx(port_out->h_port, pkt);
1235 uint64_t pkt_mask = 1LLU << i;
1237 port_out->f_action(p,
1242 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1243 port_out->n_pkts_dropped_by_ah);
1245 /* Output port TX */
1246 if (pkt_mask & p->pkts_mask)
1247 port_out->ops.f_tx(port_out->h_port,
1254 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1255 uint64_t pkt_mask = 1LLU << i;
1256 struct rte_mbuf *pkt;
1257 struct rte_port_out *port_out;
1258 uint32_t port_out_id;
1260 if ((pkt_mask & pkts_mask) == 0)
1264 port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1266 port_out = &p->ports_out[port_out_id];
1268 /* Output port user actions */
1269 if (port_out->f_action == NULL) /* Output port TX */
1270 port_out->ops.f_tx(port_out->h_port, pkt);
1272 port_out->f_action(p,
1277 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1278 port_out->n_pkts_dropped_by_ah);
1280 /* Output port TX */
1281 if (pkt_mask & p->pkts_mask)
1282 port_out->ops.f_tx(port_out->h_port,
1290 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1292 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1293 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1296 for (i = 0; i < n_pkts; i++)
1297 rte_pktmbuf_free(p->pkts[i]);
1301 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1302 uint64_t pkt_mask = 1LLU << i;
1304 if ((pkt_mask & pkts_mask) == 0)
1307 rte_pktmbuf_free(p->pkts[i]);
1313 rte_pipeline_run(struct rte_pipeline *p)
1315 struct rte_port_in *port_in = p->port_in_next;
1316 uint32_t n_pkts, table_id;
1318 if (port_in == NULL)
1322 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1323 port_in->burst_size);
1325 p->port_in_next = port_in->next;
1329 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1330 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1331 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1332 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1333 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1335 /* Input port user actions */
1336 if (port_in->f_action != NULL) {
1337 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1339 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1340 port_in->n_pkts_dropped_by_ah);
1344 for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1345 struct rte_table *table;
1346 uint64_t lookup_hit_mask, lookup_miss_mask;
1349 table = &p->tables[table_id];
1350 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1351 &lookup_hit_mask, (void **) p->entries);
1352 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1355 if (lookup_miss_mask != 0) {
1356 struct rte_pipeline_table_entry *default_entry =
1357 table->default_entry;
1359 p->pkts_mask = lookup_miss_mask;
1361 /* Table user actions */
1362 if (table->f_action_miss != NULL) {
1363 table->f_action_miss(p,
1369 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1370 table->n_pkts_dropped_by_lkp_miss_ah);
1373 /* Table reserved actions */
1374 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1375 (p->pkts_mask != 0))
1376 rte_pipeline_action_handler_port_bulk(p,
1378 default_entry->port_id);
1380 uint32_t pos = default_entry->action;
1382 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1384 p->action_mask0[pos] |= p->pkts_mask;
1386 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1387 table->n_pkts_dropped_lkp_miss);
1392 if (lookup_hit_mask != 0) {
1393 p->pkts_mask = lookup_hit_mask;
1395 /* Table user actions */
1396 if (table->f_action_hit != NULL) {
1397 table->f_action_hit(p,
1403 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1404 table->n_pkts_dropped_by_lkp_hit_ah);
1407 /* Table reserved actions */
1408 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1409 rte_pipeline_compute_masks(p, p->pkts_mask);
1410 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1412 RTE_PIPELINE_ACTION_DROP];
1413 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1415 RTE_PIPELINE_ACTION_PORT];
1416 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1418 RTE_PIPELINE_ACTION_PORT_META];
1419 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1421 RTE_PIPELINE_ACTION_TABLE];
1423 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1424 table->n_pkts_dropped_lkp_hit);
1427 /* Prepare for next iteration */
1428 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1429 table_id = table->table_next_id;
1430 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1433 /* Table reserved action PORT */
1434 rte_pipeline_action_handler_port(p,
1435 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1437 /* Table reserved action PORT META */
1438 rte_pipeline_action_handler_port_meta(p,
1439 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1441 /* Table reserved action DROP */
1442 rte_pipeline_action_handler_drop(p,
1443 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1445 /* Pick candidate for next port IN to serve */
1446 p->port_in_next = port_in->next;
1448 return (int) n_pkts;
1452 rte_pipeline_flush(struct rte_pipeline *p)
1456 /* Check input arguments */
1458 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1463 for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1464 struct rte_port_out *port = &p->ports_out[port_id];
1466 if (port->ops.f_flush != NULL)
1467 port->ops.f_flush(port->h_port);
1474 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1475 uint32_t port_id, struct rte_mbuf *pkt)
1477 struct rte_port_out *port_out = &p->ports_out[port_id];
1479 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1484 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1487 pkts_mask &= p->pkts_mask;
1488 p->pkts_mask &= ~pkts_mask;
1493 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1496 pkts_mask &= p->pkts_mask;
1497 p->pkts_mask &= ~pkts_mask;
1498 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1500 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1504 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1505 struct rte_pipeline_port_in_stats *stats, int clear)
1507 struct rte_port_in *port;
1511 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1516 if (port_id >= p->num_ports_in) {
1517 RTE_LOG(ERR, PIPELINE,
1518 "%s: port IN ID %u is out of range\n",
1523 port = &p->ports_in[port_id];
1525 if (port->ops.f_stats != NULL) {
1526 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1529 } else if (stats != NULL)
1530 memset(&stats->stats, 0, sizeof(stats->stats));
1533 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1536 port->n_pkts_dropped_by_ah = 0;
1541 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1542 struct rte_pipeline_port_out_stats *stats, int clear)
1544 struct rte_port_out *port;
1548 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1552 if (port_id >= p->num_ports_out) {
1553 RTE_LOG(ERR, PIPELINE,
1554 "%s: port OUT ID %u is out of range\n", __func__, port_id);
1558 port = &p->ports_out[port_id];
1559 if (port->ops.f_stats != NULL) {
1560 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1563 } else if (stats != NULL)
1564 memset(&stats->stats, 0, sizeof(stats->stats));
1567 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1570 port->n_pkts_dropped_by_ah = 0;
1575 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1576 struct rte_pipeline_table_stats *stats, int clear)
1578 struct rte_table *table;
1582 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1587 if (table_id >= p->num_tables) {
1588 RTE_LOG(ERR, PIPELINE,
1589 "%s: table %u is out of range\n", __func__, table_id);
1593 table = &p->tables[table_id];
1594 if (table->ops.f_stats != NULL) {
1595 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1598 } else if (stats != NULL)
1599 memset(&stats->stats, 0, sizeof(stats->stats));
1601 if (stats != NULL) {
1602 stats->n_pkts_dropped_by_lkp_hit_ah =
1603 table->n_pkts_dropped_by_lkp_hit_ah;
1604 stats->n_pkts_dropped_by_lkp_miss_ah =
1605 table->n_pkts_dropped_by_lkp_miss_ah;
1606 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1607 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1611 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1612 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1613 table->n_pkts_dropped_lkp_hit = 0;
1614 table->n_pkts_dropped_lkp_miss = 0;