1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
8 #include <rte_common.h>
10 #include <rte_malloc.h>
11 #include <rte_string_fns.h>
13 #include "rte_pipeline.h"
15 #define RTE_TABLE_INVALID UINT32_MAX
17 #ifdef RTE_PIPELINE_STATS_COLLECT
19 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \
20 ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
22 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \
23 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
25 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \
26 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
28 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \
30 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \
31 mask ^= (p)->pkts_drop_mask; \
32 (counter) += __builtin_popcountll(mask); \
37 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
38 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
39 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
40 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
45 /* Input parameters */
46 struct rte_port_in_ops ops;
47 rte_pipeline_port_in_action_handler f_action;
51 /* The table to which this port is connected */
54 /* Handle to low-level port */
57 /* List of enabled ports */
58 struct rte_port_in *next;
61 uint64_t n_pkts_dropped_by_ah;
65 /* Input parameters */
66 struct rte_port_out_ops ops;
67 rte_pipeline_port_out_action_handler f_action;
70 /* Handle to low-level port */
74 uint64_t n_pkts_dropped_by_ah;
78 /* Input parameters */
79 struct rte_table_ops ops;
80 rte_pipeline_table_action_handler_hit f_action_hit;
81 rte_pipeline_table_action_handler_miss f_action_miss;
83 struct rte_pipeline_table_entry *default_entry;
86 uint32_t table_next_id;
87 uint32_t table_next_id_valid;
89 /* Handle to the low-level table object */
93 uint64_t n_pkts_dropped_by_lkp_hit_ah;
94 uint64_t n_pkts_dropped_by_lkp_miss_ah;
95 uint64_t n_pkts_dropped_lkp_hit;
96 uint64_t n_pkts_dropped_lkp_miss;
99 #define RTE_PIPELINE_MAX_NAME_SZ 124
101 struct rte_pipeline {
102 /* Input parameters */
103 char name[RTE_PIPELINE_MAX_NAME_SZ];
105 uint32_t offset_port_id;
107 /* Internal tables */
108 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
109 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
110 struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
112 /* Occupancy of internal tables */
113 uint32_t num_ports_in;
114 uint32_t num_ports_out;
117 /* List of enabled ports */
118 uint64_t enabled_port_in_mask;
119 struct rte_port_in *port_in_next;
121 /* Pipeline run structures */
122 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
123 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
124 uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
125 uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
127 uint64_t n_pkts_ah_drop;
128 uint64_t pkts_drop_mask;
129 } __rte_cache_aligned;
131 static inline uint32_t
132 rte_mask_get_next(uint64_t mask, uint32_t pos)
134 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
135 (mask >> ((pos + 1) & 0x3F));
136 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
139 static inline uint32_t
140 rte_mask_get_prev(uint64_t mask, uint32_t pos)
142 uint64_t mask_rot = (mask >> (pos & 0x3F)) |
143 (mask << ((64 - pos) & 0x3F));
144 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
148 rte_pipeline_table_free(struct rte_table *table);
151 rte_pipeline_port_in_free(struct rte_port_in *port);
154 rte_pipeline_port_out_free(struct rte_port_out *port);
161 rte_pipeline_check_params(struct rte_pipeline_params *params)
163 if (params == NULL) {
164 RTE_LOG(ERR, PIPELINE,
165 "%s: Incorrect value for parameter params\n", __func__);
170 if (params->name == NULL) {
171 RTE_LOG(ERR, PIPELINE,
172 "%s: Incorrect value for parameter name\n", __func__);
177 if (params->socket_id < 0) {
178 RTE_LOG(ERR, PIPELINE,
179 "%s: Incorrect value for parameter socket_id\n",
187 struct rte_pipeline *
188 rte_pipeline_create(struct rte_pipeline_params *params)
190 struct rte_pipeline *p;
193 /* Check input parameters */
194 status = rte_pipeline_check_params(params);
196 RTE_LOG(ERR, PIPELINE,
197 "%s: Pipeline params check failed (%d)\n",
202 /* Allocate memory for the pipeline on requested socket */
203 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
204 RTE_CACHE_LINE_SIZE, params->socket_id);
207 RTE_LOG(ERR, PIPELINE,
208 "%s: Pipeline memory allocation failed\n", __func__);
212 /* Save input parameters */
213 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ);
214 p->socket_id = params->socket_id;
215 p->offset_port_id = params->offset_port_id;
217 /* Initialize pipeline internal data structure */
219 p->num_ports_out = 0;
221 p->enabled_port_in_mask = 0;
222 p->port_in_next = NULL;
224 p->n_pkts_ah_drop = 0;
230 rte_pipeline_free(struct rte_pipeline *p)
234 /* Check input parameters */
236 RTE_LOG(ERR, PIPELINE,
237 "%s: rte_pipeline parameter is NULL\n", __func__);
241 /* Free input ports */
242 for (i = 0; i < p->num_ports_in; i++) {
243 struct rte_port_in *port = &p->ports_in[i];
245 rte_pipeline_port_in_free(port);
249 for (i = 0; i < p->num_tables; i++) {
250 struct rte_table *table = &p->tables[i];
252 rte_pipeline_table_free(table);
255 /* Free output ports */
256 for (i = 0; i < p->num_ports_out; i++) {
257 struct rte_port_out *port = &p->ports_out[i];
259 rte_pipeline_port_out_free(port);
262 /* Free pipeline memory */
273 rte_table_check_params(struct rte_pipeline *p,
274 struct rte_pipeline_table_params *params,
278 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
282 if (params == NULL) {
283 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
287 if (table_id == NULL) {
288 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
294 if (params->ops == NULL) {
295 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
300 if (params->ops->f_create == NULL) {
301 RTE_LOG(ERR, PIPELINE,
302 "%s: f_create function pointer is NULL\n", __func__);
306 if (params->ops->f_lookup == NULL) {
307 RTE_LOG(ERR, PIPELINE,
308 "%s: f_lookup function pointer is NULL\n", __func__);
312 /* De we have room for one more table? */
313 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
314 RTE_LOG(ERR, PIPELINE,
315 "%s: Incorrect value for num_tables parameter\n",
324 rte_pipeline_table_create(struct rte_pipeline *p,
325 struct rte_pipeline_table_params *params,
328 struct rte_table *table;
329 struct rte_pipeline_table_entry *default_entry;
331 uint32_t entry_size, id;
334 /* Check input arguments */
335 status = rte_table_check_params(p, params, table_id);
340 table = &p->tables[id];
342 /* Allocate space for the default table entry */
343 entry_size = sizeof(struct rte_pipeline_table_entry) +
344 params->action_data_size;
345 default_entry = rte_zmalloc_socket(
346 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
347 if (default_entry == NULL) {
348 RTE_LOG(ERR, PIPELINE,
349 "%s: Failed to allocate default entry\n", __func__);
353 /* Create the table */
354 h_table = params->ops->f_create(params->arg_create, p->socket_id,
356 if (h_table == NULL) {
357 rte_free(default_entry);
358 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
362 /* Commit current table to the pipeline */
366 /* Save input parameters */
367 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
368 table->f_action_hit = params->f_action_hit;
369 table->f_action_miss = params->f_action_miss;
370 table->arg_ah = params->arg_ah;
371 table->entry_size = entry_size;
373 /* Clear the lookup miss actions (to be set later through API) */
374 table->default_entry = default_entry;
375 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
377 /* Initialize table internal data structure */
378 table->h_table = h_table;
379 table->table_next_id = 0;
380 table->table_next_id_valid = 0;
386 rte_pipeline_table_free(struct rte_table *table)
388 if (table->ops.f_free != NULL)
389 table->ops.f_free(table->h_table);
391 rte_free(table->default_entry);
395 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
397 struct rte_pipeline_table_entry *default_entry,
398 struct rte_pipeline_table_entry **default_entry_ptr)
400 struct rte_table *table;
402 /* Check input arguments */
404 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
409 if (default_entry == NULL) {
410 RTE_LOG(ERR, PIPELINE,
411 "%s: default_entry parameter is NULL\n", __func__);
415 if (table_id >= p->num_tables) {
416 RTE_LOG(ERR, PIPELINE,
417 "%s: table_id %d out of range\n", __func__, table_id);
421 table = &p->tables[table_id];
423 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
424 table->table_next_id_valid &&
425 (default_entry->table_id != table->table_next_id)) {
426 RTE_LOG(ERR, PIPELINE,
427 "%s: Tree-like topologies not allowed\n", __func__);
431 /* Set the lookup miss actions */
432 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
433 (table->table_next_id_valid == 0)) {
434 table->table_next_id = default_entry->table_id;
435 table->table_next_id_valid = 1;
438 memcpy(table->default_entry, default_entry, table->entry_size);
440 *default_entry_ptr = table->default_entry;
445 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
447 struct rte_pipeline_table_entry *entry)
449 struct rte_table *table;
451 /* Check input arguments */
453 RTE_LOG(ERR, PIPELINE,
454 "%s: pipeline parameter is NULL\n", __func__);
458 if (table_id >= p->num_tables) {
459 RTE_LOG(ERR, PIPELINE,
460 "%s: table_id %d out of range\n", __func__, table_id);
464 table = &p->tables[table_id];
466 /* Save the current contents of the default entry */
468 memcpy(entry, table->default_entry, table->entry_size);
470 /* Clear the lookup miss actions */
471 memset(table->default_entry, 0, table->entry_size);
472 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
478 rte_pipeline_table_entry_add(struct rte_pipeline *p,
481 struct rte_pipeline_table_entry *entry,
483 struct rte_pipeline_table_entry **entry_ptr)
485 struct rte_table *table;
487 /* Check input arguments */
489 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
495 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
500 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
505 if (table_id >= p->num_tables) {
506 RTE_LOG(ERR, PIPELINE,
507 "%s: table_id %d out of range\n", __func__, table_id);
511 table = &p->tables[table_id];
513 if (table->ops.f_add == NULL) {
514 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
519 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
520 table->table_next_id_valid &&
521 (entry->table_id != table->table_next_id)) {
522 RTE_LOG(ERR, PIPELINE,
523 "%s: Tree-like topologies not allowed\n", __func__);
528 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
529 (table->table_next_id_valid == 0)) {
530 table->table_next_id = entry->table_id;
531 table->table_next_id_valid = 1;
534 return (table->ops.f_add)(table->h_table, key, (void *) entry,
535 key_found, (void **) entry_ptr);
539 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
543 struct rte_pipeline_table_entry *entry)
545 struct rte_table *table;
547 /* Check input arguments */
549 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
555 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
560 if (table_id >= p->num_tables) {
561 RTE_LOG(ERR, PIPELINE,
562 "%s: table_id %d out of range\n", __func__, table_id);
566 table = &p->tables[table_id];
568 if (table->ops.f_delete == NULL) {
569 RTE_LOG(ERR, PIPELINE,
570 "%s: f_delete function pointer NULL\n", __func__);
574 return (table->ops.f_delete)(table->h_table, key, key_found, entry);
577 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
580 struct rte_pipeline_table_entry **entries,
583 struct rte_pipeline_table_entry **entries_ptr)
585 struct rte_table *table;
588 /* Check input arguments */
590 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
596 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
600 if (entries == NULL) {
601 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
606 if (table_id >= p->num_tables) {
607 RTE_LOG(ERR, PIPELINE,
608 "%s: table_id %d out of range\n", __func__, table_id);
612 table = &p->tables[table_id];
614 if (table->ops.f_add_bulk == NULL) {
615 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
620 for (i = 0; i < n_keys; i++) {
621 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
622 table->table_next_id_valid &&
623 (entries[i]->table_id != table->table_next_id)) {
624 RTE_LOG(ERR, PIPELINE,
625 "%s: Tree-like topologies not allowed\n", __func__);
631 for (i = 0; i < n_keys; i++) {
632 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
633 (table->table_next_id_valid == 0)) {
634 table->table_next_id = entries[i]->table_id;
635 table->table_next_id_valid = 1;
639 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
640 n_keys, key_found, (void **) entries_ptr);
643 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
648 struct rte_pipeline_table_entry **entries)
650 struct rte_table *table;
652 /* Check input arguments */
654 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
660 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
665 if (table_id >= p->num_tables) {
666 RTE_LOG(ERR, PIPELINE,
667 "%s: table_id %d out of range\n", __func__, table_id);
671 table = &p->tables[table_id];
673 if (table->ops.f_delete_bulk == NULL) {
674 RTE_LOG(ERR, PIPELINE,
675 "%s: f_delete function pointer NULL\n", __func__);
679 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
688 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
689 struct rte_pipeline_port_in_params *params,
693 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
697 if (params == NULL) {
698 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
701 if (port_id == NULL) {
702 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
708 if (params->ops == NULL) {
709 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
714 if (params->ops->f_create == NULL) {
715 RTE_LOG(ERR, PIPELINE,
716 "%s: f_create function pointer NULL\n", __func__);
720 if (params->ops->f_rx == NULL) {
721 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
727 if ((params->burst_size == 0) ||
728 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
729 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
734 /* Do we have room for one more port? */
735 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
736 RTE_LOG(ERR, PIPELINE,
737 "%s: invalid value for num_ports_in\n", __func__);
745 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
746 struct rte_pipeline_port_out_params *params,
750 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
755 if (params == NULL) {
756 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
760 if (port_id == NULL) {
761 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
767 if (params->ops == NULL) {
768 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
773 if (params->ops->f_create == NULL) {
774 RTE_LOG(ERR, PIPELINE,
775 "%s: f_create function pointer NULL\n", __func__);
779 if (params->ops->f_tx == NULL) {
780 RTE_LOG(ERR, PIPELINE,
781 "%s: f_tx function pointer NULL\n", __func__);
785 if (params->ops->f_tx_bulk == NULL) {
786 RTE_LOG(ERR, PIPELINE,
787 "%s: f_tx_bulk function pointer NULL\n", __func__);
791 /* Do we have room for one more port? */
792 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
793 RTE_LOG(ERR, PIPELINE,
794 "%s: invalid value for num_ports_out\n", __func__);
802 rte_pipeline_port_in_create(struct rte_pipeline *p,
803 struct rte_pipeline_port_in_params *params,
806 struct rte_port_in *port;
811 /* Check input arguments */
812 status = rte_pipeline_port_in_check_params(p, params, port_id);
816 id = p->num_ports_in;
817 port = &p->ports_in[id];
819 /* Create the port */
820 h_port = params->ops->f_create(params->arg_create, p->socket_id);
821 if (h_port == NULL) {
822 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
826 /* Commit current table to the pipeline */
830 /* Save input parameters */
831 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
832 port->f_action = params->f_action;
833 port->arg_ah = params->arg_ah;
834 port->burst_size = params->burst_size;
836 /* Initialize port internal data structure */
837 port->table_id = RTE_TABLE_INVALID;
838 port->h_port = h_port;
845 rte_pipeline_port_in_free(struct rte_port_in *port)
847 if (port->ops.f_free != NULL)
848 port->ops.f_free(port->h_port);
852 rte_pipeline_port_out_create(struct rte_pipeline *p,
853 struct rte_pipeline_port_out_params *params,
856 struct rte_port_out *port;
861 /* Check input arguments */
862 status = rte_pipeline_port_out_check_params(p, params, port_id);
866 id = p->num_ports_out;
867 port = &p->ports_out[id];
869 /* Create the port */
870 h_port = params->ops->f_create(params->arg_create, p->socket_id);
871 if (h_port == NULL) {
872 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
876 /* Commit current table to the pipeline */
880 /* Save input parameters */
881 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
882 port->f_action = params->f_action;
883 port->arg_ah = params->arg_ah;
885 /* Initialize port internal data structure */
886 port->h_port = h_port;
892 rte_pipeline_port_out_free(struct rte_port_out *port)
894 if (port->ops.f_free != NULL)
895 port->ops.f_free(port->h_port);
899 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
903 struct rte_port_in *port;
905 /* Check input arguments */
907 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
912 if (port_id >= p->num_ports_in) {
913 RTE_LOG(ERR, PIPELINE,
914 "%s: port IN ID %u is out of range\n",
919 if (table_id >= p->num_tables) {
920 RTE_LOG(ERR, PIPELINE,
921 "%s: Table ID %u is out of range\n",
926 port = &p->ports_in[port_id];
927 port->table_id = table_id;
933 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
935 struct rte_port_in *port, *port_prev, *port_next;
937 uint32_t port_prev_id, port_next_id;
939 /* Check input arguments */
941 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
946 if (port_id >= p->num_ports_in) {
947 RTE_LOG(ERR, PIPELINE,
948 "%s: port IN ID %u is out of range\n",
953 port = &p->ports_in[port_id];
955 /* Return if current input port is already enabled */
956 port_mask = 1LLU << port_id;
957 if (p->enabled_port_in_mask & port_mask)
960 p->enabled_port_in_mask |= port_mask;
962 /* Add current input port to the pipeline chain of enabled ports */
963 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
964 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
966 port_prev = &p->ports_in[port_prev_id];
967 port_next = &p->ports_in[port_next_id];
969 port_prev->next = port;
970 port->next = port_next;
972 /* Check if list of enabled ports was previously empty */
973 if (p->enabled_port_in_mask == port_mask)
974 p->port_in_next = port;
980 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
982 struct rte_port_in *port, *port_prev, *port_next;
984 uint32_t port_prev_id, port_next_id;
986 /* Check input arguments */
988 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
993 if (port_id >= p->num_ports_in) {
994 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
999 port = &p->ports_in[port_id];
1001 /* Return if current input port is already disabled */
1002 port_mask = 1LLU << port_id;
1003 if ((p->enabled_port_in_mask & port_mask) == 0)
1006 p->enabled_port_in_mask &= ~port_mask;
1008 /* Return if no other enabled ports */
1009 if (p->enabled_port_in_mask == 0) {
1010 p->port_in_next = NULL;
1015 /* Add current input port to the pipeline chain of enabled ports */
1016 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1017 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1019 port_prev = &p->ports_in[port_prev_id];
1020 port_next = &p->ports_in[port_next_id];
1022 port_prev->next = port_next;
1024 /* Check if the port which has just been disabled is next to serve */
1025 if (port == p->port_in_next)
1026 p->port_in_next = port_next;
1036 rte_pipeline_check(struct rte_pipeline *p)
1038 uint32_t port_in_id;
1040 /* Check input arguments */
1042 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1047 /* Check that pipeline has at least one input port, one table and one
1049 if (p->num_ports_in == 0) {
1050 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1054 if (p->num_tables == 0) {
1055 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1059 if (p->num_ports_out == 0) {
1060 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1065 /* Check that all input ports are connected */
1066 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1067 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1069 if (port_in->table_id == RTE_TABLE_INVALID) {
1070 RTE_LOG(ERR, PIPELINE,
1071 "%s: Port IN ID %u is not connected\n",
1072 __func__, port_in_id);
1081 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1083 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1084 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1085 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1086 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1088 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1089 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1092 for (i = 0; i < n_pkts; i++) {
1093 uint64_t pkt_mask = 1LLU << i;
1094 uint32_t pos = p->entries[i]->action;
1096 p->action_mask1[pos] |= pkt_mask;
1101 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1102 uint64_t pkt_mask = 1LLU << i;
1105 if ((pkt_mask & pkts_mask) == 0)
1108 pos = p->entries[i]->action;
1109 p->action_mask1[pos] |= pkt_mask;
1115 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1116 uint64_t pkts_mask, uint32_t port_id)
1118 struct rte_port_out *port_out = &p->ports_out[port_id];
1120 p->pkts_mask = pkts_mask;
1122 /* Output port user actions */
1123 if (port_out->f_action != NULL) {
1124 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1126 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1127 port_out->n_pkts_dropped_by_ah);
1130 /* Output port TX */
1131 if (p->pkts_mask != 0)
1132 port_out->ops.f_tx_bulk(port_out->h_port,
1138 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1140 p->pkts_mask = pkts_mask;
1142 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1143 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1146 for (i = 0; i < n_pkts; i++) {
1147 struct rte_mbuf *pkt = p->pkts[i];
1148 uint32_t port_out_id = p->entries[i]->port_id;
1149 struct rte_port_out *port_out =
1150 &p->ports_out[port_out_id];
1152 /* Output port user actions */
1153 if (port_out->f_action == NULL) /* Output port TX */
1154 port_out->ops.f_tx(port_out->h_port, pkt);
1156 uint64_t pkt_mask = 1LLU << i;
1158 port_out->f_action(p,
1163 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1164 port_out->n_pkts_dropped_by_ah);
1166 /* Output port TX */
1167 if (pkt_mask & p->pkts_mask)
1168 port_out->ops.f_tx(port_out->h_port,
1175 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1176 uint64_t pkt_mask = 1LLU << i;
1177 struct rte_mbuf *pkt;
1178 struct rte_port_out *port_out;
1179 uint32_t port_out_id;
1181 if ((pkt_mask & pkts_mask) == 0)
1185 port_out_id = p->entries[i]->port_id;
1186 port_out = &p->ports_out[port_out_id];
1188 /* Output port user actions */
1189 if (port_out->f_action == NULL) /* Output port TX */
1190 port_out->ops.f_tx(port_out->h_port, pkt);
1192 port_out->f_action(p,
1197 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1198 port_out->n_pkts_dropped_by_ah);
1200 /* Output port TX */
1201 if (pkt_mask & p->pkts_mask)
1202 port_out->ops.f_tx(port_out->h_port,
1210 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1213 p->pkts_mask = pkts_mask;
1215 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1216 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1219 for (i = 0; i < n_pkts; i++) {
1220 struct rte_mbuf *pkt = p->pkts[i];
1221 uint32_t port_out_id =
1222 RTE_MBUF_METADATA_UINT32(pkt,
1224 struct rte_port_out *port_out = &p->ports_out[
1227 /* Output port user actions */
1228 if (port_out->f_action == NULL) /* Output port TX */
1229 port_out->ops.f_tx(port_out->h_port, pkt);
1231 uint64_t pkt_mask = 1LLU << i;
1233 port_out->f_action(p,
1238 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1239 port_out->n_pkts_dropped_by_ah);
1241 /* Output port TX */
1242 if (pkt_mask & p->pkts_mask)
1243 port_out->ops.f_tx(port_out->h_port,
1250 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1251 uint64_t pkt_mask = 1LLU << i;
1252 struct rte_mbuf *pkt;
1253 struct rte_port_out *port_out;
1254 uint32_t port_out_id;
1256 if ((pkt_mask & pkts_mask) == 0)
1260 port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1262 port_out = &p->ports_out[port_out_id];
1264 /* Output port user actions */
1265 if (port_out->f_action == NULL) /* Output port TX */
1266 port_out->ops.f_tx(port_out->h_port, pkt);
1268 port_out->f_action(p,
1273 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1274 port_out->n_pkts_dropped_by_ah);
1276 /* Output port TX */
1277 if (pkt_mask & p->pkts_mask)
1278 port_out->ops.f_tx(port_out->h_port,
1286 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1288 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1289 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1292 for (i = 0; i < n_pkts; i++)
1293 rte_pktmbuf_free(p->pkts[i]);
1297 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1298 uint64_t pkt_mask = 1LLU << i;
1300 if ((pkt_mask & pkts_mask) == 0)
1303 rte_pktmbuf_free(p->pkts[i]);
1309 rte_pipeline_run(struct rte_pipeline *p)
1311 struct rte_port_in *port_in = p->port_in_next;
1312 uint32_t n_pkts, table_id;
1314 if (port_in == NULL)
1318 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1319 port_in->burst_size);
1321 p->port_in_next = port_in->next;
1325 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1326 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1327 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1328 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1329 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1331 /* Input port user actions */
1332 if (port_in->f_action != NULL) {
1333 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1335 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1336 port_in->n_pkts_dropped_by_ah);
1340 for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1341 struct rte_table *table;
1342 uint64_t lookup_hit_mask, lookup_miss_mask;
1345 table = &p->tables[table_id];
1346 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1347 &lookup_hit_mask, (void **) p->entries);
1348 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1351 if (lookup_miss_mask != 0) {
1352 struct rte_pipeline_table_entry *default_entry =
1353 table->default_entry;
1355 p->pkts_mask = lookup_miss_mask;
1357 /* Table user actions */
1358 if (table->f_action_miss != NULL) {
1359 table->f_action_miss(p,
1365 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1366 table->n_pkts_dropped_by_lkp_miss_ah);
1369 /* Table reserved actions */
1370 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1371 (p->pkts_mask != 0))
1372 rte_pipeline_action_handler_port_bulk(p,
1374 default_entry->port_id);
1376 uint32_t pos = default_entry->action;
1378 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1380 p->action_mask0[pos] |= p->pkts_mask;
1382 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1383 table->n_pkts_dropped_lkp_miss);
1388 if (lookup_hit_mask != 0) {
1389 p->pkts_mask = lookup_hit_mask;
1391 /* Table user actions */
1392 if (table->f_action_hit != NULL) {
1393 table->f_action_hit(p,
1399 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1400 table->n_pkts_dropped_by_lkp_hit_ah);
1403 /* Table reserved actions */
1404 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1405 rte_pipeline_compute_masks(p, p->pkts_mask);
1406 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1408 RTE_PIPELINE_ACTION_DROP];
1409 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1411 RTE_PIPELINE_ACTION_PORT];
1412 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1414 RTE_PIPELINE_ACTION_PORT_META];
1415 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1417 RTE_PIPELINE_ACTION_TABLE];
1419 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1420 table->n_pkts_dropped_lkp_hit);
1423 /* Prepare for next iteration */
1424 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1425 table_id = table->table_next_id;
1426 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1429 /* Table reserved action PORT */
1430 rte_pipeline_action_handler_port(p,
1431 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1433 /* Table reserved action PORT META */
1434 rte_pipeline_action_handler_port_meta(p,
1435 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1437 /* Table reserved action DROP */
1438 rte_pipeline_action_handler_drop(p,
1439 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1441 /* Pick candidate for next port IN to serve */
1442 p->port_in_next = port_in->next;
1444 return (int) n_pkts;
1448 rte_pipeline_flush(struct rte_pipeline *p)
1452 /* Check input arguments */
1454 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1459 for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1460 struct rte_port_out *port = &p->ports_out[port_id];
1462 if (port->ops.f_flush != NULL)
1463 port->ops.f_flush(port->h_port);
1470 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1471 uint32_t port_id, struct rte_mbuf *pkt)
1473 struct rte_port_out *port_out = &p->ports_out[port_id];
1475 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1480 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1483 pkts_mask &= p->pkts_mask;
1484 p->pkts_mask &= ~pkts_mask;
1489 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1492 pkts_mask &= p->pkts_mask;
1493 p->pkts_mask &= ~pkts_mask;
1494 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1496 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1500 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1501 struct rte_pipeline_port_in_stats *stats, int clear)
1503 struct rte_port_in *port;
1507 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1512 if (port_id >= p->num_ports_in) {
1513 RTE_LOG(ERR, PIPELINE,
1514 "%s: port IN ID %u is out of range\n",
1519 port = &p->ports_in[port_id];
1521 if (port->ops.f_stats != NULL) {
1522 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1525 } else if (stats != NULL)
1526 memset(&stats->stats, 0, sizeof(stats->stats));
1529 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1532 port->n_pkts_dropped_by_ah = 0;
1537 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1538 struct rte_pipeline_port_out_stats *stats, int clear)
1540 struct rte_port_out *port;
1544 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1548 if (port_id >= p->num_ports_out) {
1549 RTE_LOG(ERR, PIPELINE,
1550 "%s: port OUT ID %u is out of range\n", __func__, port_id);
1554 port = &p->ports_out[port_id];
1555 if (port->ops.f_stats != NULL) {
1556 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1559 } else if (stats != NULL)
1560 memset(&stats->stats, 0, sizeof(stats->stats));
1563 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1566 port->n_pkts_dropped_by_ah = 0;
1571 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1572 struct rte_pipeline_table_stats *stats, int clear)
1574 struct rte_table *table;
1578 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1583 if (table_id >= p->num_tables) {
1584 RTE_LOG(ERR, PIPELINE,
1585 "%s: table %u is out of range\n", __func__, table_id);
1589 table = &p->tables[table_id];
1590 if (table->ops.f_stats != NULL) {
1591 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1594 } else if (stats != NULL)
1595 memset(&stats->stats, 0, sizeof(stats->stats));
1597 if (stats != NULL) {
1598 stats->n_pkts_dropped_by_lkp_hit_ah =
1599 table->n_pkts_dropped_by_lkp_hit_ah;
1600 stats->n_pkts_dropped_by_lkp_miss_ah =
1601 table->n_pkts_dropped_by_lkp_miss_ah;
1602 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1603 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1607 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1608 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1609 table->n_pkts_dropped_lkp_hit = 0;
1610 table->n_pkts_dropped_lkp_miss = 0;