X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_pipeline%2Frte_pipeline.c;h=8611a88b9c54f562d9db1c125e0cc394573d6fe0;hb=4c00cfdc0ea225f2518a35db928ad1ab02b2a724;hp=56022f4cb88c62a1c7b97b449bd88e42d6a43a20;hpb=53c9ada02a33ebf3037e77e65be426a4b1504b5b;p=dpdk.git diff --git a/lib/librte_pipeline/rte_pipeline.c b/lib/librte_pipeline/rte_pipeline.c index 56022f4cb8..8611a88b9c 100644 --- a/lib/librte_pipeline/rte_pipeline.c +++ b/lib/librte_pipeline/rte_pipeline.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,7 +36,6 @@ #include #include -#include #include #include #include @@ -49,14 +48,30 @@ #define RTE_TABLE_INVALID UINT32_MAX #ifdef RTE_PIPELINE_STATS_COLLECT -#define RTE_PIPELINE_STATS_ADD(counter, val) \ - ({ (counter) += (val); }) -#define RTE_PIPELINE_STATS_ADD_M(counter, mask) \ - ({ (counter) += __builtin_popcountll(mask); }) +#define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \ + ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); }) + +#define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \ + ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; }) + +#define RTE_PIPELINE_STATS_TABLE_DROP0(p) \ + ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; }) + +#define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \ +({ \ + uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \ + mask ^= (p)->pkts_drop_mask; \ + (counter) += __builtin_popcountll(mask); \ +}) + #else -#define RTE_PIPELINE_STATS_ADD(counter, val) -#define RTE_PIPELINE_STATS_ADD_M(counter, mask) + +#define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) +#define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) +#define RTE_PIPELINE_STATS_TABLE_DROP0(p) +#define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) + #endif struct rte_port_in { @@ -75,6 +90,7 @@ struct rte_port_in { /* List of enabled ports */ struct rte_port_in *next; + /* Statistics */ uint64_t n_pkts_dropped_by_ah; }; @@ -82,12 +98,12 @@ struct rte_port_out { /* Input parameters */ struct rte_port_out_ops ops; rte_pipeline_port_out_action_handler f_action; - rte_pipeline_port_out_action_handler_bulk f_action_bulk; void *arg_ah; /* Handle to low-level port */ void *h_port; + /* Statistics */ uint64_t n_pkts_dropped_by_ah; }; @@ -106,7 +122,7 @@ struct rte_table { /* Handle to the low-level table object */ void *h_table; - /* Stats for this table. */ + /* Statistics */ uint64_t n_pkts_dropped_by_lkp_hit_ah; uint64_t n_pkts_dropped_by_lkp_miss_ah; uint64_t n_pkts_dropped_lkp_hit; @@ -133,13 +149,16 @@ struct rte_pipeline { /* List of enabled ports */ uint64_t enabled_port_in_mask; - struct rte_port_in *port_in_first; + struct rte_port_in *port_in_next; /* Pipeline run structures */ struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX]; uint64_t action_mask0[RTE_PIPELINE_ACTIONS]; uint64_t action_mask1[RTE_PIPELINE_ACTIONS]; + uint64_t pkts_mask; + uint64_t n_pkts_ah_drop; + uint64_t pkts_drop_mask; } __rte_cache_aligned; static inline uint32_t @@ -234,7 +253,9 @@ rte_pipeline_create(struct rte_pipeline_params *params) p->num_ports_out = 0; p->num_tables = 0; p->enabled_port_in_mask = 0; - p->port_in_first = NULL; + p->port_in_next = NULL; + p->pkts_mask = 0; + p->n_pkts_ah_drop = 0; return p; } @@ -759,9 +780,6 @@ rte_pipeline_port_out_check_params(struct rte_pipeline *p, struct rte_pipeline_port_out_params *params, uint32_t *port_id) { - rte_pipeline_port_out_action_handler f_ah; - rte_pipeline_port_out_action_handler_bulk f_ah_bulk; - if (p == NULL) { RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__); @@ -794,7 +812,7 @@ rte_pipeline_port_out_check_params(struct rte_pipeline *p, if (params->ops->f_tx == NULL) { RTE_LOG(ERR, PIPELINE, - "%s: f_tx function pointer NULL\n", __func__); + "%s: f_tx function pointer NULL\n", __func__); return -EINVAL; } @@ -804,15 +822,6 @@ rte_pipeline_port_out_check_params(struct rte_pipeline *p, return -EINVAL; } - f_ah = params->f_action; - f_ah_bulk = params->f_action_bulk; - if (((f_ah != NULL) && (f_ah_bulk == NULL)) || - ((f_ah == NULL) && (f_ah_bulk != NULL))) { - RTE_LOG(ERR, PIPELINE, "%s: Action handlers have to be either" - "both enabled or both disabled\n", __func__); - return -EINVAL; - } - /* Do we have room for one more port? */ if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) { RTE_LOG(ERR, PIPELINE, @@ -905,7 +914,6 @@ rte_pipeline_port_out_create(struct rte_pipeline *p, /* Save input parameters */ memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops)); port->f_action = params->f_action; - port->f_action_bulk = params->f_action_bulk; port->arg_ah = params->arg_ah; /* Initialize port internal data structure */ @@ -959,9 +967,8 @@ int rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) { struct rte_port_in *port, *port_prev, *port_next; - struct rte_port_in *port_first, *port_last; uint64_t port_mask; - uint32_t port_prev_id, port_next_id, port_first_id, port_last_id; + uint32_t port_prev_id, port_next_id; /* Check input arguments */ if (p == NULL) { @@ -977,6 +984,8 @@ rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) return -EINVAL; } + port = &p->ports_in[port_id]; + /* Return if current input port is already enabled */ port_mask = 1LLU << port_id; if (p->enabled_port_in_mask & port_mask) @@ -990,20 +999,13 @@ rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) port_prev = &p->ports_in[port_prev_id]; port_next = &p->ports_in[port_next_id]; - port = &p->ports_in[port_id]; port_prev->next = port; port->next = port_next; - /* Update the first and last input ports in the chain */ - port_first_id = __builtin_ctzll(p->enabled_port_in_mask); - port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask); - - port_first = &p->ports_in[port_first_id]; - port_last = &p->ports_in[port_last_id]; - - p->port_in_first = port_first; - port_last->next = NULL; + /* Check if list of enabled ports was previously empty */ + if (p->enabled_port_in_mask == port_mask) + p->port_in_next = port; return 0; } @@ -1011,9 +1013,9 @@ rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) int rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) { - struct rte_port_in *port_prev, *port_next, *port_first, *port_last; + struct rte_port_in *port, *port_prev, *port_next; uint64_t port_mask; - uint32_t port_prev_id, port_next_id, port_first_id, port_last_id; + uint32_t port_prev_id, port_next_id; /* Check input arguments */ if (p == NULL) { @@ -1028,15 +1030,18 @@ rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) return -EINVAL; } + port = &p->ports_in[port_id]; + /* Return if current input port is already disabled */ port_mask = 1LLU << port_id; if ((p->enabled_port_in_mask & port_mask) == 0) return 0; + p->enabled_port_in_mask &= ~port_mask; + /* Return if no other enabled ports */ - if (__builtin_popcountll(p->enabled_port_in_mask) == 1) { - p->enabled_port_in_mask &= ~port_mask; - p->port_in_first = NULL; + if (p->enabled_port_in_mask == 0) { + p->port_in_next = NULL; return 0; } @@ -1049,17 +1054,10 @@ rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) port_next = &p->ports_in[port_next_id]; port_prev->next = port_next; - p->enabled_port_in_mask &= ~port_mask; - - /* Update the first and last input ports in the chain */ - port_first_id = __builtin_ctzll(p->enabled_port_in_mask); - port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask); - port_first = &p->ports_in[port_first_id]; - port_last = &p->ports_in[port_last_id]; - - p->port_in_first = port_first; - port_last->next = NULL; + /* Check if the port which has just been disabled is next to serve */ + if (port == p->port_in_next) + p->port_in_next = port_next; return 0; } @@ -1149,28 +1147,32 @@ rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask) static inline void rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p, - uint64_t pkts_mask, uint32_t port_id) + uint64_t pkts_mask, uint32_t port_id) { struct rte_port_out *port_out = &p->ports_out[port_id]; + p->pkts_mask = pkts_mask; + /* Output port user actions */ - if (port_out->f_action_bulk != NULL) { - uint64_t mask = pkts_mask; + if (port_out->f_action != NULL) { + port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah); - port_out->f_action_bulk(p->pkts, &pkts_mask, port_out->arg_ah); - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask ^ mask; - RTE_PIPELINE_STATS_ADD_M(port_out->n_pkts_dropped_by_ah, - pkts_mask ^ mask); + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); } /* Output port TX */ - if (pkts_mask != 0) - port_out->ops.f_tx_bulk(port_out->h_port, p->pkts, pkts_mask); + if (p->pkts_mask != 0) + port_out->ops.f_tx_bulk(port_out->h_port, + p->pkts, + p->pkts_mask); } static inline void rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) { + p->pkts_mask = pkts_mask; + if ((pkts_mask & (pkts_mask + 1)) == 0) { uint64_t n_pkts = __builtin_popcountll(pkts_mask); uint32_t i; @@ -1185,18 +1187,18 @@ rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) if (port_out->f_action == NULL) /* Output port TX */ port_out->ops.f_tx(port_out->h_port, pkt); else { - uint64_t pkt_mask = 1LLU; + uint64_t pkt_mask = 1LLU << i; - port_out->f_action(pkt, &pkt_mask, + port_out->f_action(p, + p->pkts, + pkt_mask, port_out->arg_ah); - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= - (pkt_mask ^ 1LLU) << i; - RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, - pkt_mask ^ 1LLU); + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); /* Output port TX */ - if (pkt_mask != 0) + if (pkt_mask & p->pkts_mask) port_out->ops.f_tx(port_out->h_port, pkt); } @@ -1221,18 +1223,16 @@ rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) if (port_out->f_action == NULL) /* Output port TX */ port_out->ops.f_tx(port_out->h_port, pkt); else { - pkt_mask = 1LLU; - - port_out->f_action(pkt, &pkt_mask, + port_out->f_action(p, + p->pkts, + pkt_mask, port_out->arg_ah); - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= - (pkt_mask ^ 1LLU) << i; - RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, - pkt_mask ^ 1LLU); + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); /* Output port TX */ - if (pkt_mask != 0) + if (pkt_mask & p->pkts_mask) port_out->ops.f_tx(port_out->h_port, pkt); } @@ -1244,6 +1244,8 @@ static inline void rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, uint64_t pkts_mask) { + p->pkts_mask = pkts_mask; + if ((pkts_mask & (pkts_mask + 1)) == 0) { uint64_t n_pkts = __builtin_popcountll(pkts_mask); uint32_t i; @@ -1260,18 +1262,18 @@ rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, if (port_out->f_action == NULL) /* Output port TX */ port_out->ops.f_tx(port_out->h_port, pkt); else { - uint64_t pkt_mask = 1LLU; + uint64_t pkt_mask = 1LLU << i; - port_out->f_action(pkt, &pkt_mask, + port_out->f_action(p, + p->pkts, + pkt_mask, port_out->arg_ah); - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= - (pkt_mask ^ 1LLU) << i; - RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, - pkt_mask ^ 1ULL); + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); /* Output port TX */ - if (pkt_mask != 0) + if (pkt_mask & p->pkts_mask) port_out->ops.f_tx(port_out->h_port, pkt); } @@ -1297,18 +1299,16 @@ rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, if (port_out->f_action == NULL) /* Output port TX */ port_out->ops.f_tx(port_out->h_port, pkt); else { - pkt_mask = 1LLU; - - port_out->f_action(pkt, &pkt_mask, + port_out->f_action(p, + p->pkts, + pkt_mask, port_out->arg_ah); - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= - (pkt_mask ^ 1LLU) << i; - RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, - pkt_mask ^ 1ULL); + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_out->n_pkts_dropped_by_ah); /* Output port TX */ - if (pkt_mask != 0) + if (pkt_mask & p->pkts_mask) port_out->ops.f_tx(port_out->h_port, pkt); } @@ -1342,136 +1342,140 @@ rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask) int rte_pipeline_run(struct rte_pipeline *p) { - struct rte_port_in *port_in; - - for (port_in = p->port_in_first; port_in != NULL; - port_in = port_in->next) { - uint64_t pkts_mask; - uint32_t n_pkts, table_id; - - /* Input port RX */ - n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, - port_in->burst_size); - if (n_pkts == 0) - continue; - - pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); - p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; - p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; - p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; - p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; + struct rte_port_in *port_in = p->port_in_next; + uint32_t n_pkts, table_id; - /* Input port user actions */ - if (port_in->f_action != NULL) { - uint64_t mask = pkts_mask; + if (port_in == NULL) + return 0; - port_in->f_action(p->pkts, n_pkts, &pkts_mask, port_in->arg_ah); - mask ^= pkts_mask; - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask; - RTE_PIPELINE_STATS_ADD_M(port_in->n_pkts_dropped_by_ah, mask); - } + /* Input port RX */ + n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, + port_in->burst_size); + if (n_pkts == 0) { + p->port_in_next = port_in->next; + return 0; + } - /* Table */ - for (table_id = port_in->table_id; pkts_mask != 0; ) { - struct rte_table *table; - uint64_t lookup_hit_mask, lookup_miss_mask; - - /* Lookup */ - table = &p->tables[table_id]; - table->ops.f_lookup(table->h_table, p->pkts, pkts_mask, - &lookup_hit_mask, (void **) p->entries); - lookup_miss_mask = pkts_mask & (~lookup_hit_mask); - - /* Lookup miss */ - if (lookup_miss_mask != 0) { - struct rte_pipeline_table_entry *default_entry = - table->default_entry; - - /* Table user actions */ - if (table->f_action_miss != NULL) { - uint64_t mask = lookup_miss_mask; - - table->f_action_miss(p->pkts, - &lookup_miss_mask, - default_entry, table->arg_ah); - mask ^= lookup_miss_mask; - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask; - RTE_PIPELINE_STATS_ADD_M( - table->n_pkts_dropped_by_lkp_miss_ah, mask); - } - - /* Table reserved actions */ - if ((default_entry->action == - RTE_PIPELINE_ACTION_PORT) && - (lookup_miss_mask != 0)) - rte_pipeline_action_handler_port_bulk(p, - lookup_miss_mask, - default_entry->port_id); - else { - uint32_t pos = default_entry->action; - - p->action_mask0[pos] = lookup_miss_mask; - if (pos == RTE_PIPELINE_ACTION_DROP) { - RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_miss, - lookup_miss_mask); - } - } - } + p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); + p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; + p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; + p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; + p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; + + /* Input port user actions */ + if (port_in->f_action != NULL) { + port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + port_in->n_pkts_dropped_by_ah); + } - /* Lookup hit */ - if (lookup_hit_mask != 0) { - /* Table user actions */ - if (table->f_action_hit != NULL) { - uint64_t mask = lookup_hit_mask; - - table->f_action_hit(p->pkts, - &lookup_hit_mask, - p->entries, table->arg_ah); - mask ^= lookup_hit_mask; - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask; - RTE_PIPELINE_STATS_ADD_M( - table->n_pkts_dropped_by_lkp_hit_ah, mask); - } - - /* Table reserved actions */ - rte_pipeline_compute_masks(p, lookup_hit_mask); - p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= - p->action_mask1[ - RTE_PIPELINE_ACTION_DROP]; - p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= - p->action_mask1[ - RTE_PIPELINE_ACTION_PORT]; - p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= - p->action_mask1[ - RTE_PIPELINE_ACTION_PORT_META]; - p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= - p->action_mask1[ - RTE_PIPELINE_ACTION_TABLE]; - - RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_hit, - p->action_mask1[RTE_PIPELINE_ACTION_DROP]); + /* Table */ + for (table_id = port_in->table_id; p->pkts_mask != 0; ) { + struct rte_table *table; + uint64_t lookup_hit_mask, lookup_miss_mask; + + /* Lookup */ + table = &p->tables[table_id]; + table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask, + &lookup_hit_mask, (void **) p->entries); + lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask); + + /* Lookup miss */ + if (lookup_miss_mask != 0) { + struct rte_pipeline_table_entry *default_entry = + table->default_entry; + + p->pkts_mask = lookup_miss_mask; + + /* Table user actions */ + if (table->f_action_miss != NULL) { + table->f_action_miss(p, + p->pkts, + lookup_miss_mask, + default_entry, + table->arg_ah); + + RTE_PIPELINE_STATS_AH_DROP_READ(p, + table->n_pkts_dropped_by_lkp_miss_ah); } - /* Prepare for next iteration */ - pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; - table_id = table->table_next_id; - p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; + /* Table reserved actions */ + if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) && + (p->pkts_mask != 0)) + rte_pipeline_action_handler_port_bulk(p, + p->pkts_mask, + default_entry->port_id); + else { + uint32_t pos = default_entry->action; + + RTE_PIPELINE_STATS_TABLE_DROP0(p); + + p->action_mask0[pos] |= p->pkts_mask; + + RTE_PIPELINE_STATS_TABLE_DROP1(p, + table->n_pkts_dropped_lkp_miss); + } } - /* Table reserved action PORT */ - rte_pipeline_action_handler_port(p, - p->action_mask0[RTE_PIPELINE_ACTION_PORT]); + /* Lookup hit */ + if (lookup_hit_mask != 0) { + p->pkts_mask = lookup_hit_mask; + + /* Table user actions */ + if (table->f_action_hit != NULL) { + table->f_action_hit(p, + p->pkts, + lookup_hit_mask, + p->entries, + table->arg_ah); - /* Table reserved action PORT META */ - rte_pipeline_action_handler_port_meta(p, - p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); + RTE_PIPELINE_STATS_AH_DROP_READ(p, + table->n_pkts_dropped_by_lkp_hit_ah); + } + + /* Table reserved actions */ + RTE_PIPELINE_STATS_TABLE_DROP0(p); + rte_pipeline_compute_masks(p, p->pkts_mask); + p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= + p->action_mask1[ + RTE_PIPELINE_ACTION_DROP]; + p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= + p->action_mask1[ + RTE_PIPELINE_ACTION_PORT]; + p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= + p->action_mask1[ + RTE_PIPELINE_ACTION_PORT_META]; + p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= + p->action_mask1[ + RTE_PIPELINE_ACTION_TABLE]; + + RTE_PIPELINE_STATS_TABLE_DROP1(p, + table->n_pkts_dropped_lkp_hit); + } - /* Table reserved action DROP */ - rte_pipeline_action_handler_drop(p, - p->action_mask0[RTE_PIPELINE_ACTION_DROP]); + /* Prepare for next iteration */ + p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; + table_id = table->table_next_id; + p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; } - return 0; + /* Table reserved action PORT */ + rte_pipeline_action_handler_port(p, + p->action_mask0[RTE_PIPELINE_ACTION_PORT]); + + /* Table reserved action PORT META */ + rte_pipeline_action_handler_port_meta(p, + p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); + + /* Table reserved action DROP */ + rte_pipeline_action_handler_drop(p, + p->action_mask0[RTE_PIPELINE_ACTION_DROP]); + + /* Pick candidate for next port IN to serve */ + p->port_in_next = port_in->next; + + return (int) n_pkts; } int @@ -1498,29 +1502,35 @@ rte_pipeline_flush(struct rte_pipeline *p) int rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, - uint32_t port_id, struct rte_mbuf *pkt) + uint32_t port_id, struct rte_mbuf *pkt) { struct rte_port_out *port_out = &p->ports_out[port_id]; - /* Output port user actions */ - if (port_out->f_action == NULL) - port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ - else { - uint64_t pkt_mask = 1LLU; - - port_out->f_action(pkt, &pkt_mask, port_out->arg_ah); - - if (pkt_mask != 0) /* Output port TX */ - port_out->ops.f_tx(port_out->h_port, pkt); - else { - rte_pktmbuf_free(pkt); - RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, 1); - } - } + port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ + + return 0; +} + +int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, + uint64_t pkts_mask) +{ + pkts_mask &= p->pkts_mask; + p->pkts_mask &= ~pkts_mask; return 0; } +int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, + uint64_t pkts_mask) +{ + pkts_mask &= p->pkts_mask; + p->pkts_mask &= ~pkts_mask; + p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask; + + RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask); + return 0; +} + int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, struct rte_pipeline_port_in_stats *stats, int clear) { @@ -1636,4 +1646,3 @@ int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, return 0; } -