app/testpmd: port info prints dynamically mapped flow types
[dpdk.git] / lib / librte_pipeline / rte_pipeline.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <string.h>
35 #include <stdio.h>
36
37 #include <rte_common.h>
38 #include <rte_memory.h>
39 #include <rte_memzone.h>
40 #include <rte_cycles.h>
41 #include <rte_prefetch.h>
42 #include <rte_branch_prediction.h>
43 #include <rte_mbuf.h>
44 #include <rte_malloc.h>
45 #include <rte_string_fns.h>
46
47 #include "rte_pipeline.h"
48
49 #define RTE_TABLE_INVALID                                 UINT32_MAX
50
51 #ifdef RTE_PIPELINE_STATS_COLLECT
52
53 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)                       \
54         ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
55
56 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)                     \
57         ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
58
59 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)                               \
60         ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
61
62 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)                      \
63 ({                                                                      \
64         uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP];    \
65         mask ^= (p)->pkts_drop_mask;                                    \
66         (counter) += __builtin_popcountll(mask);                        \
67 })
68
69 #else
70
71 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
72 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
73 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
74 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
75
76 #endif
77
78 struct rte_port_in {
79         /* Input parameters */
80         struct rte_port_in_ops ops;
81         rte_pipeline_port_in_action_handler f_action;
82         void *arg_ah;
83         uint32_t burst_size;
84
85         /* The table to which this port is connected */
86         uint32_t table_id;
87
88         /* Handle to low-level port */
89         void *h_port;
90
91         /* List of enabled ports */
92         struct rte_port_in *next;
93
94         /* Statistics */
95         uint64_t n_pkts_dropped_by_ah;
96 };
97
98 struct rte_port_out {
99         /* Input parameters */
100         struct rte_port_out_ops ops;
101         rte_pipeline_port_out_action_handler f_action;
102         void *arg_ah;
103
104         /* Handle to low-level port */
105         void *h_port;
106
107         /* Statistics */
108         uint64_t n_pkts_dropped_by_ah;
109 };
110
111 struct rte_table {
112         /* Input parameters */
113         struct rte_table_ops ops;
114         rte_pipeline_table_action_handler_hit f_action_hit;
115         rte_pipeline_table_action_handler_miss f_action_miss;
116         void *arg_ah;
117         struct rte_pipeline_table_entry *default_entry;
118         uint32_t entry_size;
119
120         uint32_t table_next_id;
121         uint32_t table_next_id_valid;
122
123         /* Handle to the low-level table object */
124         void *h_table;
125
126         /* Statistics */
127         uint64_t n_pkts_dropped_by_lkp_hit_ah;
128         uint64_t n_pkts_dropped_by_lkp_miss_ah;
129         uint64_t n_pkts_dropped_lkp_hit;
130         uint64_t n_pkts_dropped_lkp_miss;
131 };
132
133 #define RTE_PIPELINE_MAX_NAME_SZ                           124
134
135 struct rte_pipeline {
136         /* Input parameters */
137         char name[RTE_PIPELINE_MAX_NAME_SZ];
138         int socket_id;
139         uint32_t offset_port_id;
140
141         /* Internal tables */
142         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
143         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
144         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
145
146         /* Occupancy of internal tables */
147         uint32_t num_ports_in;
148         uint32_t num_ports_out;
149         uint32_t num_tables;
150
151         /* List of enabled ports */
152         uint64_t enabled_port_in_mask;
153         struct rte_port_in *port_in_next;
154
155         /* Pipeline run structures */
156         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
157         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
158         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
159         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
160         uint64_t pkts_mask;
161         uint64_t n_pkts_ah_drop;
162         uint64_t pkts_drop_mask;
163 } __rte_cache_aligned;
164
165 static inline uint32_t
166 rte_mask_get_next(uint64_t mask, uint32_t pos)
167 {
168         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
169                         (mask >> ((pos + 1) & 0x3F));
170         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
171 }
172
173 static inline uint32_t
174 rte_mask_get_prev(uint64_t mask, uint32_t pos)
175 {
176         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
177                         (mask << ((64 - pos) & 0x3F));
178         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
179 }
180
181 static void
182 rte_pipeline_table_free(struct rte_table *table);
183
184 static void
185 rte_pipeline_port_in_free(struct rte_port_in *port);
186
187 static void
188 rte_pipeline_port_out_free(struct rte_port_out *port);
189
190 /*
191  * Pipeline
192  *
193  */
194 static int
195 rte_pipeline_check_params(struct rte_pipeline_params *params)
196 {
197         if (params == NULL) {
198                 RTE_LOG(ERR, PIPELINE,
199                         "%s: Incorrect value for parameter params\n", __func__);
200                 return -EINVAL;
201         }
202
203         /* name */
204         if (params->name == NULL) {
205                 RTE_LOG(ERR, PIPELINE,
206                         "%s: Incorrect value for parameter name\n", __func__);
207                 return -EINVAL;
208         }
209
210         /* socket */
211         if ((params->socket_id < 0) ||
212             (params->socket_id >= RTE_MAX_NUMA_NODES)) {
213                 RTE_LOG(ERR, PIPELINE,
214                         "%s: Incorrect value for parameter socket_id\n",
215                         __func__);
216                 return -EINVAL;
217         }
218
219         return 0;
220 }
221
222 struct rte_pipeline *
223 rte_pipeline_create(struct rte_pipeline_params *params)
224 {
225         struct rte_pipeline *p;
226         int status;
227
228         /* Check input parameters */
229         status = rte_pipeline_check_params(params);
230         if (status != 0) {
231                 RTE_LOG(ERR, PIPELINE,
232                         "%s: Pipeline params check failed (%d)\n",
233                         __func__, status);
234                 return NULL;
235         }
236
237         /* Allocate memory for the pipeline on requested socket */
238         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
239                         RTE_CACHE_LINE_SIZE, params->socket_id);
240
241         if (p == NULL) {
242                 RTE_LOG(ERR, PIPELINE,
243                         "%s: Pipeline memory allocation failed\n", __func__);
244                 return NULL;
245         }
246
247         /* Save input parameters */
248         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
249         p->socket_id = params->socket_id;
250         p->offset_port_id = params->offset_port_id;
251
252         /* Initialize pipeline internal data structure */
253         p->num_ports_in = 0;
254         p->num_ports_out = 0;
255         p->num_tables = 0;
256         p->enabled_port_in_mask = 0;
257         p->port_in_next = NULL;
258         p->pkts_mask = 0;
259         p->n_pkts_ah_drop = 0;
260
261         return p;
262 }
263
264 int
265 rte_pipeline_free(struct rte_pipeline *p)
266 {
267         uint32_t i;
268
269         /* Check input parameters */
270         if (p == NULL) {
271                 RTE_LOG(ERR, PIPELINE,
272                         "%s: rte_pipeline parameter is NULL\n", __func__);
273                 return -EINVAL;
274         }
275
276         /* Free input ports */
277         for (i = 0; i < p->num_ports_in; i++) {
278                 struct rte_port_in *port = &p->ports_in[i];
279
280                 rte_pipeline_port_in_free(port);
281         }
282
283         /* Free tables */
284         for (i = 0; i < p->num_tables; i++) {
285                 struct rte_table *table = &p->tables[i];
286
287                 rte_pipeline_table_free(table);
288         }
289
290         /* Free output ports */
291         for (i = 0; i < p->num_ports_out; i++) {
292                 struct rte_port_out *port = &p->ports_out[i];
293
294                 rte_pipeline_port_out_free(port);
295         }
296
297         /* Free pipeline memory */
298         rte_free(p);
299
300         return 0;
301 }
302
303 /*
304  * Table
305  *
306  */
307 static int
308 rte_table_check_params(struct rte_pipeline *p,
309                 struct rte_pipeline_table_params *params,
310                 uint32_t *table_id)
311 {
312         if (p == NULL) {
313                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
314                         __func__);
315                 return -EINVAL;
316         }
317         if (params == NULL) {
318                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
319                         __func__);
320                 return -EINVAL;
321         }
322         if (table_id == NULL) {
323                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
324                         __func__);
325                 return -EINVAL;
326         }
327
328         /* ops */
329         if (params->ops == NULL) {
330                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
331                         __func__);
332                 return -EINVAL;
333         }
334
335         if (params->ops->f_create == NULL) {
336                 RTE_LOG(ERR, PIPELINE,
337                         "%s: f_create function pointer is NULL\n", __func__);
338                 return -EINVAL;
339         }
340
341         if (params->ops->f_lookup == NULL) {
342                 RTE_LOG(ERR, PIPELINE,
343                         "%s: f_lookup function pointer is NULL\n", __func__);
344                 return -EINVAL;
345         }
346
347         /* De we have room for one more table? */
348         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
349                 RTE_LOG(ERR, PIPELINE,
350                         "%s: Incorrect value for num_tables parameter\n",
351                         __func__);
352                 return -EINVAL;
353         }
354
355         return 0;
356 }
357
358 int
359 rte_pipeline_table_create(struct rte_pipeline *p,
360                 struct rte_pipeline_table_params *params,
361                 uint32_t *table_id)
362 {
363         struct rte_table *table;
364         struct rte_pipeline_table_entry *default_entry;
365         void *h_table;
366         uint32_t entry_size, id;
367         int status;
368
369         /* Check input arguments */
370         status = rte_table_check_params(p, params, table_id);
371         if (status != 0)
372                 return status;
373
374         id = p->num_tables;
375         table = &p->tables[id];
376
377         /* Allocate space for the default table entry */
378         entry_size = sizeof(struct rte_pipeline_table_entry) +
379                 params->action_data_size;
380         default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
381                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
382         if (default_entry == NULL) {
383                 RTE_LOG(ERR, PIPELINE,
384                         "%s: Failed to allocate default entry\n", __func__);
385                 return -EINVAL;
386         }
387
388         /* Create the table */
389         h_table = params->ops->f_create(params->arg_create, p->socket_id,
390                 entry_size);
391         if (h_table == NULL) {
392                 rte_free(default_entry);
393                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
394                 return -EINVAL;
395         }
396
397         /* Commit current table to the pipeline */
398         p->num_tables++;
399         *table_id = id;
400
401         /* Save input parameters */
402         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
403         table->f_action_hit = params->f_action_hit;
404         table->f_action_miss = params->f_action_miss;
405         table->arg_ah = params->arg_ah;
406         table->entry_size = entry_size;
407
408         /* Clear the lookup miss actions (to be set later through API) */
409         table->default_entry = default_entry;
410         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
411
412         /* Initialize table internal data structure */
413         table->h_table = h_table;
414         table->table_next_id = 0;
415         table->table_next_id_valid = 0;
416
417         return 0;
418 }
419
420 void
421 rte_pipeline_table_free(struct rte_table *table)
422 {
423         if (table->ops.f_free != NULL)
424                 table->ops.f_free(table->h_table);
425
426         rte_free(table->default_entry);
427 }
428
429 int
430 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
431         uint32_t table_id,
432         struct rte_pipeline_table_entry *default_entry,
433         struct rte_pipeline_table_entry **default_entry_ptr)
434 {
435         struct rte_table *table;
436
437         /* Check input arguments */
438         if (p == NULL) {
439                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
440                         __func__);
441                 return -EINVAL;
442         }
443
444         if (default_entry == NULL) {
445                 RTE_LOG(ERR, PIPELINE,
446                         "%s: default_entry parameter is NULL\n", __func__);
447                 return -EINVAL;
448         }
449
450         if (table_id >= p->num_tables) {
451                 RTE_LOG(ERR, PIPELINE,
452                         "%s: table_id %d out of range\n", __func__, table_id);
453                 return -EINVAL;
454         }
455
456         table = &p->tables[table_id];
457
458         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
459                 table->table_next_id_valid &&
460                 (default_entry->table_id != table->table_next_id)) {
461                 RTE_LOG(ERR, PIPELINE,
462                         "%s: Tree-like topologies not allowed\n", __func__);
463                 return -EINVAL;
464         }
465
466         /* Set the lookup miss actions */
467         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
468                 (table->table_next_id_valid == 0)) {
469                 table->table_next_id = default_entry->table_id;
470                 table->table_next_id_valid = 1;
471         }
472
473         memcpy(table->default_entry, default_entry, table->entry_size);
474
475         *default_entry_ptr = table->default_entry;
476         return 0;
477 }
478
479 int
480 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
481                 uint32_t table_id,
482                 struct rte_pipeline_table_entry *entry)
483 {
484         struct rte_table *table;
485
486         /* Check input arguments */
487         if (p == NULL) {
488                 RTE_LOG(ERR, PIPELINE,
489                         "%s: pipeline parameter is NULL\n", __func__);
490                 return -EINVAL;
491         }
492
493         if (table_id >= p->num_tables) {
494                 RTE_LOG(ERR, PIPELINE,
495                         "%s: table_id %d out of range\n", __func__, table_id);
496                 return -EINVAL;
497         }
498
499         table = &p->tables[table_id];
500
501         /* Save the current contents of the default entry */
502         if (entry)
503                 memcpy(entry, table->default_entry, table->entry_size);
504
505         /* Clear the lookup miss actions */
506         memset(table->default_entry, 0, table->entry_size);
507         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
508
509         return 0;
510 }
511
512 int
513 rte_pipeline_table_entry_add(struct rte_pipeline *p,
514                 uint32_t table_id,
515                 void *key,
516                 struct rte_pipeline_table_entry *entry,
517                 int *key_found,
518                 struct rte_pipeline_table_entry **entry_ptr)
519 {
520         struct rte_table *table;
521
522         /* Check input arguments */
523         if (p == NULL) {
524                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
525                         __func__);
526                 return -EINVAL;
527         }
528
529         if (key == NULL) {
530                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
531                 return -EINVAL;
532         }
533
534         if (entry == NULL) {
535                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
536                         __func__);
537                 return -EINVAL;
538         }
539
540         if (table_id >= p->num_tables) {
541                 RTE_LOG(ERR, PIPELINE,
542                         "%s: table_id %d out of range\n", __func__, table_id);
543                 return -EINVAL;
544         }
545
546         table = &p->tables[table_id];
547
548         if (table->ops.f_add == NULL) {
549                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
550                         __func__);
551                 return -EINVAL;
552         }
553
554         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
555                 table->table_next_id_valid &&
556                 (entry->table_id != table->table_next_id)) {
557                 RTE_LOG(ERR, PIPELINE,
558                         "%s: Tree-like topologies not allowed\n", __func__);
559                 return -EINVAL;
560         }
561
562         /* Add entry */
563         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
564                 (table->table_next_id_valid == 0)) {
565                 table->table_next_id = entry->table_id;
566                 table->table_next_id_valid = 1;
567         }
568
569         return (table->ops.f_add)(table->h_table, key, (void *) entry,
570                 key_found, (void **) entry_ptr);
571 }
572
573 int
574 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
575                 uint32_t table_id,
576                 void *key,
577                 int *key_found,
578                 struct rte_pipeline_table_entry *entry)
579 {
580         struct rte_table *table;
581
582         /* Check input arguments */
583         if (p == NULL) {
584                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
585                         __func__);
586                 return -EINVAL;
587         }
588
589         if (key == NULL) {
590                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
591                         __func__);
592                 return -EINVAL;
593         }
594
595         if (table_id >= p->num_tables) {
596                 RTE_LOG(ERR, PIPELINE,
597                         "%s: table_id %d out of range\n", __func__, table_id);
598                 return -EINVAL;
599         }
600
601         table = &p->tables[table_id];
602
603         if (table->ops.f_delete == NULL) {
604                 RTE_LOG(ERR, PIPELINE,
605                         "%s: f_delete function pointer NULL\n", __func__);
606                 return -EINVAL;
607         }
608
609         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
610 }
611
612 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
613         uint32_t table_id,
614         void **keys,
615         struct rte_pipeline_table_entry **entries,
616         uint32_t n_keys,
617         int *key_found,
618         struct rte_pipeline_table_entry **entries_ptr)
619 {
620         struct rte_table *table;
621         uint32_t i;
622
623         /* Check input arguments */
624         if (p == NULL) {
625                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
626                         __func__);
627                 return -EINVAL;
628         }
629
630         if (keys == NULL) {
631                 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
632                 return -EINVAL;
633         }
634
635         if (entries == NULL) {
636                 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
637                         __func__);
638                 return -EINVAL;
639         }
640
641         if (table_id >= p->num_tables) {
642                 RTE_LOG(ERR, PIPELINE,
643                         "%s: table_id %d out of range\n", __func__, table_id);
644                 return -EINVAL;
645         }
646
647         table = &p->tables[table_id];
648
649         if (table->ops.f_add_bulk == NULL) {
650                 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
651                         __func__);
652                 return -EINVAL;
653         }
654
655         for (i = 0; i < n_keys; i++) {
656                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
657                         table->table_next_id_valid &&
658                         (entries[i]->table_id != table->table_next_id)) {
659                         RTE_LOG(ERR, PIPELINE,
660                                 "%s: Tree-like topologies not allowed\n", __func__);
661                         return -EINVAL;
662                 }
663         }
664
665         /* Add entry */
666         for (i = 0; i < n_keys; i++) {
667                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
668                         (table->table_next_id_valid == 0)) {
669                         table->table_next_id = entries[i]->table_id;
670                         table->table_next_id_valid = 1;
671                 }
672         }
673
674         return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
675                 n_keys, key_found, (void **) entries_ptr);
676 }
677
678 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
679         uint32_t table_id,
680         void **keys,
681         uint32_t n_keys,
682         int *key_found,
683         struct rte_pipeline_table_entry **entries)
684 {
685         struct rte_table *table;
686
687         /* Check input arguments */
688         if (p == NULL) {
689                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
690                         __func__);
691                 return -EINVAL;
692         }
693
694         if (keys == NULL) {
695                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
696                         __func__);
697                 return -EINVAL;
698         }
699
700         if (table_id >= p->num_tables) {
701                 RTE_LOG(ERR, PIPELINE,
702                         "%s: table_id %d out of range\n", __func__, table_id);
703                 return -EINVAL;
704         }
705
706         table = &p->tables[table_id];
707
708         if (table->ops.f_delete_bulk == NULL) {
709                 RTE_LOG(ERR, PIPELINE,
710                         "%s: f_delete function pointer NULL\n", __func__);
711                 return -EINVAL;
712         }
713
714         return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
715                         (void **) entries);
716 }
717
718 /*
719  * Port
720  *
721  */
722 static int
723 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
724                 struct rte_pipeline_port_in_params *params,
725                 uint32_t *port_id)
726 {
727         if (p == NULL) {
728                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
729                         __func__);
730                 return -EINVAL;
731         }
732         if (params == NULL) {
733                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
734                 return -EINVAL;
735         }
736         if (port_id == NULL) {
737                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
738                         __func__);
739                 return -EINVAL;
740         }
741
742         /* ops */
743         if (params->ops == NULL) {
744                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
745                         __func__);
746                 return -EINVAL;
747         }
748
749         if (params->ops->f_create == NULL) {
750                 RTE_LOG(ERR, PIPELINE,
751                         "%s: f_create function pointer NULL\n", __func__);
752                 return -EINVAL;
753         }
754
755         if (params->ops->f_rx == NULL) {
756                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
757                         __func__);
758                 return -EINVAL;
759         }
760
761         /* burst_size */
762         if ((params->burst_size == 0) ||
763                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
764                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
765                         __func__);
766                 return -EINVAL;
767         }
768
769         /* Do we have room for one more port? */
770         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
771                 RTE_LOG(ERR, PIPELINE,
772                         "%s: invalid value for num_ports_in\n", __func__);
773                 return -EINVAL;
774         }
775
776         return 0;
777 }
778
779 static int
780 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
781                 struct rte_pipeline_port_out_params *params,
782                 uint32_t *port_id)
783 {
784         if (p == NULL) {
785                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
786                         __func__);
787                 return -EINVAL;
788         }
789
790         if (params == NULL) {
791                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
792                 return -EINVAL;
793         }
794
795         if (port_id == NULL) {
796                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
797                         __func__);
798                 return -EINVAL;
799         }
800
801         /* ops */
802         if (params->ops == NULL) {
803                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
804                         __func__);
805                 return -EINVAL;
806         }
807
808         if (params->ops->f_create == NULL) {
809                 RTE_LOG(ERR, PIPELINE,
810                         "%s: f_create function pointer NULL\n", __func__);
811                 return -EINVAL;
812         }
813
814         if (params->ops->f_tx == NULL) {
815                 RTE_LOG(ERR, PIPELINE,
816                         "%s: f_tx function pointer NULL\n", __func__);
817                 return -EINVAL;
818         }
819
820         if (params->ops->f_tx_bulk == NULL) {
821                 RTE_LOG(ERR, PIPELINE,
822                         "%s: f_tx_bulk function pointer NULL\n", __func__);
823                 return -EINVAL;
824         }
825
826         /* Do we have room for one more port? */
827         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
828                 RTE_LOG(ERR, PIPELINE,
829                         "%s: invalid value for num_ports_out\n", __func__);
830                 return -EINVAL;
831         }
832
833         return 0;
834 }
835
836 int
837 rte_pipeline_port_in_create(struct rte_pipeline *p,
838                 struct rte_pipeline_port_in_params *params,
839                 uint32_t *port_id)
840 {
841         struct rte_port_in *port;
842         void *h_port;
843         uint32_t id;
844         int status;
845
846         /* Check input arguments */
847         status = rte_pipeline_port_in_check_params(p, params, port_id);
848         if (status != 0)
849                 return status;
850
851         id = p->num_ports_in;
852         port = &p->ports_in[id];
853
854         /* Create the port */
855         h_port = params->ops->f_create(params->arg_create, p->socket_id);
856         if (h_port == NULL) {
857                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
858                 return -EINVAL;
859         }
860
861         /* Commit current table to the pipeline */
862         p->num_ports_in++;
863         *port_id = id;
864
865         /* Save input parameters */
866         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
867         port->f_action = params->f_action;
868         port->arg_ah = params->arg_ah;
869         port->burst_size = params->burst_size;
870
871         /* Initialize port internal data structure */
872         port->table_id = RTE_TABLE_INVALID;
873         port->h_port = h_port;
874         port->next = NULL;
875
876         return 0;
877 }
878
879 void
880 rte_pipeline_port_in_free(struct rte_port_in *port)
881 {
882         if (port->ops.f_free != NULL)
883                 port->ops.f_free(port->h_port);
884 }
885
886 int
887 rte_pipeline_port_out_create(struct rte_pipeline *p,
888                 struct rte_pipeline_port_out_params *params,
889                 uint32_t *port_id)
890 {
891         struct rte_port_out *port;
892         void *h_port;
893         uint32_t id;
894         int status;
895
896         /* Check input arguments */
897         status = rte_pipeline_port_out_check_params(p, params, port_id);
898         if (status != 0)
899                 return status;
900
901         id = p->num_ports_out;
902         port = &p->ports_out[id];
903
904         /* Create the port */
905         h_port = params->ops->f_create(params->arg_create, p->socket_id);
906         if (h_port == NULL) {
907                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
908                 return -EINVAL;
909         }
910
911         /* Commit current table to the pipeline */
912         p->num_ports_out++;
913         *port_id = id;
914
915         /* Save input parameters */
916         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
917         port->f_action = params->f_action;
918         port->arg_ah = params->arg_ah;
919
920         /* Initialize port internal data structure */
921         port->h_port = h_port;
922
923         return 0;
924 }
925
926 void
927 rte_pipeline_port_out_free(struct rte_port_out *port)
928 {
929         if (port->ops.f_free != NULL)
930                 port->ops.f_free(port->h_port);
931 }
932
933 int
934 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
935                 uint32_t port_id,
936                 uint32_t table_id)
937 {
938         struct rte_port_in *port;
939
940         /* Check input arguments */
941         if (p == NULL) {
942                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
943                         __func__);
944                 return -EINVAL;
945         }
946
947         if (port_id >= p->num_ports_in) {
948                 RTE_LOG(ERR, PIPELINE,
949                         "%s: port IN ID %u is out of range\n",
950                         __func__, port_id);
951                 return -EINVAL;
952         }
953
954         if (table_id >= p->num_tables) {
955                 RTE_LOG(ERR, PIPELINE,
956                         "%s: Table ID %u is out of range\n",
957                         __func__, table_id);
958                 return -EINVAL;
959         }
960
961         port = &p->ports_in[port_id];
962         port->table_id = table_id;
963
964         return 0;
965 }
966
967 int
968 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
969 {
970         struct rte_port_in *port, *port_prev, *port_next;
971         uint64_t port_mask;
972         uint32_t port_prev_id, port_next_id;
973
974         /* Check input arguments */
975         if (p == NULL) {
976                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
977                         __func__);
978                 return -EINVAL;
979         }
980
981         if (port_id >= p->num_ports_in) {
982                 RTE_LOG(ERR, PIPELINE,
983                         "%s: port IN ID %u is out of range\n",
984                         __func__, port_id);
985                 return -EINVAL;
986         }
987
988         port = &p->ports_in[port_id];
989
990         /* Return if current input port is already enabled */
991         port_mask = 1LLU << port_id;
992         if (p->enabled_port_in_mask & port_mask)
993                 return 0;
994
995         p->enabled_port_in_mask |= port_mask;
996
997         /* Add current input port to the pipeline chain of enabled ports */
998         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
999         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1000
1001         port_prev = &p->ports_in[port_prev_id];
1002         port_next = &p->ports_in[port_next_id];
1003
1004         port_prev->next = port;
1005         port->next = port_next;
1006
1007         /* Check if list of enabled ports was previously empty */
1008         if (p->enabled_port_in_mask == port_mask)
1009                 p->port_in_next = port;
1010
1011         return 0;
1012 }
1013
1014 int
1015 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
1016 {
1017         struct rte_port_in *port, *port_prev, *port_next;
1018         uint64_t port_mask;
1019         uint32_t port_prev_id, port_next_id;
1020
1021         /* Check input arguments */
1022         if (p == NULL) {
1023                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1024                 __func__);
1025                 return -EINVAL;
1026         }
1027
1028         if (port_id >= p->num_ports_in) {
1029                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
1030                         __func__, port_id);
1031                 return -EINVAL;
1032         }
1033
1034         port = &p->ports_in[port_id];
1035
1036         /* Return if current input port is already disabled */
1037         port_mask = 1LLU << port_id;
1038         if ((p->enabled_port_in_mask & port_mask) == 0)
1039                 return 0;
1040
1041         p->enabled_port_in_mask &= ~port_mask;
1042
1043         /* Return if no other enabled ports */
1044         if (p->enabled_port_in_mask == 0) {
1045                 p->port_in_next = NULL;
1046
1047                 return 0;
1048         }
1049
1050         /* Add current input port to the pipeline chain of enabled ports */
1051         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1052         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1053
1054         port_prev = &p->ports_in[port_prev_id];
1055         port_next = &p->ports_in[port_next_id];
1056
1057         port_prev->next = port_next;
1058
1059         /* Check if the port which has just been disabled is next to serve */
1060         if (port == p->port_in_next)
1061                 p->port_in_next = port_next;
1062
1063         return 0;
1064 }
1065
1066 /*
1067  * Pipeline run-time
1068  *
1069  */
1070 int
1071 rte_pipeline_check(struct rte_pipeline *p)
1072 {
1073         uint32_t port_in_id;
1074
1075         /* Check input arguments */
1076         if (p == NULL) {
1077                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1078                         __func__);
1079                 return -EINVAL;
1080         }
1081
1082         /* Check that pipeline has at least one input port, one table and one
1083         output port */
1084         if (p->num_ports_in == 0) {
1085                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1086                         __func__);
1087                 return -EINVAL;
1088         }
1089         if (p->num_tables == 0) {
1090                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1091                         __func__);
1092                 return -EINVAL;
1093         }
1094         if (p->num_ports_out == 0) {
1095                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1096                         __func__);
1097                 return -EINVAL;
1098         }
1099
1100         /* Check that all input ports are connected */
1101         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1102                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1103
1104                 if (port_in->table_id == RTE_TABLE_INVALID) {
1105                         RTE_LOG(ERR, PIPELINE,
1106                                 "%s: Port IN ID %u is not connected\n",
1107                                 __func__, port_in_id);
1108                         return -EINVAL;
1109                 }
1110         }
1111
1112         return 0;
1113 }
1114
1115 static inline void
1116 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1117 {
1118         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1119         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1120         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1121         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1122
1123         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1124                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1125                 uint32_t i;
1126
1127                 for (i = 0; i < n_pkts; i++) {
1128                         uint64_t pkt_mask = 1LLU << i;
1129                         uint32_t pos = p->entries[i]->action;
1130
1131                         p->action_mask1[pos] |= pkt_mask;
1132                 }
1133         } else {
1134                 uint32_t i;
1135
1136                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1137                         uint64_t pkt_mask = 1LLU << i;
1138                         uint32_t pos;
1139
1140                         if ((pkt_mask & pkts_mask) == 0)
1141                                 continue;
1142
1143                         pos = p->entries[i]->action;
1144                         p->action_mask1[pos] |= pkt_mask;
1145                 }
1146         }
1147 }
1148
1149 static inline void
1150 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1151         uint64_t pkts_mask, uint32_t port_id)
1152 {
1153         struct rte_port_out *port_out = &p->ports_out[port_id];
1154
1155         p->pkts_mask = pkts_mask;
1156
1157         /* Output port user actions */
1158         if (port_out->f_action != NULL) {
1159                 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1160
1161                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1162                         port_out->n_pkts_dropped_by_ah);
1163         }
1164
1165         /* Output port TX */
1166         if (p->pkts_mask != 0)
1167                 port_out->ops.f_tx_bulk(port_out->h_port,
1168                         p->pkts,
1169                         p->pkts_mask);
1170 }
1171
1172 static inline void
1173 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1174 {
1175         p->pkts_mask = pkts_mask;
1176
1177         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1178                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1179                 uint32_t i;
1180
1181                 for (i = 0; i < n_pkts; i++) {
1182                         struct rte_mbuf *pkt = p->pkts[i];
1183                         uint32_t port_out_id = p->entries[i]->port_id;
1184                         struct rte_port_out *port_out =
1185                                 &p->ports_out[port_out_id];
1186
1187                         /* Output port user actions */
1188                         if (port_out->f_action == NULL) /* Output port TX */
1189                                 port_out->ops.f_tx(port_out->h_port, pkt);
1190                         else {
1191                                 uint64_t pkt_mask = 1LLU << i;
1192
1193                                 port_out->f_action(p,
1194                                         p->pkts,
1195                                         pkt_mask,
1196                                         port_out->arg_ah);
1197
1198                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1199                                         port_out->n_pkts_dropped_by_ah);
1200
1201                                 /* Output port TX */
1202                                 if (pkt_mask & p->pkts_mask)
1203                                         port_out->ops.f_tx(port_out->h_port,
1204                                                 pkt);
1205                         }
1206                 }
1207         } else {
1208                 uint32_t i;
1209
1210                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1211                         uint64_t pkt_mask = 1LLU << i;
1212                         struct rte_mbuf *pkt;
1213                         struct rte_port_out *port_out;
1214                         uint32_t port_out_id;
1215
1216                         if ((pkt_mask & pkts_mask) == 0)
1217                                 continue;
1218
1219                         pkt = p->pkts[i];
1220                         port_out_id = p->entries[i]->port_id;
1221                         port_out = &p->ports_out[port_out_id];
1222
1223                         /* Output port user actions */
1224                         if (port_out->f_action == NULL) /* Output port TX */
1225                                 port_out->ops.f_tx(port_out->h_port, pkt);
1226                         else {
1227                                 port_out->f_action(p,
1228                                         p->pkts,
1229                                         pkt_mask,
1230                                         port_out->arg_ah);
1231
1232                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1233                                         port_out->n_pkts_dropped_by_ah);
1234
1235                                 /* Output port TX */
1236                                 if (pkt_mask & p->pkts_mask)
1237                                         port_out->ops.f_tx(port_out->h_port,
1238                                                 pkt);
1239                         }
1240                 }
1241         }
1242 }
1243
1244 static inline void
1245 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1246         uint64_t pkts_mask)
1247 {
1248         p->pkts_mask = pkts_mask;
1249
1250         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1251                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1252                 uint32_t i;
1253
1254                 for (i = 0; i < n_pkts; i++) {
1255                         struct rte_mbuf *pkt = p->pkts[i];
1256                         uint32_t port_out_id =
1257                                 RTE_MBUF_METADATA_UINT32(pkt,
1258                                         p->offset_port_id);
1259                         struct rte_port_out *port_out = &p->ports_out[
1260                                 port_out_id];
1261
1262                         /* Output port user actions */
1263                         if (port_out->f_action == NULL) /* Output port TX */
1264                                 port_out->ops.f_tx(port_out->h_port, pkt);
1265                         else {
1266                                 uint64_t pkt_mask = 1LLU << i;
1267
1268                                 port_out->f_action(p,
1269                                         p->pkts,
1270                                         pkt_mask,
1271                                         port_out->arg_ah);
1272
1273                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1274                                         port_out->n_pkts_dropped_by_ah);
1275
1276                                 /* Output port TX */
1277                                 if (pkt_mask & p->pkts_mask)
1278                                         port_out->ops.f_tx(port_out->h_port,
1279                                                 pkt);
1280                         }
1281                 }
1282         } else {
1283                 uint32_t i;
1284
1285                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1286                         uint64_t pkt_mask = 1LLU << i;
1287                         struct rte_mbuf *pkt;
1288                         struct rte_port_out *port_out;
1289                         uint32_t port_out_id;
1290
1291                         if ((pkt_mask & pkts_mask) == 0)
1292                                 continue;
1293
1294                         pkt = p->pkts[i];
1295                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1296                                 p->offset_port_id);
1297                         port_out = &p->ports_out[port_out_id];
1298
1299                         /* Output port user actions */
1300                         if (port_out->f_action == NULL) /* Output port TX */
1301                                 port_out->ops.f_tx(port_out->h_port, pkt);
1302                         else {
1303                                 port_out->f_action(p,
1304                                         p->pkts,
1305                                         pkt_mask,
1306                                         port_out->arg_ah);
1307
1308                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1309                                         port_out->n_pkts_dropped_by_ah);
1310
1311                                 /* Output port TX */
1312                                 if (pkt_mask & p->pkts_mask)
1313                                         port_out->ops.f_tx(port_out->h_port,
1314                                                 pkt);
1315                         }
1316                 }
1317         }
1318 }
1319
1320 static inline void
1321 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1322 {
1323         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1324                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1325                 uint32_t i;
1326
1327                 for (i = 0; i < n_pkts; i++)
1328                         rte_pktmbuf_free(p->pkts[i]);
1329         } else {
1330                 uint32_t i;
1331
1332                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1333                         uint64_t pkt_mask = 1LLU << i;
1334
1335                         if ((pkt_mask & pkts_mask) == 0)
1336                                 continue;
1337
1338                         rte_pktmbuf_free(p->pkts[i]);
1339                 }
1340         }
1341 }
1342
1343 int
1344 rte_pipeline_run(struct rte_pipeline *p)
1345 {
1346         struct rte_port_in *port_in = p->port_in_next;
1347         uint32_t n_pkts, table_id;
1348
1349         if (port_in == NULL)
1350                 return 0;
1351
1352         /* Input port RX */
1353         n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1354                 port_in->burst_size);
1355         if (n_pkts == 0) {
1356                 p->port_in_next = port_in->next;
1357                 return 0;
1358         }
1359
1360         p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1361         p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1362         p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1363         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1364         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1365
1366         /* Input port user actions */
1367         if (port_in->f_action != NULL) {
1368                 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1369
1370                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1371                         port_in->n_pkts_dropped_by_ah);
1372         }
1373
1374         /* Table */
1375         for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1376                 struct rte_table *table;
1377                 uint64_t lookup_hit_mask, lookup_miss_mask;
1378
1379                 /* Lookup */
1380                 table = &p->tables[table_id];
1381                 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1382                         &lookup_hit_mask, (void **) p->entries);
1383                 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1384
1385                 /* Lookup miss */
1386                 if (lookup_miss_mask != 0) {
1387                         struct rte_pipeline_table_entry *default_entry =
1388                                 table->default_entry;
1389
1390                         p->pkts_mask = lookup_miss_mask;
1391
1392                         /* Table user actions */
1393                         if (table->f_action_miss != NULL) {
1394                                 table->f_action_miss(p,
1395                                         p->pkts,
1396                                         lookup_miss_mask,
1397                                         default_entry,
1398                                         table->arg_ah);
1399
1400                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1401                                         table->n_pkts_dropped_by_lkp_miss_ah);
1402                         }
1403
1404                         /* Table reserved actions */
1405                         if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1406                                 (p->pkts_mask != 0))
1407                                 rte_pipeline_action_handler_port_bulk(p,
1408                                         p->pkts_mask,
1409                                         default_entry->port_id);
1410                         else {
1411                                 uint32_t pos = default_entry->action;
1412
1413                                 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1414
1415                                 p->action_mask0[pos] |= p->pkts_mask;
1416
1417                                 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1418                                         table->n_pkts_dropped_lkp_miss);
1419                         }
1420                 }
1421
1422                 /* Lookup hit */
1423                 if (lookup_hit_mask != 0) {
1424                         p->pkts_mask = lookup_hit_mask;
1425
1426                         /* Table user actions */
1427                         if (table->f_action_hit != NULL) {
1428                                 table->f_action_hit(p,
1429                                         p->pkts,
1430                                         lookup_hit_mask,
1431                                         p->entries,
1432                                         table->arg_ah);
1433
1434                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1435                                         table->n_pkts_dropped_by_lkp_hit_ah);
1436                         }
1437
1438                         /* Table reserved actions */
1439                         RTE_PIPELINE_STATS_TABLE_DROP0(p);
1440                         rte_pipeline_compute_masks(p, p->pkts_mask);
1441                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1442                                 p->action_mask1[
1443                                         RTE_PIPELINE_ACTION_DROP];
1444                         p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1445                                 p->action_mask1[
1446                                         RTE_PIPELINE_ACTION_PORT];
1447                         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1448                                 p->action_mask1[
1449                                         RTE_PIPELINE_ACTION_PORT_META];
1450                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1451                                 p->action_mask1[
1452                                         RTE_PIPELINE_ACTION_TABLE];
1453
1454                         RTE_PIPELINE_STATS_TABLE_DROP1(p,
1455                                 table->n_pkts_dropped_lkp_hit);
1456                 }
1457
1458                 /* Prepare for next iteration */
1459                 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1460                 table_id = table->table_next_id;
1461                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1462         }
1463
1464         /* Table reserved action PORT */
1465         rte_pipeline_action_handler_port(p,
1466                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1467
1468         /* Table reserved action PORT META */
1469         rte_pipeline_action_handler_port_meta(p,
1470                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1471
1472         /* Table reserved action DROP */
1473         rte_pipeline_action_handler_drop(p,
1474                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1475
1476         /* Pick candidate for next port IN to serve */
1477         p->port_in_next = port_in->next;
1478
1479         return (int) n_pkts;
1480 }
1481
1482 int
1483 rte_pipeline_flush(struct rte_pipeline *p)
1484 {
1485         uint32_t port_id;
1486
1487         /* Check input arguments */
1488         if (p == NULL) {
1489                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1490                         __func__);
1491                 return -EINVAL;
1492         }
1493
1494         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1495                 struct rte_port_out *port = &p->ports_out[port_id];
1496
1497                 if (port->ops.f_flush != NULL)
1498                         port->ops.f_flush(port->h_port);
1499         }
1500
1501         return 0;
1502 }
1503
1504 int
1505 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1506         uint32_t port_id, struct rte_mbuf *pkt)
1507 {
1508         struct rte_port_out *port_out = &p->ports_out[port_id];
1509
1510         port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1511
1512         return 0;
1513 }
1514
1515 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1516         uint64_t pkts_mask)
1517 {
1518         pkts_mask &= p->pkts_mask;
1519         p->pkts_mask &= ~pkts_mask;
1520
1521         return 0;
1522 }
1523
1524 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1525         uint64_t pkts_mask)
1526 {
1527         pkts_mask &= p->pkts_mask;
1528         p->pkts_mask &= ~pkts_mask;
1529         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1530
1531         RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1532         return 0;
1533 }
1534
1535 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1536         struct rte_pipeline_port_in_stats *stats, int clear)
1537 {
1538         struct rte_port_in *port;
1539         int retval;
1540
1541         if (p == NULL) {
1542                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1543                         __func__);
1544                 return -EINVAL;
1545         }
1546
1547         if (port_id >= p->num_ports_in) {
1548                 RTE_LOG(ERR, PIPELINE,
1549                         "%s: port IN ID %u is out of range\n",
1550                         __func__, port_id);
1551                 return -EINVAL;
1552         }
1553
1554         port = &p->ports_in[port_id];
1555
1556         if (port->ops.f_stats != NULL) {
1557                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1558                 if (retval)
1559                         return retval;
1560         } else if (stats != NULL)
1561                 memset(&stats->stats, 0, sizeof(stats->stats));
1562
1563         if (stats != NULL)
1564                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1565
1566         if (clear != 0)
1567                 port->n_pkts_dropped_by_ah = 0;
1568
1569         return 0;
1570 }
1571
1572 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1573         struct rte_pipeline_port_out_stats *stats, int clear)
1574 {
1575         struct rte_port_out *port;
1576         int retval;
1577
1578         if (p == NULL) {
1579                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1580                 return -EINVAL;
1581         }
1582
1583         if (port_id >= p->num_ports_out) {
1584                 RTE_LOG(ERR, PIPELINE,
1585                         "%s: port OUT ID %u is out of range\n", __func__, port_id);
1586                 return -EINVAL;
1587         }
1588
1589         port = &p->ports_out[port_id];
1590         if (port->ops.f_stats != NULL) {
1591                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1592                 if (retval != 0)
1593                         return retval;
1594         } else if (stats != NULL)
1595                 memset(&stats->stats, 0, sizeof(stats->stats));
1596
1597         if (stats != NULL)
1598                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1599
1600         if (clear != 0)
1601                 port->n_pkts_dropped_by_ah = 0;
1602
1603         return 0;
1604 }
1605
1606 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1607         struct rte_pipeline_table_stats *stats, int clear)
1608 {
1609         struct rte_table *table;
1610         int retval;
1611
1612         if (p == NULL) {
1613                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1614                         __func__);
1615                 return -EINVAL;
1616         }
1617
1618         if (table_id >= p->num_tables) {
1619                 RTE_LOG(ERR, PIPELINE,
1620                                 "%s: table %u is out of range\n", __func__, table_id);
1621                 return -EINVAL;
1622         }
1623
1624         table = &p->tables[table_id];
1625         if (table->ops.f_stats != NULL) {
1626                 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1627                 if (retval != 0)
1628                         return retval;
1629         } else if (stats != NULL)
1630                 memset(&stats->stats, 0, sizeof(stats->stats));
1631
1632         if (stats != NULL) {
1633                 stats->n_pkts_dropped_by_lkp_hit_ah =
1634                         table->n_pkts_dropped_by_lkp_hit_ah;
1635                 stats->n_pkts_dropped_by_lkp_miss_ah =
1636                         table->n_pkts_dropped_by_lkp_miss_ah;
1637                 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1638                 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1639         }
1640
1641         if (clear != 0) {
1642                 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1643                 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1644                 table->n_pkts_dropped_lkp_hit = 0;
1645                 table->n_pkts_dropped_lkp_miss = 0;
1646         }
1647
1648         return 0;
1649 }