lib/librte_pipeline/rte_pipeline.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_memory.h>
  39 #include <rte_memzone.h>
  40 #include <rte_cycles.h>
  41 #include <rte_prefetch.h>
  42 #include <rte_branch_prediction.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_malloc.h>
  45 #include <rte_string_fns.h>
  46
  47 #include "rte_pipeline.h"
  48
  49 #define RTE_TABLE_INVALID                                 UINT32_MAX
  50
  51 struct rte_port_in {
  52         /* Input parameters */
  53         struct rte_port_in_ops ops;
  54         rte_pipeline_port_in_action_handler f_action;
  55         void *arg_ah;
  56         uint32_t burst_size;
  57
  58         /* The table to which this port is connected */
  59         uint32_t table_id;
  60
  61         /* Handle to low-level port */
  62         void *h_port;
  63
  64         /* List of enabled ports */
  65         struct rte_port_in *next;
  66 };
  67
  68 struct rte_port_out {
  69         /* Input parameters */
  70         struct rte_port_out_ops ops;
  71         rte_pipeline_port_out_action_handler f_action;
  72         rte_pipeline_port_out_action_handler_bulk f_action_bulk;
  73         void *arg_ah;
  74
  75         /* Handle to low-level port */
  76         void *h_port;
  77 };
  78
  79 struct rte_table {
  80         /* Input parameters */
  81         struct rte_table_ops ops;
  82         rte_pipeline_table_action_handler_hit f_action_hit;
  83         rte_pipeline_table_action_handler_miss f_action_miss;
  84         void *arg_ah;
  85         struct rte_pipeline_table_entry *default_entry;
  86         uint32_t entry_size;
  87
  88         uint32_t table_next_id;
  89         uint32_t table_next_id_valid;
  90
  91         /* Handle to the low-level table object */
  92         void *h_table;
  93 };
  94
  95 #define RTE_PIPELINE_MAX_NAME_SZ                           124
  96
  97 struct rte_pipeline {
  98         /* Input parameters */
  99         char name[RTE_PIPELINE_MAX_NAME_SZ];
 100         int socket_id;
 101         uint32_t offset_port_id;
 102
 103         /* Internal tables */
 104         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 105         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 106         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 107
 108         /* Occupancy of internal tables */
 109         uint32_t num_ports_in;
 110         uint32_t num_ports_out;
 111         uint32_t num_tables;
 112
 113         /* List of enabled ports */
 114         uint64_t enabled_port_in_mask;
 115         struct rte_port_in *port_in_first;
 116
 117         /* Pipeline run structures */
 118         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 119         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 120         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 121         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 122 } __rte_cache_aligned;
 123
 124 static inline uint32_t
 125 rte_mask_get_next(uint64_t mask, uint32_t pos)
 126 {
 127         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 128                         (mask >> ((pos + 1) & 0x3F));
 129         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 130 }
 131
 132 static inline uint32_t
 133 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 134 {
 135         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 136                         (mask << ((64 - pos) & 0x3F));
 137         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 138 }
 139
 140 static void
 141 rte_pipeline_table_free(struct rte_table *table);
 142
 143 static void
 144 rte_pipeline_port_in_free(struct rte_port_in *port);
 145
 146 static void
 147 rte_pipeline_port_out_free(struct rte_port_out *port);
 148
 149 /*
 150  * Pipeline
 151  *
 152  */
 153 static int
 154 rte_pipeline_check_params(struct rte_pipeline_params *params)
 155 {
 156         if (params == NULL) {
 157                 RTE_LOG(ERR, PIPELINE,
 158                         "%s: Incorrect value for parameter params\n", __func__);
 159                 return -EINVAL;
 160         }
 161
 162         /* name */
 163         if (params->name == NULL) {
 164                 RTE_LOG(ERR, PIPELINE,
 165                         "%s: Incorrect value for parameter name\n", __func__);
 166                 return -EINVAL;
 167         }
 168
 169         /* socket */
 170         if ((params->socket_id < 0) ||
 171             (params->socket_id >= RTE_MAX_NUMA_NODES)) {
 172                 RTE_LOG(ERR, PIPELINE,
 173                         "%s: Incorrect value for parameter socket_id\n",
 174                         __func__);
 175                 return -EINVAL;
 176         }
 177
 178         return 0;
 179 }
 180
 181 struct rte_pipeline *
 182 rte_pipeline_create(struct rte_pipeline_params *params)
 183 {
 184         struct rte_pipeline *p;
 185         int status;
 186
 187         /* Check input parameters */
 188         status = rte_pipeline_check_params(params);
 189         if (status != 0) {
 190                 RTE_LOG(ERR, PIPELINE,
 191                         "%s: Pipeline params check failed (%d)\n",
 192                         __func__, status);
 193                 return NULL;
 194         }
 195
 196         /* Allocate memory for the pipeline on requested socket */
 197         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 198                         RTE_CACHE_LINE_SIZE, params->socket_id);
 199
 200         if (p == NULL) {
 201                 RTE_LOG(ERR, PIPELINE,
 202                         "%s: Pipeline memory allocation failed\n", __func__);
 203                 return NULL;
 204         }
 205
 206         /* Save input parameters */
 207         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
 208         p->socket_id = params->socket_id;
 209         p->offset_port_id = params->offset_port_id;
 210
 211         /* Initialize pipeline internal data structure */
 212         p->num_ports_in = 0;
 213         p->num_ports_out = 0;
 214         p->num_tables = 0;
 215         p->enabled_port_in_mask = 0;
 216         p->port_in_first = NULL;
 217
 218         return p;
 219 }
 220
 221 int
 222 rte_pipeline_free(struct rte_pipeline *p)
 223 {
 224         uint32_t i;
 225
 226         /* Check input parameters */
 227         if (p == NULL) {
 228                 RTE_LOG(ERR, PIPELINE,
 229                         "%s: rte_pipeline parameter is NULL\n", __func__);
 230                 return -EINVAL;
 231         }
 232
 233         /* Free input ports */
 234         for (i = 0; i < p->num_ports_in; i++) {
 235                 struct rte_port_in *port = &p->ports_in[i];
 236
 237                 rte_pipeline_port_in_free(port);
 238         }
 239
 240         /* Free tables */
 241         for (i = 0; i < p->num_tables; i++) {
 242                 struct rte_table *table = &p->tables[i];
 243
 244                 rte_pipeline_table_free(table);
 245         }
 246
 247         /* Free output ports */
 248         for (i = 0; i < p->num_ports_out; i++) {
 249                 struct rte_port_out *port = &p->ports_out[i];
 250
 251                 rte_pipeline_port_out_free(port);
 252         }
 253
 254         /* Free pipeline memory */
 255         rte_free(p);
 256
 257         return 0;
 258 }
 259
 260 /*
 261  * Table
 262  *
 263  */
 264 static int
 265 rte_table_check_params(struct rte_pipeline *p,
 266                 struct rte_pipeline_table_params *params,
 267                 uint32_t *table_id)
 268 {
 269         if (p == NULL) {
 270                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 271                         __func__);
 272                 return -EINVAL;
 273         }
 274         if (params == NULL) {
 275                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 276                         __func__);
 277                 return -EINVAL;
 278         }
 279         if (table_id == NULL) {
 280                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 281                         __func__);
 282                 return -EINVAL;
 283         }
 284
 285         /* ops */
 286         if (params->ops == NULL) {
 287                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 288                         __func__);
 289                 return -EINVAL;
 290         }
 291
 292         if (params->ops->f_create == NULL) {
 293                 RTE_LOG(ERR, PIPELINE,
 294                         "%s: f_create function pointer is NULL\n", __func__);
 295                 return -EINVAL;
 296         }
 297
 298         if (params->ops->f_lookup == NULL) {
 299                 RTE_LOG(ERR, PIPELINE,
 300                         "%s: f_lookup function pointer is NULL\n", __func__);
 301                 return -EINVAL;
 302         }
 303
 304         /* De we have room for one more table? */
 305         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 306                 RTE_LOG(ERR, PIPELINE,
 307                         "%s: Incorrect value for num_tables parameter\n",
 308                         __func__);
 309                 return -EINVAL;
 310         }
 311
 312         return 0;
 313 }
 314
 315 int
 316 rte_pipeline_table_create(struct rte_pipeline *p,
 317                 struct rte_pipeline_table_params *params,
 318                 uint32_t *table_id)
 319 {
 320         struct rte_table *table;
 321         struct rte_pipeline_table_entry *default_entry;
 322         void *h_table;
 323         uint32_t entry_size, id;
 324         int status;
 325
 326         /* Check input arguments */
 327         status = rte_table_check_params(p, params, table_id);
 328         if (status != 0)
 329                 return status;
 330
 331         id = p->num_tables;
 332         table = &p->tables[id];
 333
 334         /* Allocate space for the default table entry */
 335         entry_size = sizeof(struct rte_pipeline_table_entry) +
 336                 params->action_data_size;
 337         default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
 338                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 339         if (default_entry == NULL) {
 340                 RTE_LOG(ERR, PIPELINE,
 341                         "%s: Failed to allocate default entry\n", __func__);
 342                 return -EINVAL;
 343         }
 344
 345         /* Create the table */
 346         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 347                 entry_size);
 348         if (h_table == NULL) {
 349                 rte_free(default_entry);
 350                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 351                 return -EINVAL;
 352         }
 353
 354         /* Commit current table to the pipeline */
 355         p->num_tables++;
 356         *table_id = id;
 357
 358         /* Save input parameters */
 359         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 360         table->f_action_hit = params->f_action_hit;
 361         table->f_action_miss = params->f_action_miss;
 362         table->arg_ah = params->arg_ah;
 363         table->entry_size = entry_size;
 364
 365         /* Clear the lookup miss actions (to be set later through API) */
 366         table->default_entry = default_entry;
 367         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 368
 369         /* Initialize table internal data structure */
 370         table->h_table = h_table;
 371         table->table_next_id = 0;
 372         table->table_next_id_valid = 0;
 373
 374         return 0;
 375 }
 376
 377 void
 378 rte_pipeline_table_free(struct rte_table *table)
 379 {
 380         if (table->ops.f_free != NULL)
 381                 table->ops.f_free(table->h_table);
 382
 383         rte_free(table->default_entry);
 384 }
 385
 386 int
 387 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 388         uint32_t table_id,
 389         struct rte_pipeline_table_entry *default_entry,
 390         struct rte_pipeline_table_entry **default_entry_ptr)
 391 {
 392         struct rte_table *table;
 393
 394         /* Check input arguments */
 395         if (p == NULL) {
 396                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 397                         __func__);
 398                 return -EINVAL;
 399         }
 400
 401         if (default_entry == NULL) {
 402                 RTE_LOG(ERR, PIPELINE,
 403                         "%s: default_entry parameter is NULL\n", __func__);
 404                 return -EINVAL;
 405         }
 406
 407         if (table_id >= p->num_tables) {
 408                 RTE_LOG(ERR, PIPELINE,
 409                         "%s: table_id %d out of range\n", __func__, table_id);
 410                 return -EINVAL;
 411         }
 412
 413         table = &p->tables[table_id];
 414
 415         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 416                 table->table_next_id_valid &&
 417                 (default_entry->table_id != table->table_next_id)) {
 418                 RTE_LOG(ERR, PIPELINE,
 419                         "%s: Tree-like topologies not allowed\n", __func__);
 420                 return -EINVAL;
 421         }
 422
 423         /* Set the lookup miss actions */
 424         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 425                 (table->table_next_id_valid == 0)) {
 426                 table->table_next_id = default_entry->table_id;
 427                 table->table_next_id_valid = 1;
 428         }
 429
 430         memcpy(table->default_entry, default_entry, table->entry_size);
 431
 432         *default_entry_ptr = table->default_entry;
 433         return 0;
 434 }
 435
 436 int
 437 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 438                 uint32_t table_id,
 439                 struct rte_pipeline_table_entry *entry)
 440 {
 441         struct rte_table *table;
 442
 443         /* Check input arguments */
 444         if (p == NULL) {
 445                 RTE_LOG(ERR, PIPELINE,
 446                         "%s: pipeline parameter is NULL\n", __func__);
 447                 return -EINVAL;
 448         }
 449
 450         if (table_id >= p->num_tables) {
 451                 RTE_LOG(ERR, PIPELINE,
 452                         "%s: table_id %d out of range\n", __func__, table_id);
 453                 return -EINVAL;
 454         }
 455
 456         table = &p->tables[table_id];
 457
 458         /* Save the current contents of the default entry */
 459         if (entry)
 460                 memcpy(entry, table->default_entry, table->entry_size);
 461
 462         /* Clear the lookup miss actions */
 463         memset(table->default_entry, 0, table->entry_size);
 464         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 465
 466         return 0;
 467 }
 468
 469 int
 470 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 471                 uint32_t table_id,
 472                 void *key,
 473                 struct rte_pipeline_table_entry *entry,
 474                 int *key_found,
 475                 struct rte_pipeline_table_entry **entry_ptr)
 476 {
 477         struct rte_table *table;
 478
 479         /* Check input arguments */
 480         if (p == NULL) {
 481                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 482                         __func__);
 483                 return -EINVAL;
 484         }
 485
 486         if (key == NULL) {
 487                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 488                 return -EINVAL;
 489         }
 490
 491         if (entry == NULL) {
 492                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 493                         __func__);
 494                 return -EINVAL;
 495         }
 496
 497         if (table_id >= p->num_tables) {
 498                 RTE_LOG(ERR, PIPELINE,
 499                         "%s: table_id %d out of range\n", __func__, table_id);
 500                 return -EINVAL;
 501         }
 502
 503         table = &p->tables[table_id];
 504
 505         if (table->ops.f_add == NULL) {
 506                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 507                         __func__);
 508                 return -EINVAL;
 509         }
 510
 511         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 512                 table->table_next_id_valid &&
 513                 (entry->table_id != table->table_next_id)) {
 514                 RTE_LOG(ERR, PIPELINE,
 515                         "%s: Tree-like topologies not allowed\n", __func__);
 516                 return -EINVAL;
 517         }
 518
 519         /* Add entry */
 520         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 521                 (table->table_next_id_valid == 0)) {
 522                 table->table_next_id = entry->table_id;
 523                 table->table_next_id_valid = 1;
 524         }
 525
 526         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 527                 key_found, (void **) entry_ptr);
 528 }
 529
 530 int
 531 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 532                 uint32_t table_id,
 533                 void *key,
 534                 int *key_found,
 535                 struct rte_pipeline_table_entry *entry)
 536 {
 537         struct rte_table *table;
 538
 539         /* Check input arguments */
 540         if (p == NULL) {
 541                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 542                         __func__);
 543                 return -EINVAL;
 544         }
 545
 546         if (key == NULL) {
 547                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 548                         __func__);
 549                 return -EINVAL;
 550         }
 551
 552         if (table_id >= p->num_tables) {
 553                 RTE_LOG(ERR, PIPELINE,
 554                         "%s: table_id %d out of range\n", __func__, table_id);
 555                 return -EINVAL;
 556         }
 557
 558         table = &p->tables[table_id];
 559
 560         if (table->ops.f_delete == NULL) {
 561                 RTE_LOG(ERR, PIPELINE,
 562                         "%s: f_delete function pointer NULL\n", __func__);
 563                 return -EINVAL;
 564         }
 565
 566         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 567 }
 568
 569 /*
 570  * Port
 571  *
 572  */
 573 static int
 574 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 575                 struct rte_pipeline_port_in_params *params,
 576                 uint32_t *port_id)
 577 {
 578         if (p == NULL) {
 579                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 580                         __func__);
 581                 return -EINVAL;
 582         }
 583         if (params == NULL) {
 584                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 585                 return -EINVAL;
 586         }
 587         if (port_id == NULL) {
 588                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 589                         __func__);
 590                 return -EINVAL;
 591         }
 592
 593         /* ops */
 594         if (params->ops == NULL) {
 595                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 596                         __func__);
 597                 return -EINVAL;
 598         }
 599
 600         if (params->ops->f_create == NULL) {
 601                 RTE_LOG(ERR, PIPELINE,
 602                         "%s: f_create function pointer NULL\n", __func__);
 603                 return -EINVAL;
 604         }
 605
 606         if (params->ops->f_rx == NULL) {
 607                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 608                         __func__);
 609                 return -EINVAL;
 610         }
 611
 612         /* burst_size */
 613         if ((params->burst_size == 0) ||
 614                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 615                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 616                         __func__);
 617                 return -EINVAL;
 618         }
 619
 620         /* Do we have room for one more port? */
 621         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 622                 RTE_LOG(ERR, PIPELINE,
 623                         "%s: invalid value for num_ports_in\n", __func__);
 624                 return -EINVAL;
 625         }
 626
 627         return 0;
 628 }
 629
 630 static int
 631 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 632                 struct rte_pipeline_port_out_params *params,
 633                 uint32_t *port_id)
 634 {
 635         rte_pipeline_port_out_action_handler f_ah;
 636         rte_pipeline_port_out_action_handler_bulk f_ah_bulk;
 637
 638         if (p == NULL) {
 639                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 640                         __func__);
 641                 return -EINVAL;
 642         }
 643
 644         if (params == NULL) {
 645                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 646                 return -EINVAL;
 647         }
 648
 649         if (port_id == NULL) {
 650                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 651                         __func__);
 652                 return -EINVAL;
 653         }
 654
 655         /* ops */
 656         if (params->ops == NULL) {
 657                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 658                         __func__);
 659                 return -EINVAL;
 660         }
 661
 662         if (params->ops->f_create == NULL) {
 663                 RTE_LOG(ERR, PIPELINE,
 664                         "%s: f_create function pointer NULL\n", __func__);
 665                 return -EINVAL;
 666         }
 667
 668         if (params->ops->f_tx == NULL) {
 669                 RTE_LOG(ERR, PIPELINE,
 670                                 "%s: f_tx function pointer NULL\n", __func__);
 671                 return -EINVAL;
 672         }
 673
 674         if (params->ops->f_tx_bulk == NULL) {
 675                 RTE_LOG(ERR, PIPELINE,
 676                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 677                 return -EINVAL;
 678         }
 679
 680         f_ah = params->f_action;
 681         f_ah_bulk = params->f_action_bulk;
 682         if (((f_ah != NULL) && (f_ah_bulk == NULL)) ||
 683             ((f_ah == NULL) && (f_ah_bulk != NULL))) {
 684                 RTE_LOG(ERR, PIPELINE, "%s: Action handlers have to be either"
 685                         "both enabled or both disabled\n", __func__);
 686                 return -EINVAL;
 687         }
 688
 689         /* Do we have room for one more port? */
 690         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 691                 RTE_LOG(ERR, PIPELINE,
 692                         "%s: invalid value for num_ports_out\n", __func__);
 693                 return -EINVAL;
 694         }
 695
 696         return 0;
 697 }
 698
 699 int
 700 rte_pipeline_port_in_create(struct rte_pipeline *p,
 701                 struct rte_pipeline_port_in_params *params,
 702                 uint32_t *port_id)
 703 {
 704         struct rte_port_in *port;
 705         void *h_port;
 706         uint32_t id;
 707         int status;
 708
 709         /* Check input arguments */
 710         status = rte_pipeline_port_in_check_params(p, params, port_id);
 711         if (status != 0)
 712                 return status;
 713
 714         id = p->num_ports_in;
 715         port = &p->ports_in[id];
 716
 717         /* Create the port */
 718         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 719         if (h_port == NULL) {
 720                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 721                 return -EINVAL;
 722         }
 723
 724         /* Commit current table to the pipeline */
 725         p->num_ports_in++;
 726         *port_id = id;
 727
 728         /* Save input parameters */
 729         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 730         port->f_action = params->f_action;
 731         port->arg_ah = params->arg_ah;
 732         port->burst_size = params->burst_size;
 733
 734         /* Initialize port internal data structure */
 735         port->table_id = RTE_TABLE_INVALID;
 736         port->h_port = h_port;
 737         port->next = NULL;
 738
 739         return 0;
 740 }
 741
 742 void
 743 rte_pipeline_port_in_free(struct rte_port_in *port)
 744 {
 745         if (port->ops.f_free != NULL)
 746                 port->ops.f_free(port->h_port);
 747 }
 748
 749 int
 750 rte_pipeline_port_out_create(struct rte_pipeline *p,
 751                 struct rte_pipeline_port_out_params *params,
 752                 uint32_t *port_id)
 753 {
 754         struct rte_port_out *port;
 755         void *h_port;
 756         uint32_t id;
 757         int status;
 758
 759         /* Check input arguments */
 760         status = rte_pipeline_port_out_check_params(p, params, port_id);
 761         if (status != 0)
 762                 return status;
 763
 764         id = p->num_ports_out;
 765         port = &p->ports_out[id];
 766
 767         /* Create the port */
 768         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 769         if (h_port == NULL) {
 770                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 771                 return -EINVAL;
 772         }
 773
 774         /* Commit current table to the pipeline */
 775         p->num_ports_out++;
 776         *port_id = id;
 777
 778         /* Save input parameters */
 779         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 780         port->f_action = params->f_action;
 781         port->f_action_bulk = params->f_action_bulk;
 782         port->arg_ah = params->arg_ah;
 783
 784         /* Initialize port internal data structure */
 785         port->h_port = h_port;
 786
 787         return 0;
 788 }
 789
 790 void
 791 rte_pipeline_port_out_free(struct rte_port_out *port)
 792 {
 793         if (port->ops.f_free != NULL)
 794                 port->ops.f_free(port->h_port);
 795 }
 796
 797 int
 798 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 799                 uint32_t port_id,
 800                 uint32_t table_id)
 801 {
 802         struct rte_port_in *port;
 803
 804         /* Check input arguments */
 805         if (p == NULL) {
 806                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 807                         __func__);
 808                 return -EINVAL;
 809         }
 810
 811         if (port_id >= p->num_ports_in) {
 812                 RTE_LOG(ERR, PIPELINE,
 813                         "%s: port IN ID %u is out of range\n",
 814                         __func__, port_id);
 815                 return -EINVAL;
 816         }
 817
 818         if (table_id >= p->num_tables) {
 819                 RTE_LOG(ERR, PIPELINE,
 820                         "%s: Table ID %u is out of range\n",
 821                         __func__, table_id);
 822                 return -EINVAL;
 823         }
 824
 825         port = &p->ports_in[port_id];
 826         port->table_id = table_id;
 827
 828         return 0;
 829 }
 830
 831 int
 832 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 833 {
 834         struct rte_port_in *port, *port_prev, *port_next;
 835         struct rte_port_in *port_first, *port_last;
 836         uint64_t port_mask;
 837         uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
 838
 839         /* Check input arguments */
 840         if (p == NULL) {
 841                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 842                         __func__);
 843                 return -EINVAL;
 844         }
 845
 846         if (port_id >= p->num_ports_in) {
 847                 RTE_LOG(ERR, PIPELINE,
 848                         "%s: port IN ID %u is out of range\n",
 849                         __func__, port_id);
 850                 return -EINVAL;
 851         }
 852
 853         /* Return if current input port is already enabled */
 854         port_mask = 1LLU << port_id;
 855         if (p->enabled_port_in_mask & port_mask)
 856                 return 0;
 857
 858         p->enabled_port_in_mask |= port_mask;
 859
 860         /* Add current input port to the pipeline chain of enabled ports */
 861         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 862         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 863
 864         port_prev = &p->ports_in[port_prev_id];
 865         port_next = &p->ports_in[port_next_id];
 866         port = &p->ports_in[port_id];
 867
 868         port_prev->next = port;
 869         port->next = port_next;
 870
 871         /* Update the first and last input ports in the chain */
 872         port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
 873         port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
 874
 875         port_first = &p->ports_in[port_first_id];
 876         port_last = &p->ports_in[port_last_id];
 877
 878         p->port_in_first = port_first;
 879         port_last->next = NULL;
 880
 881         return 0;
 882 }
 883
 884 int
 885 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
 886 {
 887         struct rte_port_in *port_prev, *port_next, *port_first, *port_last;
 888         uint64_t port_mask;
 889         uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
 890
 891         /* Check input arguments */
 892         if (p == NULL) {
 893                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 894                 __func__);
 895                 return -EINVAL;
 896         }
 897
 898         if (port_id >= p->num_ports_in) {
 899                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
 900                         __func__, port_id);
 901                 return -EINVAL;
 902         }
 903
 904         /* Return if current input port is already disabled */
 905         port_mask = 1LLU << port_id;
 906         if ((p->enabled_port_in_mask & port_mask) == 0)
 907                 return 0;
 908
 909         /* Return if no other enabled ports */
 910         if (__builtin_popcountll(p->enabled_port_in_mask) == 1) {
 911                 p->enabled_port_in_mask &= ~port_mask;
 912                 p->port_in_first = NULL;
 913
 914                 return 0;
 915         }
 916
 917         /* Add current input port to the pipeline chain of enabled ports */
 918         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 919         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 920
 921         port_prev = &p->ports_in[port_prev_id];
 922         port_next = &p->ports_in[port_next_id];
 923
 924         port_prev->next = port_next;
 925         p->enabled_port_in_mask &= ~port_mask;
 926
 927         /* Update the first and last input ports in the chain */
 928         port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
 929         port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
 930
 931         port_first = &p->ports_in[port_first_id];
 932         port_last = &p->ports_in[port_last_id];
 933
 934         p->port_in_first = port_first;
 935         port_last->next = NULL;
 936
 937         return 0;
 938 }
 939
 940 /*
 941  * Pipeline run-time
 942  *
 943  */
 944 int
 945 rte_pipeline_check(struct rte_pipeline *p)
 946 {
 947         uint32_t port_in_id;
 948
 949         /* Check input arguments */
 950         if (p == NULL) {
 951                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 952                         __func__);
 953                 return -EINVAL;
 954         }
 955
 956         /* Check that pipeline has at least one input port, one table and one
 957         output port */
 958         if (p->num_ports_in == 0) {
 959                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
 960                         __func__);
 961                 return -EINVAL;
 962         }
 963         if (p->num_tables == 0) {
 964                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
 965                         __func__);
 966                 return -EINVAL;
 967         }
 968         if (p->num_ports_out == 0) {
 969                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
 970                         __func__);
 971                 return -EINVAL;
 972         }
 973
 974         /* Check that all input ports are connected */
 975         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
 976                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
 977
 978                 if (port_in->table_id == RTE_TABLE_INVALID) {
 979                         RTE_LOG(ERR, PIPELINE,
 980                                 "%s: Port IN ID %u is not connected\n",
 981                                 __func__, port_in_id);
 982                         return -EINVAL;
 983                 }
 984         }
 985
 986         return 0;
 987 }
 988
 989 static inline void
 990 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
 991 {
 992         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
 993         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
 994         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
 995         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
 996
 997         if ((pkts_mask & (pkts_mask + 1)) == 0) {
 998                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
 999                 uint32_t i;
1000
1001                 for (i = 0; i < n_pkts; i++) {
1002                         uint64_t pkt_mask = 1LLU << i;
1003                         uint32_t pos = p->entries[i]->action;
1004
1005                         p->action_mask1[pos] |= pkt_mask;
1006                 }
1007         } else {
1008                 uint32_t i;
1009
1010                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1011                         uint64_t pkt_mask = 1LLU << i;
1012                         uint32_t pos;
1013
1014                         if ((pkt_mask & pkts_mask) == 0)
1015                                 continue;
1016
1017                         pos = p->entries[i]->action;
1018                         p->action_mask1[pos] |= pkt_mask;
1019                 }
1020         }
1021 }
1022
1023 static inline void
1024 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1025                 uint64_t pkts_mask, uint32_t port_id)
1026 {
1027         struct rte_port_out *port_out = &p->ports_out[port_id];
1028
1029         /* Output port user actions */
1030         if (port_out->f_action_bulk != NULL) {
1031                 uint64_t mask = pkts_mask;
1032
1033                 port_out->f_action_bulk(p->pkts, &pkts_mask, port_out->arg_ah);
1034                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask ^  mask;
1035         }
1036
1037         /* Output port TX */
1038         if (pkts_mask != 0)
1039                 port_out->ops.f_tx_bulk(port_out->h_port, p->pkts, pkts_mask);
1040 }
1041
1042 static inline void
1043 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1044 {
1045         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1046                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1047                 uint32_t i;
1048
1049                 for (i = 0; i < n_pkts; i++) {
1050                         struct rte_mbuf *pkt = p->pkts[i];
1051                         uint32_t port_out_id = p->entries[i]->port_id;
1052                         struct rte_port_out *port_out =
1053                                 &p->ports_out[port_out_id];
1054
1055                         /* Output port user actions */
1056                         if (port_out->f_action == NULL) /* Output port TX */
1057                                 port_out->ops.f_tx(port_out->h_port, pkt);
1058                         else {
1059                                 uint64_t pkt_mask = 1LLU;
1060
1061                                 port_out->f_action(pkt, &pkt_mask,
1062                                         port_out->arg_ah);
1063                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1064                                         (pkt_mask ^ 1LLU) << i;
1065
1066                                 /* Output port TX */
1067                                 if (pkt_mask != 0)
1068                                         port_out->ops.f_tx(port_out->h_port,
1069                                                 pkt);
1070                         }
1071                 }
1072         } else {
1073                 uint32_t i;
1074
1075                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1076                         uint64_t pkt_mask = 1LLU << i;
1077                         struct rte_mbuf *pkt;
1078                         struct rte_port_out *port_out;
1079                         uint32_t port_out_id;
1080
1081                         if ((pkt_mask & pkts_mask) == 0)
1082                                 continue;
1083
1084                         pkt = p->pkts[i];
1085                         port_out_id = p->entries[i]->port_id;
1086                         port_out = &p->ports_out[port_out_id];
1087
1088                         /* Output port user actions */
1089                         if (port_out->f_action == NULL) /* Output port TX */
1090                                 port_out->ops.f_tx(port_out->h_port, pkt);
1091                         else {
1092                                 pkt_mask = 1LLU;
1093
1094                                 port_out->f_action(pkt, &pkt_mask,
1095                                         port_out->arg_ah);
1096                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1097                                         (pkt_mask ^ 1LLU) << i;
1098
1099                                 /* Output port TX */
1100                                 if (pkt_mask != 0)
1101                                         port_out->ops.f_tx(port_out->h_port,
1102                                                 pkt);
1103                         }
1104                 }
1105         }
1106 }
1107
1108 static inline void
1109 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1110         uint64_t pkts_mask)
1111 {
1112         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1113                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1114                 uint32_t i;
1115
1116                 for (i = 0; i < n_pkts; i++) {
1117                         struct rte_mbuf *pkt = p->pkts[i];
1118                         uint32_t port_out_id =
1119                                 RTE_MBUF_METADATA_UINT32(pkt,
1120                                         p->offset_port_id);
1121                         struct rte_port_out *port_out = &p->ports_out[
1122                                 port_out_id];
1123
1124                         /* Output port user actions */
1125                         if (port_out->f_action == NULL) /* Output port TX */
1126                                 port_out->ops.f_tx(port_out->h_port, pkt);
1127                         else {
1128                                 uint64_t pkt_mask = 1LLU;
1129
1130                                 port_out->f_action(pkt, &pkt_mask,
1131                                         port_out->arg_ah);
1132                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1133                                         (pkt_mask ^ 1LLU) << i;
1134
1135                                 /* Output port TX */
1136                                 if (pkt_mask != 0)
1137                                         port_out->ops.f_tx(port_out->h_port,
1138                                                 pkt);
1139                         }
1140                 }
1141         } else {
1142                 uint32_t i;
1143
1144                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1145                         uint64_t pkt_mask = 1LLU << i;
1146                         struct rte_mbuf *pkt;
1147                         struct rte_port_out *port_out;
1148                         uint32_t port_out_id;
1149
1150                         if ((pkt_mask & pkts_mask) == 0)
1151                                 continue;
1152
1153                         pkt = p->pkts[i];
1154                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1155                                 p->offset_port_id);
1156                         port_out = &p->ports_out[port_out_id];
1157
1158                         /* Output port user actions */
1159                         if (port_out->f_action == NULL) /* Output port TX */
1160                                 port_out->ops.f_tx(port_out->h_port, pkt);
1161                         else {
1162                                 pkt_mask = 1LLU;
1163
1164                                 port_out->f_action(pkt, &pkt_mask,
1165                                         port_out->arg_ah);
1166                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1167                                         (pkt_mask ^ 1LLU) << i;
1168
1169                                 /* Output port TX */
1170                                 if (pkt_mask != 0)
1171                                         port_out->ops.f_tx(port_out->h_port,
1172                                                 pkt);
1173                         }
1174                 }
1175         }
1176 }
1177
1178 static inline void
1179 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1180 {
1181         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1182                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1183                 uint32_t i;
1184
1185                 for (i = 0; i < n_pkts; i++)
1186                         rte_pktmbuf_free(p->pkts[i]);
1187         } else {
1188                 uint32_t i;
1189
1190                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1191                         uint64_t pkt_mask = 1LLU << i;
1192
1193                         if ((pkt_mask & pkts_mask) == 0)
1194                                 continue;
1195
1196                         rte_pktmbuf_free(p->pkts[i]);
1197                 }
1198         }
1199 }
1200
1201 int
1202 rte_pipeline_run(struct rte_pipeline *p)
1203 {
1204         struct rte_port_in *port_in;
1205
1206         for (port_in = p->port_in_first; port_in != NULL;
1207                 port_in = port_in->next) {
1208                 uint64_t pkts_mask;
1209                 uint32_t n_pkts, table_id;
1210
1211                 /* Input port RX */
1212                 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1213                         port_in->burst_size);
1214                 if (n_pkts == 0)
1215                         continue;
1216
1217                 pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1218                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1219                 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1220                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1221                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1222
1223                 /* Input port user actions */
1224                 if (port_in->f_action != NULL) {
1225                         uint64_t mask = pkts_mask;
1226
1227                         port_in->f_action(p->pkts, n_pkts, &pkts_mask,
1228                                 port_in->arg_ah);
1229                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1230                                 pkts_mask ^ mask;
1231                 }
1232
1233                 /* Table */
1234                 for (table_id = port_in->table_id; pkts_mask != 0; ) {
1235                         struct rte_table *table;
1236                         uint64_t lookup_hit_mask, lookup_miss_mask;
1237
1238                         /* Lookup */
1239                         table = &p->tables[table_id];
1240                         table->ops.f_lookup(table->h_table, p->pkts, pkts_mask,
1241                                         &lookup_hit_mask, (void **) p->entries);
1242                         lookup_miss_mask = pkts_mask & (~lookup_hit_mask);
1243
1244                         /* Lookup miss */
1245                         if (lookup_miss_mask != 0) {
1246                                 struct rte_pipeline_table_entry *default_entry =
1247                                         table->default_entry;
1248
1249                                 /* Table user actions */
1250                                 if (table->f_action_miss != NULL) {
1251                                         uint64_t mask = lookup_miss_mask;
1252
1253                                         table->f_action_miss(p->pkts,
1254                                                 &lookup_miss_mask,
1255                                                 default_entry, table->arg_ah);
1256                                         p->action_mask0[
1257                                                 RTE_PIPELINE_ACTION_DROP] |=
1258                                                 lookup_miss_mask ^ mask;
1259                                 }
1260
1261                                 /* Table reserved actions */
1262                                 if ((default_entry->action ==
1263                                         RTE_PIPELINE_ACTION_PORT) &&
1264                                         (lookup_miss_mask != 0))
1265                                         rte_pipeline_action_handler_port_bulk(p,
1266                                                 lookup_miss_mask,
1267                                                 default_entry->port_id);
1268                                 else {
1269                                         uint32_t pos = default_entry->action;
1270
1271                                         p->action_mask0[pos] = lookup_miss_mask;
1272                                 }
1273                         }
1274
1275                         /* Lookup hit */
1276                         if (lookup_hit_mask != 0) {
1277                                 /* Table user actions */
1278                                 if (table->f_action_hit != NULL) {
1279                                         uint64_t mask = lookup_hit_mask;
1280
1281                                         table->f_action_hit(p->pkts,
1282                                                 &lookup_hit_mask,
1283                                                 p->entries, table->arg_ah);
1284                                         p->action_mask0[
1285                                                 RTE_PIPELINE_ACTION_DROP] |=
1286                                                 lookup_hit_mask ^ mask;
1287                                 }
1288
1289                                 /* Table reserved actions */
1290                                 rte_pipeline_compute_masks(p, lookup_hit_mask);
1291                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1292                                         p->action_mask1[
1293                                                 RTE_PIPELINE_ACTION_DROP];
1294                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1295                                         p->action_mask1[
1296                                                 RTE_PIPELINE_ACTION_PORT];
1297                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1298                                         p->action_mask1[
1299                                                 RTE_PIPELINE_ACTION_PORT_META];
1300                                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1301                                         p->action_mask1[
1302                                                 RTE_PIPELINE_ACTION_TABLE];
1303                         }
1304
1305                         /* Prepare for next iteration */
1306                         pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1307                         table_id = table->table_next_id;
1308                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1309                 }
1310
1311                 /* Table reserved action PORT */
1312                 rte_pipeline_action_handler_port(p,
1313                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1314
1315                 /* Table reserved action PORT META */
1316                 rte_pipeline_action_handler_port_meta(p,
1317                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1318
1319                 /* Table reserved action DROP */
1320                 rte_pipeline_action_handler_drop(p,
1321                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1322         }
1323
1324         return 0;
1325 }
1326
1327 int
1328 rte_pipeline_flush(struct rte_pipeline *p)
1329 {
1330         uint32_t port_id;
1331
1332         /* Check input arguments */
1333         if (p == NULL) {
1334                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1335                         __func__);
1336                 return -EINVAL;
1337         }
1338
1339         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1340                 struct rte_port_out *port = &p->ports_out[port_id];
1341
1342                 if (port->ops.f_flush != NULL)
1343                         port->ops.f_flush(port->h_port);
1344         }
1345
1346         return 0;
1347 }
1348
1349 int
1350 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1351                 uint32_t port_id, struct rte_mbuf *pkt)
1352 {
1353         struct rte_port_out *port_out = &p->ports_out[port_id];
1354
1355         /* Output port user actions */
1356         if (port_out->f_action == NULL)
1357                 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1358         else {
1359                 uint64_t pkt_mask = 1LLU;
1360
1361                 port_out->f_action(pkt, &pkt_mask, port_out->arg_ah);
1362
1363                 if (pkt_mask != 0) /* Output port TX */
1364                         port_out->ops.f_tx(port_out->h_port, pkt);
1365                 else
1366                         rte_pktmbuf_free(pkt);
1367         }
1368
1369         return 0;
1370 }