lib/librte_pipeline/rte_pipeline.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_memory.h>
  39 #include <rte_memzone.h>
  40 #include <rte_cycles.h>
  41 #include <rte_prefetch.h>
  42 #include <rte_branch_prediction.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_malloc.h>
  45 #include <rte_string_fns.h>
  46
  47 #include "rte_pipeline.h"
  48
  49 #define RTE_TABLE_INVALID                                 UINT32_MAX
  50
  51 struct rte_port_in {
  52         /* Input parameters */
  53         struct rte_port_in_ops ops;
  54         rte_pipeline_port_in_action_handler f_action;
  55         void *arg_ah;
  56         uint32_t burst_size;
  57
  58         /* The table to which this port is connected */
  59         uint32_t table_id;
  60
  61         /* Handle to low-level port */
  62         void *h_port;
  63
  64         /* List of enabled ports */
  65         struct rte_port_in *next;
  66 };
  67
  68 struct rte_port_out {
  69         /* Input parameters */
  70         struct rte_port_out_ops ops;
  71         rte_pipeline_port_out_action_handler f_action;
  72         rte_pipeline_port_out_action_handler_bulk f_action_bulk;
  73         void *arg_ah;
  74
  75         /* Handle to low-level port */
  76         void *h_port;
  77 };
  78
  79 struct rte_table {
  80         /* Input parameters */
  81         struct rte_table_ops ops;
  82         rte_pipeline_table_action_handler_hit f_action_hit;
  83         rte_pipeline_table_action_handler_miss f_action_miss;
  84         void *arg_ah;
  85         struct rte_pipeline_table_entry *default_entry;
  86         uint32_t entry_size;
  87
  88         uint32_t table_next_id;
  89         uint32_t table_next_id_valid;
  90
  91         /* Handle to the low-level table object */
  92         void *h_table;
  93 };
  94
  95 #define RTE_PIPELINE_MAX_NAME_SZ                           124
  96
  97 struct rte_pipeline {
  98         /* Input parameters */
  99         char name[RTE_PIPELINE_MAX_NAME_SZ];
 100         int socket_id;
 101         uint32_t offset_port_id;
 102
 103         /* Internal tables */
 104         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 105         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 106         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 107
 108         /* Occupancy of internal tables */
 109         uint32_t num_ports_in;
 110         uint32_t num_ports_out;
 111         uint32_t num_tables;
 112
 113         /* List of enabled ports */
 114         uint64_t enabled_port_in_mask;
 115         struct rte_port_in *port_in_first;
 116
 117         /* Pipeline run structures */
 118         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 119         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 120         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 121         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 122 } __rte_cache_aligned;
 123
 124 static inline uint32_t
 125 rte_mask_get_next(uint64_t mask, uint32_t pos)
 126 {
 127         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 128                         (mask >> ((pos + 1) & 0x3F));
 129         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 130 }
 131
 132 static inline uint32_t
 133 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 134 {
 135         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 136                         (mask << ((64 - pos) & 0x3F));
 137         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 138 }
 139
 140 static void
 141 rte_pipeline_table_free(struct rte_table *table);
 142
 143 static void
 144 rte_pipeline_port_in_free(struct rte_port_in *port);
 145
 146 static void
 147 rte_pipeline_port_out_free(struct rte_port_out *port);
 148
 149 /*
 150  * Pipeline
 151  *
 152  */
 153 static int
 154 rte_pipeline_check_params(struct rte_pipeline_params *params)
 155 {
 156         if (params == NULL) {
 157                 RTE_LOG(ERR, PIPELINE,
 158                         "%s: Incorrect value for parameter params\n", __func__);
 159                 return -EINVAL;
 160         }
 161
 162         /* name */
 163         if (params->name == NULL) {
 164                 RTE_LOG(ERR, PIPELINE,
 165                         "%s: Incorrect value for parameter name\n", __func__);
 166                 return -EINVAL;
 167         }
 168
 169         /* socket */
 170         if ((params->socket_id < 0) ||
 171             (params->socket_id >= RTE_MAX_NUMA_NODES)) {
 172                 RTE_LOG(ERR, PIPELINE,
 173                         "%s: Incorrect value for parameter socket_id\n",
 174                         __func__);
 175                 return -EINVAL;
 176         }
 177
 178         /* offset_port_id */
 179         if (params->offset_port_id & 0x3) {
 180                 RTE_LOG(ERR, PIPELINE,
 181                         "%s: Incorrect value for parameter offset_port_id\n",
 182                         __func__);
 183                 return -EINVAL;
 184         }
 185
 186         return 0;
 187 }
 188
 189 struct rte_pipeline *
 190 rte_pipeline_create(struct rte_pipeline_params *params)
 191 {
 192         struct rte_pipeline *p;
 193         int status;
 194
 195         /* Check input parameters */
 196         status = rte_pipeline_check_params(params);
 197         if (status != 0) {
 198                 RTE_LOG(ERR, PIPELINE,
 199                         "%s: Pipeline params check failed (%d)\n",
 200                         __func__, status);
 201                 return NULL;
 202         }
 203
 204         /* Allocate memory for the pipeline on requested socket */
 205         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 206                         RTE_CACHE_LINE_SIZE, params->socket_id);
 207
 208         if (p == NULL) {
 209                 RTE_LOG(ERR, PIPELINE,
 210                         "%s: Pipeline memory allocation failed\n", __func__);
 211                 return NULL;
 212         }
 213
 214         /* Save input parameters */
 215         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
 216         p->socket_id = params->socket_id;
 217         p->offset_port_id = params->offset_port_id;
 218
 219         /* Initialize pipeline internal data structure */
 220         p->num_ports_in = 0;
 221         p->num_ports_out = 0;
 222         p->num_tables = 0;
 223         p->enabled_port_in_mask = 0;
 224         p->port_in_first = NULL;
 225
 226         return p;
 227 }
 228
 229 int
 230 rte_pipeline_free(struct rte_pipeline *p)
 231 {
 232         uint32_t i;
 233
 234         /* Check input parameters */
 235         if (p == NULL) {
 236                 RTE_LOG(ERR, PIPELINE,
 237                         "%s: rte_pipeline parameter is NULL\n", __func__);
 238                 return -EINVAL;
 239         }
 240
 241         /* Free input ports */
 242         for (i = 0; i < p->num_ports_in; i++) {
 243                 struct rte_port_in *port = &p->ports_in[i];
 244
 245                 rte_pipeline_port_in_free(port);
 246         }
 247
 248         /* Free tables */
 249         for (i = 0; i < p->num_tables; i++) {
 250                 struct rte_table *table = &p->tables[i];
 251
 252                 rte_pipeline_table_free(table);
 253         }
 254
 255         /* Free output ports */
 256         for (i = 0; i < p->num_ports_out; i++) {
 257                 struct rte_port_out *port = &p->ports_out[i];
 258
 259                 rte_pipeline_port_out_free(port);
 260         }
 261
 262         /* Free pipeline memory */
 263         rte_free(p);
 264
 265         return 0;
 266 }
 267
 268 /*
 269  * Table
 270  *
 271  */
 272 static int
 273 rte_table_check_params(struct rte_pipeline *p,
 274                 struct rte_pipeline_table_params *params,
 275                 uint32_t *table_id)
 276 {
 277         if (p == NULL) {
 278                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 279                         __func__);
 280                 return -EINVAL;
 281         }
 282         if (params == NULL) {
 283                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 284                         __func__);
 285                 return -EINVAL;
 286         }
 287         if (table_id == NULL) {
 288                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 289                         __func__);
 290                 return -EINVAL;
 291         }
 292
 293         /* ops */
 294         if (params->ops == NULL) {
 295                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 296                         __func__);
 297                 return -EINVAL;
 298         }
 299
 300         if (params->ops->f_create == NULL) {
 301                 RTE_LOG(ERR, PIPELINE,
 302                         "%s: f_create function pointer is NULL\n", __func__);
 303                 return -EINVAL;
 304         }
 305
 306         if (params->ops->f_lookup == NULL) {
 307                 RTE_LOG(ERR, PIPELINE,
 308                         "%s: f_lookup function pointer is NULL\n", __func__);
 309                 return -EINVAL;
 310         }
 311
 312         /* De we have room for one more table? */
 313         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 314                 RTE_LOG(ERR, PIPELINE,
 315                         "%s: Incorrect value for num_tables parameter\n",
 316                         __func__);
 317                 return -EINVAL;
 318         }
 319
 320         return 0;
 321 }
 322
 323 int
 324 rte_pipeline_table_create(struct rte_pipeline *p,
 325                 struct rte_pipeline_table_params *params,
 326                 uint32_t *table_id)
 327 {
 328         struct rte_table *table;
 329         struct rte_pipeline_table_entry *default_entry;
 330         void *h_table;
 331         uint32_t entry_size, id;
 332         int status;
 333
 334         /* Check input arguments */
 335         status = rte_table_check_params(p, params, table_id);
 336         if (status != 0)
 337                 return status;
 338
 339         id = p->num_tables;
 340         table = &p->tables[id];
 341
 342         /* Allocate space for the default table entry */
 343         entry_size = sizeof(struct rte_pipeline_table_entry) +
 344                 params->action_data_size;
 345         default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
 346                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 347         if (default_entry == NULL) {
 348                 RTE_LOG(ERR, PIPELINE,
 349                         "%s: Failed to allocate default entry\n", __func__);
 350                 return -EINVAL;
 351         }
 352
 353         /* Create the table */
 354         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 355                 entry_size);
 356         if (h_table == NULL) {
 357                 rte_free(default_entry);
 358                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 359                 return -EINVAL;
 360         }
 361
 362         /* Commit current table to the pipeline */
 363         p->num_tables++;
 364         *table_id = id;
 365
 366         /* Save input parameters */
 367         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 368         table->f_action_hit = params->f_action_hit;
 369         table->f_action_miss = params->f_action_miss;
 370         table->arg_ah = params->arg_ah;
 371         table->entry_size = entry_size;
 372
 373         /* Clear the lookup miss actions (to be set later through API) */
 374         table->default_entry = default_entry;
 375         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 376
 377         /* Initialize table internal data structure */
 378         table->h_table = h_table;
 379         table->table_next_id = 0;
 380         table->table_next_id_valid = 0;
 381
 382         return 0;
 383 }
 384
 385 void
 386 rte_pipeline_table_free(struct rte_table *table)
 387 {
 388         if (table->ops.f_free != NULL)
 389                 table->ops.f_free(table->h_table);
 390
 391         rte_free(table->default_entry);
 392 }
 393
 394 int
 395 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 396         uint32_t table_id,
 397         struct rte_pipeline_table_entry *default_entry,
 398         struct rte_pipeline_table_entry **default_entry_ptr)
 399 {
 400         struct rte_table *table;
 401
 402         /* Check input arguments */
 403         if (p == NULL) {
 404                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 405                         __func__);
 406                 return -EINVAL;
 407         }
 408
 409         if (default_entry == NULL) {
 410                 RTE_LOG(ERR, PIPELINE,
 411                         "%s: default_entry parameter is NULL\n", __func__);
 412                 return -EINVAL;
 413         }
 414
 415         if (table_id >= p->num_tables) {
 416                 RTE_LOG(ERR, PIPELINE,
 417                         "%s: table_id %d out of range\n", __func__, table_id);
 418                 return -EINVAL;
 419         }
 420
 421         table = &p->tables[table_id];
 422
 423         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 424                 table->table_next_id_valid &&
 425                 (default_entry->table_id != table->table_next_id)) {
 426                 RTE_LOG(ERR, PIPELINE,
 427                         "%s: Tree-like topologies not allowed\n", __func__);
 428                 return -EINVAL;
 429         }
 430
 431         /* Set the lookup miss actions */
 432         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 433                 (table->table_next_id_valid == 0)) {
 434                 table->table_next_id = default_entry->table_id;
 435                 table->table_next_id_valid = 1;
 436         }
 437
 438         memcpy(table->default_entry, default_entry, table->entry_size);
 439
 440         *default_entry_ptr = table->default_entry;
 441         return 0;
 442 }
 443
 444 int
 445 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 446                 uint32_t table_id,
 447                 struct rte_pipeline_table_entry *entry)
 448 {
 449         struct rte_table *table;
 450
 451         /* Check input arguments */
 452         if (p == NULL) {
 453                 RTE_LOG(ERR, PIPELINE,
 454                         "%s: pipeline parameter is NULL\n", __func__);
 455                 return -EINVAL;
 456         }
 457
 458         if (table_id >= p->num_tables) {
 459                 RTE_LOG(ERR, PIPELINE,
 460                         "%s: table_id %d out of range\n", __func__, table_id);
 461                 return -EINVAL;
 462         }
 463
 464         table = &p->tables[table_id];
 465
 466         /* Save the current contents of the default entry */
 467         if (entry)
 468                 memcpy(entry, table->default_entry, table->entry_size);
 469
 470         /* Clear the lookup miss actions */
 471         memset(table->default_entry, 0, table->entry_size);
 472         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 473
 474         return 0;
 475 }
 476
 477 int
 478 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 479                 uint32_t table_id,
 480                 void *key,
 481                 struct rte_pipeline_table_entry *entry,
 482                 int *key_found,
 483                 struct rte_pipeline_table_entry **entry_ptr)
 484 {
 485         struct rte_table *table;
 486
 487         /* Check input arguments */
 488         if (p == NULL) {
 489                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 490                         __func__);
 491                 return -EINVAL;
 492         }
 493
 494         if (key == NULL) {
 495                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 496                 return -EINVAL;
 497         }
 498
 499         if (entry == NULL) {
 500                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 501                         __func__);
 502                 return -EINVAL;
 503         }
 504
 505         if (table_id >= p->num_tables) {
 506                 RTE_LOG(ERR, PIPELINE,
 507                         "%s: table_id %d out of range\n", __func__, table_id);
 508                 return -EINVAL;
 509         }
 510
 511         table = &p->tables[table_id];
 512
 513         if (table->ops.f_add == NULL) {
 514                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 515                         __func__);
 516                 return -EINVAL;
 517         }
 518
 519         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 520                 table->table_next_id_valid &&
 521                 (entry->table_id != table->table_next_id)) {
 522                 RTE_LOG(ERR, PIPELINE,
 523                         "%s: Tree-like topologies not allowed\n", __func__);
 524                 return -EINVAL;
 525         }
 526
 527         /* Add entry */
 528         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 529                 (table->table_next_id_valid == 0)) {
 530                 table->table_next_id = entry->table_id;
 531                 table->table_next_id_valid = 1;
 532         }
 533
 534         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 535                 key_found, (void **) entry_ptr);
 536 }
 537
 538 int
 539 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 540                 uint32_t table_id,
 541                 void *key,
 542                 int *key_found,
 543                 struct rte_pipeline_table_entry *entry)
 544 {
 545         struct rte_table *table;
 546
 547         /* Check input arguments */
 548         if (p == NULL) {
 549                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 550                         __func__);
 551                 return -EINVAL;
 552         }
 553
 554         if (key == NULL) {
 555                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 556                         __func__);
 557                 return -EINVAL;
 558         }
 559
 560         if (table_id >= p->num_tables) {
 561                 RTE_LOG(ERR, PIPELINE,
 562                         "%s: table_id %d out of range\n", __func__, table_id);
 563                 return -EINVAL;
 564         }
 565
 566         table = &p->tables[table_id];
 567
 568         if (table->ops.f_delete == NULL) {
 569                 RTE_LOG(ERR, PIPELINE,
 570                         "%s: f_delete function pointer NULL\n", __func__);
 571                 return -EINVAL;
 572         }
 573
 574         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 575 }
 576
 577 /*
 578  * Port
 579  *
 580  */
 581 static int
 582 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 583                 struct rte_pipeline_port_in_params *params,
 584                 uint32_t *port_id)
 585 {
 586         if (p == NULL) {
 587                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 588                         __func__);
 589                 return -EINVAL;
 590         }
 591         if (params == NULL) {
 592                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 593                 return -EINVAL;
 594         }
 595         if (port_id == NULL) {
 596                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 597                         __func__);
 598                 return -EINVAL;
 599         }
 600
 601         /* ops */
 602         if (params->ops == NULL) {
 603                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 604                         __func__);
 605                 return -EINVAL;
 606         }
 607
 608         if (params->ops->f_create == NULL) {
 609                 RTE_LOG(ERR, PIPELINE,
 610                         "%s: f_create function pointer NULL\n", __func__);
 611                 return -EINVAL;
 612         }
 613
 614         if (params->ops->f_rx == NULL) {
 615                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 616                         __func__);
 617                 return -EINVAL;
 618         }
 619
 620         /* burst_size */
 621         if ((params->burst_size == 0) ||
 622                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 623                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 624                         __func__);
 625                 return -EINVAL;
 626         }
 627
 628         /* Do we have room for one more port? */
 629         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 630                 RTE_LOG(ERR, PIPELINE,
 631                         "%s: invalid value for num_ports_in\n", __func__);
 632                 return -EINVAL;
 633         }
 634
 635         return 0;
 636 }
 637
 638 static int
 639 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 640                 struct rte_pipeline_port_out_params *params,
 641                 uint32_t *port_id)
 642 {
 643         rte_pipeline_port_out_action_handler f_ah;
 644         rte_pipeline_port_out_action_handler_bulk f_ah_bulk;
 645
 646         if (p == NULL) {
 647                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 648                         __func__);
 649                 return -EINVAL;
 650         }
 651
 652         if (params == NULL) {
 653                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 654                 return -EINVAL;
 655         }
 656
 657         if (port_id == NULL) {
 658                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 659                         __func__);
 660                 return -EINVAL;
 661         }
 662
 663         /* ops */
 664         if (params->ops == NULL) {
 665                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 666                         __func__);
 667                 return -EINVAL;
 668         }
 669
 670         if (params->ops->f_create == NULL) {
 671                 RTE_LOG(ERR, PIPELINE,
 672                         "%s: f_create function pointer NULL\n", __func__);
 673                 return -EINVAL;
 674         }
 675
 676         if (params->ops->f_tx == NULL) {
 677                 RTE_LOG(ERR, PIPELINE,
 678                                 "%s: f_tx function pointer NULL\n", __func__);
 679                 return -EINVAL;
 680         }
 681
 682         if (params->ops->f_tx_bulk == NULL) {
 683                 RTE_LOG(ERR, PIPELINE,
 684                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 685                 return -EINVAL;
 686         }
 687
 688         f_ah = params->f_action;
 689         f_ah_bulk = params->f_action_bulk;
 690         if (((f_ah != NULL) && (f_ah_bulk == NULL)) ||
 691             ((f_ah == NULL) && (f_ah_bulk != NULL))) {
 692                 RTE_LOG(ERR, PIPELINE, "%s: Action handlers have to be either"
 693                         "both enabled or both disabled\n", __func__);
 694                 return -EINVAL;
 695         }
 696
 697         /* Do we have room for one more port? */
 698         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 699                 RTE_LOG(ERR, PIPELINE,
 700                         "%s: invalid value for num_ports_out\n", __func__);
 701                 return -EINVAL;
 702         }
 703
 704         return 0;
 705 }
 706
 707 int
 708 rte_pipeline_port_in_create(struct rte_pipeline *p,
 709                 struct rte_pipeline_port_in_params *params,
 710                 uint32_t *port_id)
 711 {
 712         struct rte_port_in *port;
 713         void *h_port;
 714         uint32_t id;
 715         int status;
 716
 717         /* Check input arguments */
 718         status = rte_pipeline_port_in_check_params(p, params, port_id);
 719         if (status != 0)
 720                 return status;
 721
 722         id = p->num_ports_in;
 723         port = &p->ports_in[id];
 724
 725         /* Create the port */
 726         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 727         if (h_port == NULL) {
 728                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 729                 return -EINVAL;
 730         }
 731
 732         /* Commit current table to the pipeline */
 733         p->num_ports_in++;
 734         *port_id = id;
 735
 736         /* Save input parameters */
 737         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 738         port->f_action = params->f_action;
 739         port->arg_ah = params->arg_ah;
 740         port->burst_size = params->burst_size;
 741
 742         /* Initialize port internal data structure */
 743         port->table_id = RTE_TABLE_INVALID;
 744         port->h_port = h_port;
 745         port->next = NULL;
 746
 747         return 0;
 748 }
 749
 750 void
 751 rte_pipeline_port_in_free(struct rte_port_in *port)
 752 {
 753         if (port->ops.f_free != NULL)
 754                 port->ops.f_free(port->h_port);
 755 }
 756
 757 int
 758 rte_pipeline_port_out_create(struct rte_pipeline *p,
 759                 struct rte_pipeline_port_out_params *params,
 760                 uint32_t *port_id)
 761 {
 762         struct rte_port_out *port;
 763         void *h_port;
 764         uint32_t id;
 765         int status;
 766
 767         /* Check input arguments */
 768         status = rte_pipeline_port_out_check_params(p, params, port_id);
 769         if (status != 0)
 770                 return status;
 771
 772         id = p->num_ports_out;
 773         port = &p->ports_out[id];
 774
 775         /* Create the port */
 776         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 777         if (h_port == NULL) {
 778                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 779                 return -EINVAL;
 780         }
 781
 782         /* Commit current table to the pipeline */
 783         p->num_ports_out++;
 784         *port_id = id;
 785
 786         /* Save input parameters */
 787         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 788         port->f_action = params->f_action;
 789         port->f_action_bulk = params->f_action_bulk;
 790         port->arg_ah = params->arg_ah;
 791
 792         /* Initialize port internal data structure */
 793         port->h_port = h_port;
 794
 795         return 0;
 796 }
 797
 798 void
 799 rte_pipeline_port_out_free(struct rte_port_out *port)
 800 {
 801         if (port->ops.f_free != NULL)
 802                 port->ops.f_free(port->h_port);
 803 }
 804
 805 int
 806 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 807                 uint32_t port_id,
 808                 uint32_t table_id)
 809 {
 810         struct rte_port_in *port;
 811
 812         /* Check input arguments */
 813         if (p == NULL) {
 814                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 815                         __func__);
 816                 return -EINVAL;
 817         }
 818
 819         if (port_id >= p->num_ports_in) {
 820                 RTE_LOG(ERR, PIPELINE,
 821                         "%s: port IN ID %u is out of range\n",
 822                         __func__, port_id);
 823                 return -EINVAL;
 824         }
 825
 826         if (table_id >= p->num_tables) {
 827                 RTE_LOG(ERR, PIPELINE,
 828                         "%s: Table ID %u is out of range\n",
 829                         __func__, table_id);
 830                 return -EINVAL;
 831         }
 832
 833         port = &p->ports_in[port_id];
 834         port->table_id = table_id;
 835
 836         return 0;
 837 }
 838
 839 int
 840 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 841 {
 842         struct rte_port_in *port, *port_prev, *port_next;
 843         struct rte_port_in *port_first, *port_last;
 844         uint64_t port_mask;
 845         uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
 846
 847         /* Check input arguments */
 848         if (p == NULL) {
 849                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 850                         __func__);
 851                 return -EINVAL;
 852         }
 853
 854         if (port_id >= p->num_ports_in) {
 855                 RTE_LOG(ERR, PIPELINE,
 856                         "%s: port IN ID %u is out of range\n",
 857                         __func__, port_id);
 858                 return -EINVAL;
 859         }
 860
 861         /* Return if current input port is already enabled */
 862         port_mask = 1LLU << port_id;
 863         if (p->enabled_port_in_mask & port_mask)
 864                 return 0;
 865
 866         p->enabled_port_in_mask |= port_mask;
 867
 868         /* Add current input port to the pipeline chain of enabled ports */
 869         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 870         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 871
 872         port_prev = &p->ports_in[port_prev_id];
 873         port_next = &p->ports_in[port_next_id];
 874         port = &p->ports_in[port_id];
 875
 876         port_prev->next = port;
 877         port->next = port_next;
 878
 879         /* Update the first and last input ports in the chain */
 880         port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
 881         port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
 882
 883         port_first = &p->ports_in[port_first_id];
 884         port_last = &p->ports_in[port_last_id];
 885
 886         p->port_in_first = port_first;
 887         port_last->next = NULL;
 888
 889         return 0;
 890 }
 891
 892 int
 893 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
 894 {
 895         struct rte_port_in *port_prev, *port_next, *port_first, *port_last;
 896         uint64_t port_mask;
 897         uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
 898
 899         /* Check input arguments */
 900         if (p == NULL) {
 901                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 902                 __func__);
 903                 return -EINVAL;
 904         }
 905
 906         if (port_id >= p->num_ports_in) {
 907                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
 908                         __func__, port_id);
 909                 return -EINVAL;
 910         }
 911
 912         /* Return if current input port is already disabled */
 913         port_mask = 1LLU << port_id;
 914         if ((p->enabled_port_in_mask & port_mask) == 0)
 915                 return 0;
 916
 917         /* Return if no other enabled ports */
 918         if (__builtin_popcountll(p->enabled_port_in_mask) == 1) {
 919                 p->enabled_port_in_mask &= ~port_mask;
 920                 p->port_in_first = NULL;
 921
 922                 return 0;
 923         }
 924
 925         /* Add current input port to the pipeline chain of enabled ports */
 926         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 927         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 928
 929         port_prev = &p->ports_in[port_prev_id];
 930         port_next = &p->ports_in[port_next_id];
 931
 932         port_prev->next = port_next;
 933         p->enabled_port_in_mask &= ~port_mask;
 934
 935         /* Update the first and last input ports in the chain */
 936         port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
 937         port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
 938
 939         port_first = &p->ports_in[port_first_id];
 940         port_last = &p->ports_in[port_last_id];
 941
 942         p->port_in_first = port_first;
 943         port_last->next = NULL;
 944
 945         return 0;
 946 }
 947
 948 /*
 949  * Pipeline run-time
 950  *
 951  */
 952 int
 953 rte_pipeline_check(struct rte_pipeline *p)
 954 {
 955         uint32_t port_in_id;
 956
 957         /* Check input arguments */
 958         if (p == NULL) {
 959                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 960                         __func__);
 961                 return -EINVAL;
 962         }
 963
 964         /* Check that pipeline has at least one input port, one table and one
 965         output port */
 966         if (p->num_ports_in == 0) {
 967                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
 968                         __func__);
 969                 return -EINVAL;
 970         }
 971         if (p->num_tables == 0) {
 972                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
 973                         __func__);
 974                 return -EINVAL;
 975         }
 976         if (p->num_ports_out == 0) {
 977                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
 978                         __func__);
 979                 return -EINVAL;
 980         }
 981
 982         /* Check that all input ports are connected */
 983         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
 984                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
 985
 986                 if (port_in->table_id == RTE_TABLE_INVALID) {
 987                         RTE_LOG(ERR, PIPELINE,
 988                                 "%s: Port IN ID %u is not connected\n",
 989                                 __func__, port_in_id);
 990                         return -EINVAL;
 991                 }
 992         }
 993
 994         return 0;
 995 }
 996
 997 static inline void
 998 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
 999 {
1000         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1001         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1002         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1003
1004         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1005                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1006                 uint32_t i;
1007
1008                 for (i = 0; i < n_pkts; i++) {
1009                         uint64_t pkt_mask = 1LLU << i;
1010                         uint32_t pos = p->entries[i]->action;
1011
1012                         p->action_mask1[pos] |= pkt_mask;
1013                 }
1014         } else {
1015                 uint32_t i;
1016
1017                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1018                         uint64_t pkt_mask = 1LLU << i;
1019                         uint32_t pos;
1020
1021                         if ((pkt_mask & pkts_mask) == 0)
1022                                 continue;
1023
1024                         pos = p->entries[i]->action;
1025                         p->action_mask1[pos] |= pkt_mask;
1026                 }
1027         }
1028 }
1029
1030 static inline void
1031 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1032                 uint64_t pkts_mask, uint32_t port_id)
1033 {
1034         struct rte_port_out *port_out = &p->ports_out[port_id];
1035
1036         /* Output port user actions */
1037         if (port_out->f_action_bulk != NULL) {
1038                 uint64_t mask = pkts_mask;
1039
1040                 port_out->f_action_bulk(p->pkts, &pkts_mask, port_out->arg_ah);
1041                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask ^  mask;
1042         }
1043
1044         /* Output port TX */
1045         if (pkts_mask != 0)
1046                 port_out->ops.f_tx_bulk(port_out->h_port, p->pkts, pkts_mask);
1047 }
1048
1049 static inline void
1050 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1051 {
1052         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1053                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1054                 uint32_t i;
1055
1056                 for (i = 0; i < n_pkts; i++) {
1057                         struct rte_mbuf *pkt = p->pkts[i];
1058                         uint32_t port_out_id = p->entries[i]->port_id;
1059                         struct rte_port_out *port_out =
1060                                 &p->ports_out[port_out_id];
1061
1062                         /* Output port user actions */
1063                         if (port_out->f_action == NULL) /* Output port TX */
1064                                 port_out->ops.f_tx(port_out->h_port, pkt);
1065                         else {
1066                                 uint64_t pkt_mask = 1LLU;
1067
1068                                 port_out->f_action(pkt, &pkt_mask,
1069                                         port_out->arg_ah);
1070                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1071                                         (pkt_mask ^ 1LLU) << i;
1072
1073                                 /* Output port TX */
1074                                 if (pkt_mask != 0)
1075                                         port_out->ops.f_tx(port_out->h_port,
1076                                                 pkt);
1077                         }
1078                 }
1079         } else {
1080                 uint32_t i;
1081
1082                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1083                         uint64_t pkt_mask = 1LLU << i;
1084                         struct rte_mbuf *pkt;
1085                         struct rte_port_out *port_out;
1086                         uint32_t port_out_id;
1087
1088                         if ((pkt_mask & pkts_mask) == 0)
1089                                 continue;
1090
1091                         pkt = p->pkts[i];
1092                         port_out_id = p->entries[i]->port_id;
1093                         port_out = &p->ports_out[port_out_id];
1094
1095                         /* Output port user actions */
1096                         if (port_out->f_action == NULL) /* Output port TX */
1097                                 port_out->ops.f_tx(port_out->h_port, pkt);
1098                         else {
1099                                 pkt_mask = 1LLU;
1100
1101                                 port_out->f_action(pkt, &pkt_mask,
1102                                         port_out->arg_ah);
1103                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1104                                         (pkt_mask ^ 1LLU) << i;
1105
1106                                 /* Output port TX */
1107                                 if (pkt_mask != 0)
1108                                         port_out->ops.f_tx(port_out->h_port,
1109                                                 pkt);
1110                         }
1111                 }
1112         }
1113 }
1114
1115 static inline void
1116 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1117         uint64_t pkts_mask)
1118 {
1119         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1120                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1121                 uint32_t i;
1122
1123                 for (i = 0; i < n_pkts; i++) {
1124                         struct rte_mbuf *pkt = p->pkts[i];
1125                         uint32_t port_out_id =
1126                                 RTE_MBUF_METADATA_UINT32(pkt,
1127                                         p->offset_port_id);
1128                         struct rte_port_out *port_out = &p->ports_out[
1129                                 port_out_id];
1130
1131                         /* Output port user actions */
1132                         if (port_out->f_action == NULL) /* Output port TX */
1133                                 port_out->ops.f_tx(port_out->h_port, pkt);
1134                         else {
1135                                 uint64_t pkt_mask = 1LLU;
1136
1137                                 port_out->f_action(pkt, &pkt_mask,
1138                                         port_out->arg_ah);
1139                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1140                                         (pkt_mask ^ 1LLU) << i;
1141
1142                                 /* Output port TX */
1143                                 if (pkt_mask != 0)
1144                                         port_out->ops.f_tx(port_out->h_port,
1145                                                 pkt);
1146                         }
1147                 }
1148         } else {
1149                 uint32_t i;
1150
1151                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1152                         uint64_t pkt_mask = 1LLU << i;
1153                         struct rte_mbuf *pkt;
1154                         struct rte_port_out *port_out;
1155                         uint32_t port_out_id;
1156
1157                         if ((pkt_mask & pkts_mask) == 0)
1158                                 continue;
1159
1160                         pkt = p->pkts[i];
1161                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1162                                 p->offset_port_id);
1163                         port_out = &p->ports_out[port_out_id];
1164
1165                         /* Output port user actions */
1166                         if (port_out->f_action == NULL) /* Output port TX */
1167                                 port_out->ops.f_tx(port_out->h_port, pkt);
1168                         else {
1169                                 pkt_mask = 1LLU;
1170
1171                                 port_out->f_action(pkt, &pkt_mask,
1172                                         port_out->arg_ah);
1173                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1174                                         (pkt_mask ^ 1LLU) << i;
1175
1176                                 /* Output port TX */
1177                                 if (pkt_mask != 0)
1178                                         port_out->ops.f_tx(port_out->h_port,
1179                                                 pkt);
1180                         }
1181                 }
1182         }
1183 }
1184
1185 static inline void
1186 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1187 {
1188         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1189                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1190                 uint32_t i;
1191
1192                 for (i = 0; i < n_pkts; i++)
1193                         rte_pktmbuf_free(p->pkts[i]);
1194         } else {
1195                 uint32_t i;
1196
1197                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1198                         uint64_t pkt_mask = 1LLU << i;
1199
1200                         if ((pkt_mask & pkts_mask) == 0)
1201                                 continue;
1202
1203                         rte_pktmbuf_free(p->pkts[i]);
1204                 }
1205         }
1206 }
1207
1208 int
1209 rte_pipeline_run(struct rte_pipeline *p)
1210 {
1211         struct rte_port_in *port_in;
1212
1213         for (port_in = p->port_in_first; port_in != NULL;
1214                 port_in = port_in->next) {
1215                 uint64_t pkts_mask;
1216                 uint32_t n_pkts, table_id;
1217
1218                 /* Input port RX */
1219                 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1220                         port_in->burst_size);
1221                 if (n_pkts == 0)
1222                         continue;
1223
1224                 pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1225                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1226                 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1227                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1228
1229                 /* Input port user actions */
1230                 if (port_in->f_action != NULL) {
1231                         uint64_t mask = pkts_mask;
1232
1233                         port_in->f_action(p->pkts, n_pkts, &pkts_mask,
1234                                 port_in->arg_ah);
1235                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1236                                 pkts_mask ^ mask;
1237                 }
1238
1239                 /* Table */
1240                 for (table_id = port_in->table_id; pkts_mask != 0; ) {
1241                         struct rte_table *table;
1242                         uint64_t lookup_hit_mask, lookup_miss_mask;
1243
1244                         /* Lookup */
1245                         table = &p->tables[table_id];
1246                         table->ops.f_lookup(table->h_table, p->pkts, pkts_mask,
1247                                         &lookup_hit_mask, (void **) p->entries);
1248                         lookup_miss_mask = pkts_mask & (~lookup_hit_mask);
1249
1250                         /* Lookup miss */
1251                         if (lookup_miss_mask != 0) {
1252                                 struct rte_pipeline_table_entry *default_entry =
1253                                         table->default_entry;
1254
1255                                 /* Table user actions */
1256                                 if (table->f_action_miss != NULL) {
1257                                         uint64_t mask = lookup_miss_mask;
1258
1259                                         table->f_action_miss(p->pkts,
1260                                                 &lookup_miss_mask,
1261                                                 default_entry, table->arg_ah);
1262                                         p->action_mask0[
1263                                                 RTE_PIPELINE_ACTION_DROP] |=
1264                                                 lookup_miss_mask ^ mask;
1265                                 }
1266
1267                                 /* Table reserved actions */
1268                                 if ((default_entry->action ==
1269                                         RTE_PIPELINE_ACTION_PORT) &&
1270                                         (lookup_miss_mask != 0))
1271                                         rte_pipeline_action_handler_port_bulk(p,
1272                                                 lookup_miss_mask,
1273                                                 default_entry->port_id);
1274                                 else {
1275                                         uint32_t pos = default_entry->action;
1276
1277                                         p->action_mask0[pos] = lookup_miss_mask;
1278                                 }
1279                         }
1280
1281                         /* Lookup hit */
1282                         if (lookup_hit_mask != 0) {
1283                                 /* Table user actions */
1284                                 if (table->f_action_hit != NULL) {
1285                                         uint64_t mask = lookup_hit_mask;
1286
1287                                         table->f_action_hit(p->pkts,
1288                                                 &lookup_hit_mask,
1289                                                 p->entries, table->arg_ah);
1290                                         p->action_mask0[
1291                                                 RTE_PIPELINE_ACTION_DROP] |=
1292                                                 lookup_hit_mask ^ mask;
1293                                 }
1294
1295                                 /* Table reserved actions */
1296                                 rte_pipeline_compute_masks(p, lookup_hit_mask);
1297                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1298                                         p->action_mask1[
1299                                                 RTE_PIPELINE_ACTION_DROP];
1300                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1301                                         p->action_mask1[
1302                                                 RTE_PIPELINE_ACTION_PORT];
1303                                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1304                                         p->action_mask1[
1305                                                 RTE_PIPELINE_ACTION_TABLE];
1306                         }
1307
1308                         /* Prepare for next iteration */
1309                         pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1310                         table_id = table->table_next_id;
1311                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1312                 }
1313
1314                 /* Table reserved action PORT */
1315                 rte_pipeline_action_handler_port(p,
1316                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1317
1318                 /* Table reserved action PORT META */
1319                 rte_pipeline_action_handler_port_meta(p,
1320                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1321
1322                 /* Table reserved action DROP */
1323                 rte_pipeline_action_handler_drop(p,
1324                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1325         }
1326
1327         return 0;
1328 }
1329
1330 int
1331 rte_pipeline_flush(struct rte_pipeline *p)
1332 {
1333         uint32_t port_id;
1334
1335         /* Check input arguments */
1336         if (p == NULL) {
1337                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1338                         __func__);
1339                 return -EINVAL;
1340         }
1341
1342         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1343                 struct rte_port_out *port = &p->ports_out[port_id];
1344
1345                 if (port->ops.f_flush != NULL)
1346                         port->ops.f_flush(port->h_port);
1347         }
1348
1349         return 0;
1350 }
1351
1352 int
1353 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1354                 uint32_t port_id, struct rte_mbuf *pkt)
1355 {
1356         struct rte_port_out *port_out = &p->ports_out[port_id];
1357
1358         /* Output port user actions */
1359         if (port_out->f_action == NULL)
1360                 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1361         else {
1362                 uint64_t pkt_mask = 1LLU;
1363
1364                 port_out->f_action(pkt, &pkt_mask, port_out->arg_ah);
1365
1366                 if (pkt_mask != 0) /* Output port TX */
1367                         port_out->ops.f_tx(port_out->h_port, pkt);
1368                 else
1369                         rte_pktmbuf_free(pkt);
1370         }
1371
1372         return 0;
1373 }