lib/librte_pipeline/rte_pipeline.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_memory.h>
  39 #include <rte_memzone.h>
  40 #include <rte_cycles.h>
  41 #include <rte_prefetch.h>
  42 #include <rte_branch_prediction.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_malloc.h>
  45 #include <rte_string_fns.h>
  46
  47 #include "rte_pipeline.h"
  48
  49 #define RTE_TABLE_INVALID                                 UINT32_MAX
  50
  51 #ifdef RTE_PIPELINE_STATS_COLLECT
  52 #define RTE_PIPELINE_STATS_ADD(counter, val) \
  53         ({ (counter) += (val); })
  54
  55 #define RTE_PIPELINE_STATS_ADD_M(counter, mask) \
  56         ({ (counter) += __builtin_popcountll(mask); })
  57 #else
  58 #define RTE_PIPELINE_STATS_ADD(counter, val)
  59 #define RTE_PIPELINE_STATS_ADD_M(counter, mask)
  60 #endif
  61
  62 struct rte_port_in {
  63         /* Input parameters */
  64         struct rte_port_in_ops ops;
  65         rte_pipeline_port_in_action_handler f_action;
  66         void *arg_ah;
  67         uint32_t burst_size;
  68
  69         /* The table to which this port is connected */
  70         uint32_t table_id;
  71
  72         /* Handle to low-level port */
  73         void *h_port;
  74
  75         /* List of enabled ports */
  76         struct rte_port_in *next;
  77
  78         uint64_t n_pkts_dropped_by_ah;
  79 };
  80
  81 struct rte_port_out {
  82         /* Input parameters */
  83         struct rte_port_out_ops ops;
  84         rte_pipeline_port_out_action_handler f_action;
  85         rte_pipeline_port_out_action_handler_bulk f_action_bulk;
  86         void *arg_ah;
  87
  88         /* Handle to low-level port */
  89         void *h_port;
  90
  91         uint64_t n_pkts_dropped_by_ah;
  92 };
  93
  94 struct rte_table {
  95         /* Input parameters */
  96         struct rte_table_ops ops;
  97         rte_pipeline_table_action_handler_hit f_action_hit;
  98         rte_pipeline_table_action_handler_miss f_action_miss;
  99         void *arg_ah;
 100         struct rte_pipeline_table_entry *default_entry;
 101         uint32_t entry_size;
 102
 103         uint32_t table_next_id;
 104         uint32_t table_next_id_valid;
 105
 106         /* Handle to the low-level table object */
 107         void *h_table;
 108
 109         /* Stats for this table. */
 110         uint64_t n_pkts_dropped_by_lkp_hit_ah;
 111         uint64_t n_pkts_dropped_by_lkp_miss_ah;
 112         uint64_t n_pkts_dropped_lkp_hit;
 113         uint64_t n_pkts_dropped_lkp_miss;
 114 };
 115
 116 #define RTE_PIPELINE_MAX_NAME_SZ                           124
 117
 118 struct rte_pipeline {
 119         /* Input parameters */
 120         char name[RTE_PIPELINE_MAX_NAME_SZ];
 121         int socket_id;
 122         uint32_t offset_port_id;
 123
 124         /* Internal tables */
 125         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 126         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 127         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 128
 129         /* Occupancy of internal tables */
 130         uint32_t num_ports_in;
 131         uint32_t num_ports_out;
 132         uint32_t num_tables;
 133
 134         /* List of enabled ports */
 135         uint64_t enabled_port_in_mask;
 136         struct rte_port_in *port_in_first;
 137
 138         /* Pipeline run structures */
 139         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 140         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 141         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 142         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 143 } __rte_cache_aligned;
 144
 145 static inline uint32_t
 146 rte_mask_get_next(uint64_t mask, uint32_t pos)
 147 {
 148         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 149                         (mask >> ((pos + 1) & 0x3F));
 150         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 151 }
 152
 153 static inline uint32_t
 154 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 155 {
 156         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 157                         (mask << ((64 - pos) & 0x3F));
 158         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 159 }
 160
 161 static void
 162 rte_pipeline_table_free(struct rte_table *table);
 163
 164 static void
 165 rte_pipeline_port_in_free(struct rte_port_in *port);
 166
 167 static void
 168 rte_pipeline_port_out_free(struct rte_port_out *port);
 169
 170 /*
 171  * Pipeline
 172  *
 173  */
 174 static int
 175 rte_pipeline_check_params(struct rte_pipeline_params *params)
 176 {
 177         if (params == NULL) {
 178                 RTE_LOG(ERR, PIPELINE,
 179                         "%s: Incorrect value for parameter params\n", __func__);
 180                 return -EINVAL;
 181         }
 182
 183         /* name */
 184         if (params->name == NULL) {
 185                 RTE_LOG(ERR, PIPELINE,
 186                         "%s: Incorrect value for parameter name\n", __func__);
 187                 return -EINVAL;
 188         }
 189
 190         /* socket */
 191         if ((params->socket_id < 0) ||
 192             (params->socket_id >= RTE_MAX_NUMA_NODES)) {
 193                 RTE_LOG(ERR, PIPELINE,
 194                         "%s: Incorrect value for parameter socket_id\n",
 195                         __func__);
 196                 return -EINVAL;
 197         }
 198
 199         return 0;
 200 }
 201
 202 struct rte_pipeline *
 203 rte_pipeline_create(struct rte_pipeline_params *params)
 204 {
 205         struct rte_pipeline *p;
 206         int status;
 207
 208         /* Check input parameters */
 209         status = rte_pipeline_check_params(params);
 210         if (status != 0) {
 211                 RTE_LOG(ERR, PIPELINE,
 212                         "%s: Pipeline params check failed (%d)\n",
 213                         __func__, status);
 214                 return NULL;
 215         }
 216
 217         /* Allocate memory for the pipeline on requested socket */
 218         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 219                         RTE_CACHE_LINE_SIZE, params->socket_id);
 220
 221         if (p == NULL) {
 222                 RTE_LOG(ERR, PIPELINE,
 223                         "%s: Pipeline memory allocation failed\n", __func__);
 224                 return NULL;
 225         }
 226
 227         /* Save input parameters */
 228         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
 229         p->socket_id = params->socket_id;
 230         p->offset_port_id = params->offset_port_id;
 231
 232         /* Initialize pipeline internal data structure */
 233         p->num_ports_in = 0;
 234         p->num_ports_out = 0;
 235         p->num_tables = 0;
 236         p->enabled_port_in_mask = 0;
 237         p->port_in_first = NULL;
 238
 239         return p;
 240 }
 241
 242 int
 243 rte_pipeline_free(struct rte_pipeline *p)
 244 {
 245         uint32_t i;
 246
 247         /* Check input parameters */
 248         if (p == NULL) {
 249                 RTE_LOG(ERR, PIPELINE,
 250                         "%s: rte_pipeline parameter is NULL\n", __func__);
 251                 return -EINVAL;
 252         }
 253
 254         /* Free input ports */
 255         for (i = 0; i < p->num_ports_in; i++) {
 256                 struct rte_port_in *port = &p->ports_in[i];
 257
 258                 rte_pipeline_port_in_free(port);
 259         }
 260
 261         /* Free tables */
 262         for (i = 0; i < p->num_tables; i++) {
 263                 struct rte_table *table = &p->tables[i];
 264
 265                 rte_pipeline_table_free(table);
 266         }
 267
 268         /* Free output ports */
 269         for (i = 0; i < p->num_ports_out; i++) {
 270                 struct rte_port_out *port = &p->ports_out[i];
 271
 272                 rte_pipeline_port_out_free(port);
 273         }
 274
 275         /* Free pipeline memory */
 276         rte_free(p);
 277
 278         return 0;
 279 }
 280
 281 /*
 282  * Table
 283  *
 284  */
 285 static int
 286 rte_table_check_params(struct rte_pipeline *p,
 287                 struct rte_pipeline_table_params *params,
 288                 uint32_t *table_id)
 289 {
 290         if (p == NULL) {
 291                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 292                         __func__);
 293                 return -EINVAL;
 294         }
 295         if (params == NULL) {
 296                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 297                         __func__);
 298                 return -EINVAL;
 299         }
 300         if (table_id == NULL) {
 301                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 302                         __func__);
 303                 return -EINVAL;
 304         }
 305
 306         /* ops */
 307         if (params->ops == NULL) {
 308                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 309                         __func__);
 310                 return -EINVAL;
 311         }
 312
 313         if (params->ops->f_create == NULL) {
 314                 RTE_LOG(ERR, PIPELINE,
 315                         "%s: f_create function pointer is NULL\n", __func__);
 316                 return -EINVAL;
 317         }
 318
 319         if (params->ops->f_lookup == NULL) {
 320                 RTE_LOG(ERR, PIPELINE,
 321                         "%s: f_lookup function pointer is NULL\n", __func__);
 322                 return -EINVAL;
 323         }
 324
 325         /* De we have room for one more table? */
 326         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 327                 RTE_LOG(ERR, PIPELINE,
 328                         "%s: Incorrect value for num_tables parameter\n",
 329                         __func__);
 330                 return -EINVAL;
 331         }
 332
 333         return 0;
 334 }
 335
 336 int
 337 rte_pipeline_table_create(struct rte_pipeline *p,
 338                 struct rte_pipeline_table_params *params,
 339                 uint32_t *table_id)
 340 {
 341         struct rte_table *table;
 342         struct rte_pipeline_table_entry *default_entry;
 343         void *h_table;
 344         uint32_t entry_size, id;
 345         int status;
 346
 347         /* Check input arguments */
 348         status = rte_table_check_params(p, params, table_id);
 349         if (status != 0)
 350                 return status;
 351
 352         id = p->num_tables;
 353         table = &p->tables[id];
 354
 355         /* Allocate space for the default table entry */
 356         entry_size = sizeof(struct rte_pipeline_table_entry) +
 357                 params->action_data_size;
 358         default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
 359                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 360         if (default_entry == NULL) {
 361                 RTE_LOG(ERR, PIPELINE,
 362                         "%s: Failed to allocate default entry\n", __func__);
 363                 return -EINVAL;
 364         }
 365
 366         /* Create the table */
 367         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 368                 entry_size);
 369         if (h_table == NULL) {
 370                 rte_free(default_entry);
 371                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 372                 return -EINVAL;
 373         }
 374
 375         /* Commit current table to the pipeline */
 376         p->num_tables++;
 377         *table_id = id;
 378
 379         /* Save input parameters */
 380         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 381         table->f_action_hit = params->f_action_hit;
 382         table->f_action_miss = params->f_action_miss;
 383         table->arg_ah = params->arg_ah;
 384         table->entry_size = entry_size;
 385
 386         /* Clear the lookup miss actions (to be set later through API) */
 387         table->default_entry = default_entry;
 388         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 389
 390         /* Initialize table internal data structure */
 391         table->h_table = h_table;
 392         table->table_next_id = 0;
 393         table->table_next_id_valid = 0;
 394
 395         return 0;
 396 }
 397
 398 void
 399 rte_pipeline_table_free(struct rte_table *table)
 400 {
 401         if (table->ops.f_free != NULL)
 402                 table->ops.f_free(table->h_table);
 403
 404         rte_free(table->default_entry);
 405 }
 406
 407 int
 408 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 409         uint32_t table_id,
 410         struct rte_pipeline_table_entry *default_entry,
 411         struct rte_pipeline_table_entry **default_entry_ptr)
 412 {
 413         struct rte_table *table;
 414
 415         /* Check input arguments */
 416         if (p == NULL) {
 417                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 418                         __func__);
 419                 return -EINVAL;
 420         }
 421
 422         if (default_entry == NULL) {
 423                 RTE_LOG(ERR, PIPELINE,
 424                         "%s: default_entry parameter is NULL\n", __func__);
 425                 return -EINVAL;
 426         }
 427
 428         if (table_id >= p->num_tables) {
 429                 RTE_LOG(ERR, PIPELINE,
 430                         "%s: table_id %d out of range\n", __func__, table_id);
 431                 return -EINVAL;
 432         }
 433
 434         table = &p->tables[table_id];
 435
 436         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 437                 table->table_next_id_valid &&
 438                 (default_entry->table_id != table->table_next_id)) {
 439                 RTE_LOG(ERR, PIPELINE,
 440                         "%s: Tree-like topologies not allowed\n", __func__);
 441                 return -EINVAL;
 442         }
 443
 444         /* Set the lookup miss actions */
 445         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 446                 (table->table_next_id_valid == 0)) {
 447                 table->table_next_id = default_entry->table_id;
 448                 table->table_next_id_valid = 1;
 449         }
 450
 451         memcpy(table->default_entry, default_entry, table->entry_size);
 452
 453         *default_entry_ptr = table->default_entry;
 454         return 0;
 455 }
 456
 457 int
 458 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 459                 uint32_t table_id,
 460                 struct rte_pipeline_table_entry *entry)
 461 {
 462         struct rte_table *table;
 463
 464         /* Check input arguments */
 465         if (p == NULL) {
 466                 RTE_LOG(ERR, PIPELINE,
 467                         "%s: pipeline parameter is NULL\n", __func__);
 468                 return -EINVAL;
 469         }
 470
 471         if (table_id >= p->num_tables) {
 472                 RTE_LOG(ERR, PIPELINE,
 473                         "%s: table_id %d out of range\n", __func__, table_id);
 474                 return -EINVAL;
 475         }
 476
 477         table = &p->tables[table_id];
 478
 479         /* Save the current contents of the default entry */
 480         if (entry)
 481                 memcpy(entry, table->default_entry, table->entry_size);
 482
 483         /* Clear the lookup miss actions */
 484         memset(table->default_entry, 0, table->entry_size);
 485         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 486
 487         return 0;
 488 }
 489
 490 int
 491 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 492                 uint32_t table_id,
 493                 void *key,
 494                 struct rte_pipeline_table_entry *entry,
 495                 int *key_found,
 496                 struct rte_pipeline_table_entry **entry_ptr)
 497 {
 498         struct rte_table *table;
 499
 500         /* Check input arguments */
 501         if (p == NULL) {
 502                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 503                         __func__);
 504                 return -EINVAL;
 505         }
 506
 507         if (key == NULL) {
 508                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 509                 return -EINVAL;
 510         }
 511
 512         if (entry == NULL) {
 513                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 514                         __func__);
 515                 return -EINVAL;
 516         }
 517
 518         if (table_id >= p->num_tables) {
 519                 RTE_LOG(ERR, PIPELINE,
 520                         "%s: table_id %d out of range\n", __func__, table_id);
 521                 return -EINVAL;
 522         }
 523
 524         table = &p->tables[table_id];
 525
 526         if (table->ops.f_add == NULL) {
 527                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 528                         __func__);
 529                 return -EINVAL;
 530         }
 531
 532         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 533                 table->table_next_id_valid &&
 534                 (entry->table_id != table->table_next_id)) {
 535                 RTE_LOG(ERR, PIPELINE,
 536                         "%s: Tree-like topologies not allowed\n", __func__);
 537                 return -EINVAL;
 538         }
 539
 540         /* Add entry */
 541         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 542                 (table->table_next_id_valid == 0)) {
 543                 table->table_next_id = entry->table_id;
 544                 table->table_next_id_valid = 1;
 545         }
 546
 547         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 548                 key_found, (void **) entry_ptr);
 549 }
 550
 551 int
 552 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 553                 uint32_t table_id,
 554                 void *key,
 555                 int *key_found,
 556                 struct rte_pipeline_table_entry *entry)
 557 {
 558         struct rte_table *table;
 559
 560         /* Check input arguments */
 561         if (p == NULL) {
 562                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 563                         __func__);
 564                 return -EINVAL;
 565         }
 566
 567         if (key == NULL) {
 568                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 569                         __func__);
 570                 return -EINVAL;
 571         }
 572
 573         if (table_id >= p->num_tables) {
 574                 RTE_LOG(ERR, PIPELINE,
 575                         "%s: table_id %d out of range\n", __func__, table_id);
 576                 return -EINVAL;
 577         }
 578
 579         table = &p->tables[table_id];
 580
 581         if (table->ops.f_delete == NULL) {
 582                 RTE_LOG(ERR, PIPELINE,
 583                         "%s: f_delete function pointer NULL\n", __func__);
 584                 return -EINVAL;
 585         }
 586
 587         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 588 }
 589
 590 /*
 591  * Port
 592  *
 593  */
 594 static int
 595 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 596                 struct rte_pipeline_port_in_params *params,
 597                 uint32_t *port_id)
 598 {
 599         if (p == NULL) {
 600                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 601                         __func__);
 602                 return -EINVAL;
 603         }
 604         if (params == NULL) {
 605                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 606                 return -EINVAL;
 607         }
 608         if (port_id == NULL) {
 609                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 610                         __func__);
 611                 return -EINVAL;
 612         }
 613
 614         /* ops */
 615         if (params->ops == NULL) {
 616                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 617                         __func__);
 618                 return -EINVAL;
 619         }
 620
 621         if (params->ops->f_create == NULL) {
 622                 RTE_LOG(ERR, PIPELINE,
 623                         "%s: f_create function pointer NULL\n", __func__);
 624                 return -EINVAL;
 625         }
 626
 627         if (params->ops->f_rx == NULL) {
 628                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 629                         __func__);
 630                 return -EINVAL;
 631         }
 632
 633         /* burst_size */
 634         if ((params->burst_size == 0) ||
 635                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 636                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 637                         __func__);
 638                 return -EINVAL;
 639         }
 640
 641         /* Do we have room for one more port? */
 642         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 643                 RTE_LOG(ERR, PIPELINE,
 644                         "%s: invalid value for num_ports_in\n", __func__);
 645                 return -EINVAL;
 646         }
 647
 648         return 0;
 649 }
 650
 651 static int
 652 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 653                 struct rte_pipeline_port_out_params *params,
 654                 uint32_t *port_id)
 655 {
 656         rte_pipeline_port_out_action_handler f_ah;
 657         rte_pipeline_port_out_action_handler_bulk f_ah_bulk;
 658
 659         if (p == NULL) {
 660                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 661                         __func__);
 662                 return -EINVAL;
 663         }
 664
 665         if (params == NULL) {
 666                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 667                 return -EINVAL;
 668         }
 669
 670         if (port_id == NULL) {
 671                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 672                         __func__);
 673                 return -EINVAL;
 674         }
 675
 676         /* ops */
 677         if (params->ops == NULL) {
 678                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 679                         __func__);
 680                 return -EINVAL;
 681         }
 682
 683         if (params->ops->f_create == NULL) {
 684                 RTE_LOG(ERR, PIPELINE,
 685                         "%s: f_create function pointer NULL\n", __func__);
 686                 return -EINVAL;
 687         }
 688
 689         if (params->ops->f_tx == NULL) {
 690                 RTE_LOG(ERR, PIPELINE,
 691                                 "%s: f_tx function pointer NULL\n", __func__);
 692                 return -EINVAL;
 693         }
 694
 695         if (params->ops->f_tx_bulk == NULL) {
 696                 RTE_LOG(ERR, PIPELINE,
 697                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 698                 return -EINVAL;
 699         }
 700
 701         f_ah = params->f_action;
 702         f_ah_bulk = params->f_action_bulk;
 703         if (((f_ah != NULL) && (f_ah_bulk == NULL)) ||
 704             ((f_ah == NULL) && (f_ah_bulk != NULL))) {
 705                 RTE_LOG(ERR, PIPELINE, "%s: Action handlers have to be either"
 706                         "both enabled or both disabled\n", __func__);
 707                 return -EINVAL;
 708         }
 709
 710         /* Do we have room for one more port? */
 711         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 712                 RTE_LOG(ERR, PIPELINE,
 713                         "%s: invalid value for num_ports_out\n", __func__);
 714                 return -EINVAL;
 715         }
 716
 717         return 0;
 718 }
 719
 720 int
 721 rte_pipeline_port_in_create(struct rte_pipeline *p,
 722                 struct rte_pipeline_port_in_params *params,
 723                 uint32_t *port_id)
 724 {
 725         struct rte_port_in *port;
 726         void *h_port;
 727         uint32_t id;
 728         int status;
 729
 730         /* Check input arguments */
 731         status = rte_pipeline_port_in_check_params(p, params, port_id);
 732         if (status != 0)
 733                 return status;
 734
 735         id = p->num_ports_in;
 736         port = &p->ports_in[id];
 737
 738         /* Create the port */
 739         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 740         if (h_port == NULL) {
 741                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 742                 return -EINVAL;
 743         }
 744
 745         /* Commit current table to the pipeline */
 746         p->num_ports_in++;
 747         *port_id = id;
 748
 749         /* Save input parameters */
 750         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 751         port->f_action = params->f_action;
 752         port->arg_ah = params->arg_ah;
 753         port->burst_size = params->burst_size;
 754
 755         /* Initialize port internal data structure */
 756         port->table_id = RTE_TABLE_INVALID;
 757         port->h_port = h_port;
 758         port->next = NULL;
 759
 760         return 0;
 761 }
 762
 763 void
 764 rte_pipeline_port_in_free(struct rte_port_in *port)
 765 {
 766         if (port->ops.f_free != NULL)
 767                 port->ops.f_free(port->h_port);
 768 }
 769
 770 int
 771 rte_pipeline_port_out_create(struct rte_pipeline *p,
 772                 struct rte_pipeline_port_out_params *params,
 773                 uint32_t *port_id)
 774 {
 775         struct rte_port_out *port;
 776         void *h_port;
 777         uint32_t id;
 778         int status;
 779
 780         /* Check input arguments */
 781         status = rte_pipeline_port_out_check_params(p, params, port_id);
 782         if (status != 0)
 783                 return status;
 784
 785         id = p->num_ports_out;
 786         port = &p->ports_out[id];
 787
 788         /* Create the port */
 789         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 790         if (h_port == NULL) {
 791                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 792                 return -EINVAL;
 793         }
 794
 795         /* Commit current table to the pipeline */
 796         p->num_ports_out++;
 797         *port_id = id;
 798
 799         /* Save input parameters */
 800         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 801         port->f_action = params->f_action;
 802         port->f_action_bulk = params->f_action_bulk;
 803         port->arg_ah = params->arg_ah;
 804
 805         /* Initialize port internal data structure */
 806         port->h_port = h_port;
 807
 808         return 0;
 809 }
 810
 811 void
 812 rte_pipeline_port_out_free(struct rte_port_out *port)
 813 {
 814         if (port->ops.f_free != NULL)
 815                 port->ops.f_free(port->h_port);
 816 }
 817
 818 int
 819 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 820                 uint32_t port_id,
 821                 uint32_t table_id)
 822 {
 823         struct rte_port_in *port;
 824
 825         /* Check input arguments */
 826         if (p == NULL) {
 827                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 828                         __func__);
 829                 return -EINVAL;
 830         }
 831
 832         if (port_id >= p->num_ports_in) {
 833                 RTE_LOG(ERR, PIPELINE,
 834                         "%s: port IN ID %u is out of range\n",
 835                         __func__, port_id);
 836                 return -EINVAL;
 837         }
 838
 839         if (table_id >= p->num_tables) {
 840                 RTE_LOG(ERR, PIPELINE,
 841                         "%s: Table ID %u is out of range\n",
 842                         __func__, table_id);
 843                 return -EINVAL;
 844         }
 845
 846         port = &p->ports_in[port_id];
 847         port->table_id = table_id;
 848
 849         return 0;
 850 }
 851
 852 int
 853 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 854 {
 855         struct rte_port_in *port, *port_prev, *port_next;
 856         struct rte_port_in *port_first, *port_last;
 857         uint64_t port_mask;
 858         uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
 859
 860         /* Check input arguments */
 861         if (p == NULL) {
 862                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 863                         __func__);
 864                 return -EINVAL;
 865         }
 866
 867         if (port_id >= p->num_ports_in) {
 868                 RTE_LOG(ERR, PIPELINE,
 869                         "%s: port IN ID %u is out of range\n",
 870                         __func__, port_id);
 871                 return -EINVAL;
 872         }
 873
 874         /* Return if current input port is already enabled */
 875         port_mask = 1LLU << port_id;
 876         if (p->enabled_port_in_mask & port_mask)
 877                 return 0;
 878
 879         p->enabled_port_in_mask |= port_mask;
 880
 881         /* Add current input port to the pipeline chain of enabled ports */
 882         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 883         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 884
 885         port_prev = &p->ports_in[port_prev_id];
 886         port_next = &p->ports_in[port_next_id];
 887         port = &p->ports_in[port_id];
 888
 889         port_prev->next = port;
 890         port->next = port_next;
 891
 892         /* Update the first and last input ports in the chain */
 893         port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
 894         port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
 895
 896         port_first = &p->ports_in[port_first_id];
 897         port_last = &p->ports_in[port_last_id];
 898
 899         p->port_in_first = port_first;
 900         port_last->next = NULL;
 901
 902         return 0;
 903 }
 904
 905 int
 906 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
 907 {
 908         struct rte_port_in *port_prev, *port_next, *port_first, *port_last;
 909         uint64_t port_mask;
 910         uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
 911
 912         /* Check input arguments */
 913         if (p == NULL) {
 914                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 915                 __func__);
 916                 return -EINVAL;
 917         }
 918
 919         if (port_id >= p->num_ports_in) {
 920                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
 921                         __func__, port_id);
 922                 return -EINVAL;
 923         }
 924
 925         /* Return if current input port is already disabled */
 926         port_mask = 1LLU << port_id;
 927         if ((p->enabled_port_in_mask & port_mask) == 0)
 928                 return 0;
 929
 930         /* Return if no other enabled ports */
 931         if (__builtin_popcountll(p->enabled_port_in_mask) == 1) {
 932                 p->enabled_port_in_mask &= ~port_mask;
 933                 p->port_in_first = NULL;
 934
 935                 return 0;
 936         }
 937
 938         /* Add current input port to the pipeline chain of enabled ports */
 939         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 940         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 941
 942         port_prev = &p->ports_in[port_prev_id];
 943         port_next = &p->ports_in[port_next_id];
 944
 945         port_prev->next = port_next;
 946         p->enabled_port_in_mask &= ~port_mask;
 947
 948         /* Update the first and last input ports in the chain */
 949         port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
 950         port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
 951
 952         port_first = &p->ports_in[port_first_id];
 953         port_last = &p->ports_in[port_last_id];
 954
 955         p->port_in_first = port_first;
 956         port_last->next = NULL;
 957
 958         return 0;
 959 }
 960
 961 /*
 962  * Pipeline run-time
 963  *
 964  */
 965 int
 966 rte_pipeline_check(struct rte_pipeline *p)
 967 {
 968         uint32_t port_in_id;
 969
 970         /* Check input arguments */
 971         if (p == NULL) {
 972                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 973                         __func__);
 974                 return -EINVAL;
 975         }
 976
 977         /* Check that pipeline has at least one input port, one table and one
 978         output port */
 979         if (p->num_ports_in == 0) {
 980                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
 981                         __func__);
 982                 return -EINVAL;
 983         }
 984         if (p->num_tables == 0) {
 985                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
 986                         __func__);
 987                 return -EINVAL;
 988         }
 989         if (p->num_ports_out == 0) {
 990                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
 991                         __func__);
 992                 return -EINVAL;
 993         }
 994
 995         /* Check that all input ports are connected */
 996         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
 997                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
 998
 999                 if (port_in->table_id == RTE_TABLE_INVALID) {
1000                         RTE_LOG(ERR, PIPELINE,
1001                                 "%s: Port IN ID %u is not connected\n",
1002                                 __func__, port_in_id);
1003                         return -EINVAL;
1004                 }
1005         }
1006
1007         return 0;
1008 }
1009
1010 static inline void
1011 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1012 {
1013         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1014         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1015         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1016         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1017
1018         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1019                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1020                 uint32_t i;
1021
1022                 for (i = 0; i < n_pkts; i++) {
1023                         uint64_t pkt_mask = 1LLU << i;
1024                         uint32_t pos = p->entries[i]->action;
1025
1026                         p->action_mask1[pos] |= pkt_mask;
1027                 }
1028         } else {
1029                 uint32_t i;
1030
1031                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1032                         uint64_t pkt_mask = 1LLU << i;
1033                         uint32_t pos;
1034
1035                         if ((pkt_mask & pkts_mask) == 0)
1036                                 continue;
1037
1038                         pos = p->entries[i]->action;
1039                         p->action_mask1[pos] |= pkt_mask;
1040                 }
1041         }
1042 }
1043
1044 static inline void
1045 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1046                 uint64_t pkts_mask, uint32_t port_id)
1047 {
1048         struct rte_port_out *port_out = &p->ports_out[port_id];
1049
1050         /* Output port user actions */
1051         if (port_out->f_action_bulk != NULL) {
1052                 uint64_t mask = pkts_mask;
1053
1054                 port_out->f_action_bulk(p->pkts, &pkts_mask, port_out->arg_ah);
1055                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask ^  mask;
1056                 RTE_PIPELINE_STATS_ADD_M(port_out->n_pkts_dropped_by_ah,
1057                                 pkts_mask ^  mask);
1058         }
1059
1060         /* Output port TX */
1061         if (pkts_mask != 0)
1062                 port_out->ops.f_tx_bulk(port_out->h_port, p->pkts, pkts_mask);
1063 }
1064
1065 static inline void
1066 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1067 {
1068         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1069                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1070                 uint32_t i;
1071
1072                 for (i = 0; i < n_pkts; i++) {
1073                         struct rte_mbuf *pkt = p->pkts[i];
1074                         uint32_t port_out_id = p->entries[i]->port_id;
1075                         struct rte_port_out *port_out =
1076                                 &p->ports_out[port_out_id];
1077
1078                         /* Output port user actions */
1079                         if (port_out->f_action == NULL) /* Output port TX */
1080                                 port_out->ops.f_tx(port_out->h_port, pkt);
1081                         else {
1082                                 uint64_t pkt_mask = 1LLU;
1083
1084                                 port_out->f_action(pkt, &pkt_mask,
1085                                         port_out->arg_ah);
1086                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1087                                         (pkt_mask ^ 1LLU) << i;
1088
1089                                 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1090                                                 pkt_mask ^ 1LLU);
1091
1092                                 /* Output port TX */
1093                                 if (pkt_mask != 0)
1094                                         port_out->ops.f_tx(port_out->h_port,
1095                                                 pkt);
1096                         }
1097                 }
1098         } else {
1099                 uint32_t i;
1100
1101                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1102                         uint64_t pkt_mask = 1LLU << i;
1103                         struct rte_mbuf *pkt;
1104                         struct rte_port_out *port_out;
1105                         uint32_t port_out_id;
1106
1107                         if ((pkt_mask & pkts_mask) == 0)
1108                                 continue;
1109
1110                         pkt = p->pkts[i];
1111                         port_out_id = p->entries[i]->port_id;
1112                         port_out = &p->ports_out[port_out_id];
1113
1114                         /* Output port user actions */
1115                         if (port_out->f_action == NULL) /* Output port TX */
1116                                 port_out->ops.f_tx(port_out->h_port, pkt);
1117                         else {
1118                                 pkt_mask = 1LLU;
1119
1120                                 port_out->f_action(pkt, &pkt_mask,
1121                                         port_out->arg_ah);
1122                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1123                                         (pkt_mask ^ 1LLU) << i;
1124
1125                                 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1126                                                 pkt_mask ^ 1LLU);
1127
1128                                 /* Output port TX */
1129                                 if (pkt_mask != 0)
1130                                         port_out->ops.f_tx(port_out->h_port,
1131                                                 pkt);
1132                         }
1133                 }
1134         }
1135 }
1136
1137 static inline void
1138 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1139         uint64_t pkts_mask)
1140 {
1141         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1142                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1143                 uint32_t i;
1144
1145                 for (i = 0; i < n_pkts; i++) {
1146                         struct rte_mbuf *pkt = p->pkts[i];
1147                         uint32_t port_out_id =
1148                                 RTE_MBUF_METADATA_UINT32(pkt,
1149                                         p->offset_port_id);
1150                         struct rte_port_out *port_out = &p->ports_out[
1151                                 port_out_id];
1152
1153                         /* Output port user actions */
1154                         if (port_out->f_action == NULL) /* Output port TX */
1155                                 port_out->ops.f_tx(port_out->h_port, pkt);
1156                         else {
1157                                 uint64_t pkt_mask = 1LLU;
1158
1159                                 port_out->f_action(pkt, &pkt_mask,
1160                                         port_out->arg_ah);
1161                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1162                                         (pkt_mask ^ 1LLU) << i;
1163
1164                                 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1165                                                 pkt_mask ^ 1ULL);
1166
1167                                 /* Output port TX */
1168                                 if (pkt_mask != 0)
1169                                         port_out->ops.f_tx(port_out->h_port,
1170                                                 pkt);
1171                         }
1172                 }
1173         } else {
1174                 uint32_t i;
1175
1176                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1177                         uint64_t pkt_mask = 1LLU << i;
1178                         struct rte_mbuf *pkt;
1179                         struct rte_port_out *port_out;
1180                         uint32_t port_out_id;
1181
1182                         if ((pkt_mask & pkts_mask) == 0)
1183                                 continue;
1184
1185                         pkt = p->pkts[i];
1186                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1187                                 p->offset_port_id);
1188                         port_out = &p->ports_out[port_out_id];
1189
1190                         /* Output port user actions */
1191                         if (port_out->f_action == NULL) /* Output port TX */
1192                                 port_out->ops.f_tx(port_out->h_port, pkt);
1193                         else {
1194                                 pkt_mask = 1LLU;
1195
1196                                 port_out->f_action(pkt, &pkt_mask,
1197                                         port_out->arg_ah);
1198                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1199                                         (pkt_mask ^ 1LLU) << i;
1200
1201                                 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1202                                                 pkt_mask ^ 1ULL);
1203
1204                                 /* Output port TX */
1205                                 if (pkt_mask != 0)
1206                                         port_out->ops.f_tx(port_out->h_port,
1207                                                 pkt);
1208                         }
1209                 }
1210         }
1211 }
1212
1213 static inline void
1214 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1215 {
1216         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1217                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1218                 uint32_t i;
1219
1220                 for (i = 0; i < n_pkts; i++)
1221                         rte_pktmbuf_free(p->pkts[i]);
1222         } else {
1223                 uint32_t i;
1224
1225                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1226                         uint64_t pkt_mask = 1LLU << i;
1227
1228                         if ((pkt_mask & pkts_mask) == 0)
1229                                 continue;
1230
1231                         rte_pktmbuf_free(p->pkts[i]);
1232                 }
1233         }
1234 }
1235
1236 int
1237 rte_pipeline_run(struct rte_pipeline *p)
1238 {
1239         struct rte_port_in *port_in;
1240
1241         for (port_in = p->port_in_first; port_in != NULL;
1242                 port_in = port_in->next) {
1243                 uint64_t pkts_mask;
1244                 uint32_t n_pkts, table_id;
1245
1246                 /* Input port RX */
1247                 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1248                         port_in->burst_size);
1249                 if (n_pkts == 0)
1250                         continue;
1251
1252                 pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1253                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1254                 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1255                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1256                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1257
1258                 /* Input port user actions */
1259                 if (port_in->f_action != NULL) {
1260                         uint64_t mask = pkts_mask;
1261
1262                         port_in->f_action(p->pkts, n_pkts, &pkts_mask, port_in->arg_ah);
1263                         mask ^= pkts_mask;
1264                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1265                         RTE_PIPELINE_STATS_ADD_M(port_in->n_pkts_dropped_by_ah, mask);
1266                 }
1267
1268                 /* Table */
1269                 for (table_id = port_in->table_id; pkts_mask != 0; ) {
1270                         struct rte_table *table;
1271                         uint64_t lookup_hit_mask, lookup_miss_mask;
1272
1273                         /* Lookup */
1274                         table = &p->tables[table_id];
1275                         table->ops.f_lookup(table->h_table, p->pkts, pkts_mask,
1276                                         &lookup_hit_mask, (void **) p->entries);
1277                         lookup_miss_mask = pkts_mask & (~lookup_hit_mask);
1278
1279                         /* Lookup miss */
1280                         if (lookup_miss_mask != 0) {
1281                                 struct rte_pipeline_table_entry *default_entry =
1282                                         table->default_entry;
1283
1284                                 /* Table user actions */
1285                                 if (table->f_action_miss != NULL) {
1286                                         uint64_t mask = lookup_miss_mask;
1287
1288                                         table->f_action_miss(p->pkts,
1289                                                 &lookup_miss_mask,
1290                                                 default_entry, table->arg_ah);
1291                                         mask ^= lookup_miss_mask;
1292                                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1293                                         RTE_PIPELINE_STATS_ADD_M(
1294                                                 table->n_pkts_dropped_by_lkp_miss_ah, mask);
1295                                 }
1296
1297                                 /* Table reserved actions */
1298                                 if ((default_entry->action ==
1299                                         RTE_PIPELINE_ACTION_PORT) &&
1300                                         (lookup_miss_mask != 0))
1301                                         rte_pipeline_action_handler_port_bulk(p,
1302                                                 lookup_miss_mask,
1303                                                 default_entry->port_id);
1304                                 else {
1305                                         uint32_t pos = default_entry->action;
1306
1307                                         p->action_mask0[pos] = lookup_miss_mask;
1308                                         if (pos == RTE_PIPELINE_ACTION_DROP) {
1309                                                 RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_miss,
1310                                                         lookup_miss_mask);
1311                                         }
1312                                 }
1313                         }
1314
1315                         /* Lookup hit */
1316                         if (lookup_hit_mask != 0) {
1317                                 /* Table user actions */
1318                                 if (table->f_action_hit != NULL) {
1319                                         uint64_t mask = lookup_hit_mask;
1320
1321                                         table->f_action_hit(p->pkts,
1322                                                 &lookup_hit_mask,
1323                                                 p->entries, table->arg_ah);
1324                                         mask ^= lookup_hit_mask;
1325                                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1326                                         RTE_PIPELINE_STATS_ADD_M(
1327                                                 table->n_pkts_dropped_by_lkp_hit_ah, mask);
1328                                 }
1329
1330                                 /* Table reserved actions */
1331                                 rte_pipeline_compute_masks(p, lookup_hit_mask);
1332                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1333                                         p->action_mask1[
1334                                                 RTE_PIPELINE_ACTION_DROP];
1335                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1336                                         p->action_mask1[
1337                                                 RTE_PIPELINE_ACTION_PORT];
1338                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1339                                         p->action_mask1[
1340                                                 RTE_PIPELINE_ACTION_PORT_META];
1341                                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1342                                         p->action_mask1[
1343                                                 RTE_PIPELINE_ACTION_TABLE];
1344
1345                                 RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_hit,
1346                                                 p->action_mask1[RTE_PIPELINE_ACTION_DROP]);
1347                         }
1348
1349                         /* Prepare for next iteration */
1350                         pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1351                         table_id = table->table_next_id;
1352                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1353                 }
1354
1355                 /* Table reserved action PORT */
1356                 rte_pipeline_action_handler_port(p,
1357                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1358
1359                 /* Table reserved action PORT META */
1360                 rte_pipeline_action_handler_port_meta(p,
1361                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1362
1363                 /* Table reserved action DROP */
1364                 rte_pipeline_action_handler_drop(p,
1365                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1366         }
1367
1368         return 0;
1369 }
1370
1371 int
1372 rte_pipeline_flush(struct rte_pipeline *p)
1373 {
1374         uint32_t port_id;
1375
1376         /* Check input arguments */
1377         if (p == NULL) {
1378                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1379                         __func__);
1380                 return -EINVAL;
1381         }
1382
1383         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1384                 struct rte_port_out *port = &p->ports_out[port_id];
1385
1386                 if (port->ops.f_flush != NULL)
1387                         port->ops.f_flush(port->h_port);
1388         }
1389
1390         return 0;
1391 }
1392
1393 int
1394 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1395                 uint32_t port_id, struct rte_mbuf *pkt)
1396 {
1397         struct rte_port_out *port_out = &p->ports_out[port_id];
1398
1399         /* Output port user actions */
1400         if (port_out->f_action == NULL)
1401                 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1402         else {
1403                 uint64_t pkt_mask = 1LLU;
1404
1405                 port_out->f_action(pkt, &pkt_mask, port_out->arg_ah);
1406
1407                 if (pkt_mask != 0) /* Output port TX */
1408                         port_out->ops.f_tx(port_out->h_port, pkt);
1409                 else {
1410                         rte_pktmbuf_free(pkt);
1411                         RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, 1);
1412                 }
1413         }
1414
1415         return 0;
1416 }
1417
1418 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1419         struct rte_pipeline_port_in_stats *stats, int clear)
1420 {
1421         struct rte_port_in *port;
1422         int retval;
1423
1424         if (p == NULL) {
1425                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1426                         __func__);
1427                 return -EINVAL;
1428         }
1429
1430         if (port_id >= p->num_ports_in) {
1431                 RTE_LOG(ERR, PIPELINE,
1432                         "%s: port IN ID %u is out of range\n",
1433                         __func__, port_id);
1434                 return -EINVAL;
1435         }
1436
1437         port = &p->ports_in[port_id];
1438
1439         if (port->ops.f_stats != NULL) {
1440                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1441                 if (retval)
1442                         return retval;
1443         } else if (stats != NULL)
1444                 memset(&stats->stats, 0, sizeof(stats->stats));
1445
1446         if (stats != NULL)
1447                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1448
1449         if (clear != 0)
1450                 port->n_pkts_dropped_by_ah = 0;
1451
1452         return 0;
1453 }
1454
1455 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1456         struct rte_pipeline_port_out_stats *stats, int clear)
1457 {
1458         struct rte_port_out *port;
1459         int retval;
1460
1461         if (p == NULL) {
1462                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1463                 return -EINVAL;
1464         }
1465
1466         if (port_id >= p->num_ports_out) {
1467                 RTE_LOG(ERR, PIPELINE,
1468                         "%s: port OUT ID %u is out of range\n", __func__, port_id);
1469                 return -EINVAL;
1470         }
1471
1472         port = &p->ports_out[port_id];
1473         if (port->ops.f_stats != NULL) {
1474                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1475                 if (retval != 0)
1476                         return retval;
1477         } else if (stats != NULL)
1478                 memset(&stats->stats, 0, sizeof(stats->stats));
1479
1480         if (stats != NULL)
1481                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1482
1483         if (clear != 0)
1484                 port->n_pkts_dropped_by_ah = 0;
1485
1486         return 0;
1487 }
1488
1489 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1490         struct rte_pipeline_table_stats *stats, int clear)
1491 {
1492         struct rte_table *table;
1493         int retval;
1494
1495         if (p == NULL) {
1496                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1497                         __func__);
1498                 return -EINVAL;
1499         }
1500
1501         if (table_id >= p->num_tables) {
1502                 RTE_LOG(ERR, PIPELINE,
1503                                 "%s: table %u is out of range\n", __func__, table_id);
1504                 return -EINVAL;
1505         }
1506
1507         table = &p->tables[table_id];
1508         if (table->ops.f_stats != NULL) {
1509                 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1510                 if (retval != 0)
1511                         return retval;
1512         } else if (stats != NULL)
1513                 memset(&stats->stats, 0, sizeof(stats->stats));
1514
1515         if (stats != NULL) {
1516                 stats->n_pkts_dropped_by_lkp_hit_ah =
1517                         table->n_pkts_dropped_by_lkp_hit_ah;
1518                 stats->n_pkts_dropped_by_lkp_miss_ah =
1519                         table->n_pkts_dropped_by_lkp_miss_ah;
1520                 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1521                 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1522         }
1523
1524         if (clear != 0) {
1525                 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1526                 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1527                 table->n_pkts_dropped_lkp_hit = 0;
1528                 table->n_pkts_dropped_lkp_miss = 0;
1529         }
1530
1531         return 0;
1532 }
1533