lib/librte_pipeline/rte_pipeline.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_memory.h>
  39 #include <rte_memzone.h>
  40 #include <rte_cycles.h>
  41 #include <rte_prefetch.h>
  42 #include <rte_branch_prediction.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_malloc.h>
  45 #include <rte_string_fns.h>
  46
  47 #include "rte_pipeline.h"
  48
  49 #define RTE_TABLE_INVALID                                 UINT32_MAX
  50
  51 #ifdef RTE_PIPELINE_STATS_COLLECT
  52 #define RTE_PIPELINE_STATS_ADD(counter, val) \
  53         ({ (counter) += (val); })
  54
  55 #define RTE_PIPELINE_STATS_ADD_M(counter, mask) \
  56         ({ (counter) += __builtin_popcountll(mask); })
  57 #else
  58 #define RTE_PIPELINE_STATS_ADD(counter, val)
  59 #define RTE_PIPELINE_STATS_ADD_M(counter, mask)
  60 #endif
  61
  62 struct rte_port_in {
  63         /* Input parameters */
  64         struct rte_port_in_ops ops;
  65         rte_pipeline_port_in_action_handler f_action;
  66         void *arg_ah;
  67         uint32_t burst_size;
  68
  69         /* The table to which this port is connected */
  70         uint32_t table_id;
  71
  72         /* Handle to low-level port */
  73         void *h_port;
  74
  75         /* List of enabled ports */
  76         struct rte_port_in *next;
  77
  78         uint64_t n_pkts_dropped_by_ah;
  79 };
  80
  81 struct rte_port_out {
  82         /* Input parameters */
  83         struct rte_port_out_ops ops;
  84         rte_pipeline_port_out_action_handler f_action;
  85         rte_pipeline_port_out_action_handler_bulk f_action_bulk;
  86         void *arg_ah;
  87
  88         /* Handle to low-level port */
  89         void *h_port;
  90
  91         uint64_t n_pkts_dropped_by_ah;
  92 };
  93
  94 struct rte_table {
  95         /* Input parameters */
  96         struct rte_table_ops ops;
  97         rte_pipeline_table_action_handler_hit f_action_hit;
  98         rte_pipeline_table_action_handler_miss f_action_miss;
  99         void *arg_ah;
 100         struct rte_pipeline_table_entry *default_entry;
 101         uint32_t entry_size;
 102
 103         uint32_t table_next_id;
 104         uint32_t table_next_id_valid;
 105
 106         /* Handle to the low-level table object */
 107         void *h_table;
 108
 109         /* Stats for this table. */
 110         uint64_t n_pkts_dropped_by_lkp_hit_ah;
 111         uint64_t n_pkts_dropped_by_lkp_miss_ah;
 112         uint64_t n_pkts_dropped_lkp_hit;
 113         uint64_t n_pkts_dropped_lkp_miss;
 114 };
 115
 116 #define RTE_PIPELINE_MAX_NAME_SZ                           124
 117
 118 struct rte_pipeline {
 119         /* Input parameters */
 120         char name[RTE_PIPELINE_MAX_NAME_SZ];
 121         int socket_id;
 122         uint32_t offset_port_id;
 123
 124         /* Internal tables */
 125         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 126         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 127         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 128
 129         /* Occupancy of internal tables */
 130         uint32_t num_ports_in;
 131         uint32_t num_ports_out;
 132         uint32_t num_tables;
 133
 134         /* List of enabled ports */
 135         uint64_t enabled_port_in_mask;
 136         struct rte_port_in *port_in_first;
 137
 138         /* Pipeline run structures */
 139         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 140         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 141         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 142         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 143 } __rte_cache_aligned;
 144
 145 static inline uint32_t
 146 rte_mask_get_next(uint64_t mask, uint32_t pos)
 147 {
 148         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 149                         (mask >> ((pos + 1) & 0x3F));
 150         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 151 }
 152
 153 static inline uint32_t
 154 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 155 {
 156         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 157                         (mask << ((64 - pos) & 0x3F));
 158         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 159 }
 160
 161 static void
 162 rte_pipeline_table_free(struct rte_table *table);
 163
 164 static void
 165 rte_pipeline_port_in_free(struct rte_port_in *port);
 166
 167 static void
 168 rte_pipeline_port_out_free(struct rte_port_out *port);
 169
 170 /*
 171  * Pipeline
 172  *
 173  */
 174 static int
 175 rte_pipeline_check_params(struct rte_pipeline_params *params)
 176 {
 177         if (params == NULL) {
 178                 RTE_LOG(ERR, PIPELINE,
 179                         "%s: Incorrect value for parameter params\n", __func__);
 180                 return -EINVAL;
 181         }
 182
 183         /* name */
 184         if (params->name == NULL) {
 185                 RTE_LOG(ERR, PIPELINE,
 186                         "%s: Incorrect value for parameter name\n", __func__);
 187                 return -EINVAL;
 188         }
 189
 190         /* socket */
 191         if ((params->socket_id < 0) ||
 192             (params->socket_id >= RTE_MAX_NUMA_NODES)) {
 193                 RTE_LOG(ERR, PIPELINE,
 194                         "%s: Incorrect value for parameter socket_id\n",
 195                         __func__);
 196                 return -EINVAL;
 197         }
 198
 199         return 0;
 200 }
 201
 202 struct rte_pipeline *
 203 rte_pipeline_create(struct rte_pipeline_params *params)
 204 {
 205         struct rte_pipeline *p;
 206         int status;
 207
 208         /* Check input parameters */
 209         status = rte_pipeline_check_params(params);
 210         if (status != 0) {
 211                 RTE_LOG(ERR, PIPELINE,
 212                         "%s: Pipeline params check failed (%d)\n",
 213                         __func__, status);
 214                 return NULL;
 215         }
 216
 217         /* Allocate memory for the pipeline on requested socket */
 218         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 219                         RTE_CACHE_LINE_SIZE, params->socket_id);
 220
 221         if (p == NULL) {
 222                 RTE_LOG(ERR, PIPELINE,
 223                         "%s: Pipeline memory allocation failed\n", __func__);
 224                 return NULL;
 225         }
 226
 227         /* Save input parameters */
 228         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
 229         p->socket_id = params->socket_id;
 230         p->offset_port_id = params->offset_port_id;
 231
 232         /* Initialize pipeline internal data structure */
 233         p->num_ports_in = 0;
 234         p->num_ports_out = 0;
 235         p->num_tables = 0;
 236         p->enabled_port_in_mask = 0;
 237         p->port_in_first = NULL;
 238
 239         return p;
 240 }
 241
 242 int
 243 rte_pipeline_free(struct rte_pipeline *p)
 244 {
 245         uint32_t i;
 246
 247         /* Check input parameters */
 248         if (p == NULL) {
 249                 RTE_LOG(ERR, PIPELINE,
 250                         "%s: rte_pipeline parameter is NULL\n", __func__);
 251                 return -EINVAL;
 252         }
 253
 254         /* Free input ports */
 255         for (i = 0; i < p->num_ports_in; i++) {
 256                 struct rte_port_in *port = &p->ports_in[i];
 257
 258                 rte_pipeline_port_in_free(port);
 259         }
 260
 261         /* Free tables */
 262         for (i = 0; i < p->num_tables; i++) {
 263                 struct rte_table *table = &p->tables[i];
 264
 265                 rte_pipeline_table_free(table);
 266         }
 267
 268         /* Free output ports */
 269         for (i = 0; i < p->num_ports_out; i++) {
 270                 struct rte_port_out *port = &p->ports_out[i];
 271
 272                 rte_pipeline_port_out_free(port);
 273         }
 274
 275         /* Free pipeline memory */
 276         rte_free(p);
 277
 278         return 0;
 279 }
 280
 281 /*
 282  * Table
 283  *
 284  */
 285 static int
 286 rte_table_check_params(struct rte_pipeline *p,
 287                 struct rte_pipeline_table_params *params,
 288                 uint32_t *table_id)
 289 {
 290         if (p == NULL) {
 291                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 292                         __func__);
 293                 return -EINVAL;
 294         }
 295         if (params == NULL) {
 296                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 297                         __func__);
 298                 return -EINVAL;
 299         }
 300         if (table_id == NULL) {
 301                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 302                         __func__);
 303                 return -EINVAL;
 304         }
 305
 306         /* ops */
 307         if (params->ops == NULL) {
 308                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 309                         __func__);
 310                 return -EINVAL;
 311         }
 312
 313         if (params->ops->f_create == NULL) {
 314                 RTE_LOG(ERR, PIPELINE,
 315                         "%s: f_create function pointer is NULL\n", __func__);
 316                 return -EINVAL;
 317         }
 318
 319         if (params->ops->f_lookup == NULL) {
 320                 RTE_LOG(ERR, PIPELINE,
 321                         "%s: f_lookup function pointer is NULL\n", __func__);
 322                 return -EINVAL;
 323         }
 324
 325         /* De we have room for one more table? */
 326         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 327                 RTE_LOG(ERR, PIPELINE,
 328                         "%s: Incorrect value for num_tables parameter\n",
 329                         __func__);
 330                 return -EINVAL;
 331         }
 332
 333         return 0;
 334 }
 335
 336 int
 337 rte_pipeline_table_create(struct rte_pipeline *p,
 338                 struct rte_pipeline_table_params *params,
 339                 uint32_t *table_id)
 340 {
 341         struct rte_table *table;
 342         struct rte_pipeline_table_entry *default_entry;
 343         void *h_table;
 344         uint32_t entry_size, id;
 345         int status;
 346
 347         /* Check input arguments */
 348         status = rte_table_check_params(p, params, table_id);
 349         if (status != 0)
 350                 return status;
 351
 352         id = p->num_tables;
 353         table = &p->tables[id];
 354
 355         /* Allocate space for the default table entry */
 356         entry_size = sizeof(struct rte_pipeline_table_entry) +
 357                 params->action_data_size;
 358         default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
 359                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 360         if (default_entry == NULL) {
 361                 RTE_LOG(ERR, PIPELINE,
 362                         "%s: Failed to allocate default entry\n", __func__);
 363                 return -EINVAL;
 364         }
 365
 366         /* Create the table */
 367         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 368                 entry_size);
 369         if (h_table == NULL) {
 370                 rte_free(default_entry);
 371                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 372                 return -EINVAL;
 373         }
 374
 375         /* Commit current table to the pipeline */
 376         p->num_tables++;
 377         *table_id = id;
 378
 379         /* Save input parameters */
 380         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 381         table->f_action_hit = params->f_action_hit;
 382         table->f_action_miss = params->f_action_miss;
 383         table->arg_ah = params->arg_ah;
 384         table->entry_size = entry_size;
 385
 386         /* Clear the lookup miss actions (to be set later through API) */
 387         table->default_entry = default_entry;
 388         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 389
 390         /* Initialize table internal data structure */
 391         table->h_table = h_table;
 392         table->table_next_id = 0;
 393         table->table_next_id_valid = 0;
 394
 395         return 0;
 396 }
 397
 398 void
 399 rte_pipeline_table_free(struct rte_table *table)
 400 {
 401         if (table->ops.f_free != NULL)
 402                 table->ops.f_free(table->h_table);
 403
 404         rte_free(table->default_entry);
 405 }
 406
 407 int
 408 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 409         uint32_t table_id,
 410         struct rte_pipeline_table_entry *default_entry,
 411         struct rte_pipeline_table_entry **default_entry_ptr)
 412 {
 413         struct rte_table *table;
 414
 415         /* Check input arguments */
 416         if (p == NULL) {
 417                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 418                         __func__);
 419                 return -EINVAL;
 420         }
 421
 422         if (default_entry == NULL) {
 423                 RTE_LOG(ERR, PIPELINE,
 424                         "%s: default_entry parameter is NULL\n", __func__);
 425                 return -EINVAL;
 426         }
 427
 428         if (table_id >= p->num_tables) {
 429                 RTE_LOG(ERR, PIPELINE,
 430                         "%s: table_id %d out of range\n", __func__, table_id);
 431                 return -EINVAL;
 432         }
 433
 434         table = &p->tables[table_id];
 435
 436         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 437                 table->table_next_id_valid &&
 438                 (default_entry->table_id != table->table_next_id)) {
 439                 RTE_LOG(ERR, PIPELINE,
 440                         "%s: Tree-like topologies not allowed\n", __func__);
 441                 return -EINVAL;
 442         }
 443
 444         /* Set the lookup miss actions */
 445         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 446                 (table->table_next_id_valid == 0)) {
 447                 table->table_next_id = default_entry->table_id;
 448                 table->table_next_id_valid = 1;
 449         }
 450
 451         memcpy(table->default_entry, default_entry, table->entry_size);
 452
 453         *default_entry_ptr = table->default_entry;
 454         return 0;
 455 }
 456
 457 int
 458 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 459                 uint32_t table_id,
 460                 struct rte_pipeline_table_entry *entry)
 461 {
 462         struct rte_table *table;
 463
 464         /* Check input arguments */
 465         if (p == NULL) {
 466                 RTE_LOG(ERR, PIPELINE,
 467                         "%s: pipeline parameter is NULL\n", __func__);
 468                 return -EINVAL;
 469         }
 470
 471         if (table_id >= p->num_tables) {
 472                 RTE_LOG(ERR, PIPELINE,
 473                         "%s: table_id %d out of range\n", __func__, table_id);
 474                 return -EINVAL;
 475         }
 476
 477         table = &p->tables[table_id];
 478
 479         /* Save the current contents of the default entry */
 480         if (entry)
 481                 memcpy(entry, table->default_entry, table->entry_size);
 482
 483         /* Clear the lookup miss actions */
 484         memset(table->default_entry, 0, table->entry_size);
 485         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 486
 487         return 0;
 488 }
 489
 490 int
 491 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 492                 uint32_t table_id,
 493                 void *key,
 494                 struct rte_pipeline_table_entry *entry,
 495                 int *key_found,
 496                 struct rte_pipeline_table_entry **entry_ptr)
 497 {
 498         struct rte_table *table;
 499
 500         /* Check input arguments */
 501         if (p == NULL) {
 502                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 503                         __func__);
 504                 return -EINVAL;
 505         }
 506
 507         if (key == NULL) {
 508                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 509                 return -EINVAL;
 510         }
 511
 512         if (entry == NULL) {
 513                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 514                         __func__);
 515                 return -EINVAL;
 516         }
 517
 518         if (table_id >= p->num_tables) {
 519                 RTE_LOG(ERR, PIPELINE,
 520                         "%s: table_id %d out of range\n", __func__, table_id);
 521                 return -EINVAL;
 522         }
 523
 524         table = &p->tables[table_id];
 525
 526         if (table->ops.f_add == NULL) {
 527                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 528                         __func__);
 529                 return -EINVAL;
 530         }
 531
 532         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 533                 table->table_next_id_valid &&
 534                 (entry->table_id != table->table_next_id)) {
 535                 RTE_LOG(ERR, PIPELINE,
 536                         "%s: Tree-like topologies not allowed\n", __func__);
 537                 return -EINVAL;
 538         }
 539
 540         /* Add entry */
 541         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 542                 (table->table_next_id_valid == 0)) {
 543                 table->table_next_id = entry->table_id;
 544                 table->table_next_id_valid = 1;
 545         }
 546
 547         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 548                 key_found, (void **) entry_ptr);
 549 }
 550
 551 int
 552 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 553                 uint32_t table_id,
 554                 void *key,
 555                 int *key_found,
 556                 struct rte_pipeline_table_entry *entry)
 557 {
 558         struct rte_table *table;
 559
 560         /* Check input arguments */
 561         if (p == NULL) {
 562                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 563                         __func__);
 564                 return -EINVAL;
 565         }
 566
 567         if (key == NULL) {
 568                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 569                         __func__);
 570                 return -EINVAL;
 571         }
 572
 573         if (table_id >= p->num_tables) {
 574                 RTE_LOG(ERR, PIPELINE,
 575                         "%s: table_id %d out of range\n", __func__, table_id);
 576                 return -EINVAL;
 577         }
 578
 579         table = &p->tables[table_id];
 580
 581         if (table->ops.f_delete == NULL) {
 582                 RTE_LOG(ERR, PIPELINE,
 583                         "%s: f_delete function pointer NULL\n", __func__);
 584                 return -EINVAL;
 585         }
 586
 587         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 588 }
 589
 590 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
 591         uint32_t table_id,
 592         void **keys,
 593         struct rte_pipeline_table_entry **entries,
 594         uint32_t n_keys,
 595         int *key_found,
 596         struct rte_pipeline_table_entry **entries_ptr)
 597 {
 598         struct rte_table *table;
 599         uint32_t i;
 600
 601         /* Check input arguments */
 602         if (p == NULL) {
 603                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 604                         __func__);
 605                 return -EINVAL;
 606         }
 607
 608         if (keys == NULL) {
 609                 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
 610                 return -EINVAL;
 611         }
 612
 613         if (entries == NULL) {
 614                 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
 615                         __func__);
 616                 return -EINVAL;
 617         }
 618
 619         if (table_id >= p->num_tables) {
 620                 RTE_LOG(ERR, PIPELINE,
 621                         "%s: table_id %d out of range\n", __func__, table_id);
 622                 return -EINVAL;
 623         }
 624
 625         table = &p->tables[table_id];
 626
 627         if (table->ops.f_add_bulk == NULL) {
 628                 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
 629                         __func__);
 630                 return -EINVAL;
 631         }
 632
 633         for (i = 0; i < n_keys; i++) {
 634                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 635                         table->table_next_id_valid &&
 636                         (entries[i]->table_id != table->table_next_id)) {
 637                         RTE_LOG(ERR, PIPELINE,
 638                                 "%s: Tree-like topologies not allowed\n", __func__);
 639                         return -EINVAL;
 640                 }
 641         }
 642
 643         /* Add entry */
 644         for (i = 0; i < n_keys; i++) {
 645                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 646                         (table->table_next_id_valid == 0)) {
 647                         table->table_next_id = entries[i]->table_id;
 648                         table->table_next_id_valid = 1;
 649                 }
 650         }
 651
 652         return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
 653                 n_keys, key_found, (void **) entries_ptr);
 654 }
 655
 656 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
 657         uint32_t table_id,
 658         void **keys,
 659         uint32_t n_keys,
 660         int *key_found,
 661         struct rte_pipeline_table_entry **entries)
 662 {
 663         struct rte_table *table;
 664
 665         /* Check input arguments */
 666         if (p == NULL) {
 667                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 668                         __func__);
 669                 return -EINVAL;
 670         }
 671
 672         if (keys == NULL) {
 673                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 674                         __func__);
 675                 return -EINVAL;
 676         }
 677
 678         if (table_id >= p->num_tables) {
 679                 RTE_LOG(ERR, PIPELINE,
 680                         "%s: table_id %d out of range\n", __func__, table_id);
 681                 return -EINVAL;
 682         }
 683
 684         table = &p->tables[table_id];
 685
 686         if (table->ops.f_delete_bulk == NULL) {
 687                 RTE_LOG(ERR, PIPELINE,
 688                         "%s: f_delete function pointer NULL\n", __func__);
 689                 return -EINVAL;
 690         }
 691
 692         return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
 693                         (void **) entries);
 694 }
 695
 696 /*
 697  * Port
 698  *
 699  */
 700 static int
 701 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 702                 struct rte_pipeline_port_in_params *params,
 703                 uint32_t *port_id)
 704 {
 705         if (p == NULL) {
 706                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 707                         __func__);
 708                 return -EINVAL;
 709         }
 710         if (params == NULL) {
 711                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 712                 return -EINVAL;
 713         }
 714         if (port_id == NULL) {
 715                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 716                         __func__);
 717                 return -EINVAL;
 718         }
 719
 720         /* ops */
 721         if (params->ops == NULL) {
 722                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 723                         __func__);
 724                 return -EINVAL;
 725         }
 726
 727         if (params->ops->f_create == NULL) {
 728                 RTE_LOG(ERR, PIPELINE,
 729                         "%s: f_create function pointer NULL\n", __func__);
 730                 return -EINVAL;
 731         }
 732
 733         if (params->ops->f_rx == NULL) {
 734                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 735                         __func__);
 736                 return -EINVAL;
 737         }
 738
 739         /* burst_size */
 740         if ((params->burst_size == 0) ||
 741                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 742                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 743                         __func__);
 744                 return -EINVAL;
 745         }
 746
 747         /* Do we have room for one more port? */
 748         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 749                 RTE_LOG(ERR, PIPELINE,
 750                         "%s: invalid value for num_ports_in\n", __func__);
 751                 return -EINVAL;
 752         }
 753
 754         return 0;
 755 }
 756
 757 static int
 758 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 759                 struct rte_pipeline_port_out_params *params,
 760                 uint32_t *port_id)
 761 {
 762         rte_pipeline_port_out_action_handler f_ah;
 763         rte_pipeline_port_out_action_handler_bulk f_ah_bulk;
 764
 765         if (p == NULL) {
 766                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 767                         __func__);
 768                 return -EINVAL;
 769         }
 770
 771         if (params == NULL) {
 772                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 773                 return -EINVAL;
 774         }
 775
 776         if (port_id == NULL) {
 777                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 778                         __func__);
 779                 return -EINVAL;
 780         }
 781
 782         /* ops */
 783         if (params->ops == NULL) {
 784                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 785                         __func__);
 786                 return -EINVAL;
 787         }
 788
 789         if (params->ops->f_create == NULL) {
 790                 RTE_LOG(ERR, PIPELINE,
 791                         "%s: f_create function pointer NULL\n", __func__);
 792                 return -EINVAL;
 793         }
 794
 795         if (params->ops->f_tx == NULL) {
 796                 RTE_LOG(ERR, PIPELINE,
 797                                 "%s: f_tx function pointer NULL\n", __func__);
 798                 return -EINVAL;
 799         }
 800
 801         if (params->ops->f_tx_bulk == NULL) {
 802                 RTE_LOG(ERR, PIPELINE,
 803                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 804                 return -EINVAL;
 805         }
 806
 807         f_ah = params->f_action;
 808         f_ah_bulk = params->f_action_bulk;
 809         if (((f_ah != NULL) && (f_ah_bulk == NULL)) ||
 810             ((f_ah == NULL) && (f_ah_bulk != NULL))) {
 811                 RTE_LOG(ERR, PIPELINE, "%s: Action handlers have to be either"
 812                         "both enabled or both disabled\n", __func__);
 813                 return -EINVAL;
 814         }
 815
 816         /* Do we have room for one more port? */
 817         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 818                 RTE_LOG(ERR, PIPELINE,
 819                         "%s: invalid value for num_ports_out\n", __func__);
 820                 return -EINVAL;
 821         }
 822
 823         return 0;
 824 }
 825
 826 int
 827 rte_pipeline_port_in_create(struct rte_pipeline *p,
 828                 struct rte_pipeline_port_in_params *params,
 829                 uint32_t *port_id)
 830 {
 831         struct rte_port_in *port;
 832         void *h_port;
 833         uint32_t id;
 834         int status;
 835
 836         /* Check input arguments */
 837         status = rte_pipeline_port_in_check_params(p, params, port_id);
 838         if (status != 0)
 839                 return status;
 840
 841         id = p->num_ports_in;
 842         port = &p->ports_in[id];
 843
 844         /* Create the port */
 845         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 846         if (h_port == NULL) {
 847                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 848                 return -EINVAL;
 849         }
 850
 851         /* Commit current table to the pipeline */
 852         p->num_ports_in++;
 853         *port_id = id;
 854
 855         /* Save input parameters */
 856         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 857         port->f_action = params->f_action;
 858         port->arg_ah = params->arg_ah;
 859         port->burst_size = params->burst_size;
 860
 861         /* Initialize port internal data structure */
 862         port->table_id = RTE_TABLE_INVALID;
 863         port->h_port = h_port;
 864         port->next = NULL;
 865
 866         return 0;
 867 }
 868
 869 void
 870 rte_pipeline_port_in_free(struct rte_port_in *port)
 871 {
 872         if (port->ops.f_free != NULL)
 873                 port->ops.f_free(port->h_port);
 874 }
 875
 876 int
 877 rte_pipeline_port_out_create(struct rte_pipeline *p,
 878                 struct rte_pipeline_port_out_params *params,
 879                 uint32_t *port_id)
 880 {
 881         struct rte_port_out *port;
 882         void *h_port;
 883         uint32_t id;
 884         int status;
 885
 886         /* Check input arguments */
 887         status = rte_pipeline_port_out_check_params(p, params, port_id);
 888         if (status != 0)
 889                 return status;
 890
 891         id = p->num_ports_out;
 892         port = &p->ports_out[id];
 893
 894         /* Create the port */
 895         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 896         if (h_port == NULL) {
 897                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 898                 return -EINVAL;
 899         }
 900
 901         /* Commit current table to the pipeline */
 902         p->num_ports_out++;
 903         *port_id = id;
 904
 905         /* Save input parameters */
 906         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 907         port->f_action = params->f_action;
 908         port->f_action_bulk = params->f_action_bulk;
 909         port->arg_ah = params->arg_ah;
 910
 911         /* Initialize port internal data structure */
 912         port->h_port = h_port;
 913
 914         return 0;
 915 }
 916
 917 void
 918 rte_pipeline_port_out_free(struct rte_port_out *port)
 919 {
 920         if (port->ops.f_free != NULL)
 921                 port->ops.f_free(port->h_port);
 922 }
 923
 924 int
 925 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 926                 uint32_t port_id,
 927                 uint32_t table_id)
 928 {
 929         struct rte_port_in *port;
 930
 931         /* Check input arguments */
 932         if (p == NULL) {
 933                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 934                         __func__);
 935                 return -EINVAL;
 936         }
 937
 938         if (port_id >= p->num_ports_in) {
 939                 RTE_LOG(ERR, PIPELINE,
 940                         "%s: port IN ID %u is out of range\n",
 941                         __func__, port_id);
 942                 return -EINVAL;
 943         }
 944
 945         if (table_id >= p->num_tables) {
 946                 RTE_LOG(ERR, PIPELINE,
 947                         "%s: Table ID %u is out of range\n",
 948                         __func__, table_id);
 949                 return -EINVAL;
 950         }
 951
 952         port = &p->ports_in[port_id];
 953         port->table_id = table_id;
 954
 955         return 0;
 956 }
 957
 958 int
 959 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 960 {
 961         struct rte_port_in *port, *port_prev, *port_next;
 962         struct rte_port_in *port_first, *port_last;
 963         uint64_t port_mask;
 964         uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
 965
 966         /* Check input arguments */
 967         if (p == NULL) {
 968                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 969                         __func__);
 970                 return -EINVAL;
 971         }
 972
 973         if (port_id >= p->num_ports_in) {
 974                 RTE_LOG(ERR, PIPELINE,
 975                         "%s: port IN ID %u is out of range\n",
 976                         __func__, port_id);
 977                 return -EINVAL;
 978         }
 979
 980         /* Return if current input port is already enabled */
 981         port_mask = 1LLU << port_id;
 982         if (p->enabled_port_in_mask & port_mask)
 983                 return 0;
 984
 985         p->enabled_port_in_mask |= port_mask;
 986
 987         /* Add current input port to the pipeline chain of enabled ports */
 988         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 989         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 990
 991         port_prev = &p->ports_in[port_prev_id];
 992         port_next = &p->ports_in[port_next_id];
 993         port = &p->ports_in[port_id];
 994
 995         port_prev->next = port;
 996         port->next = port_next;
 997
 998         /* Update the first and last input ports in the chain */
 999         port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
1000         port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
1001
1002         port_first = &p->ports_in[port_first_id];
1003         port_last = &p->ports_in[port_last_id];
1004
1005         p->port_in_first = port_first;
1006         port_last->next = NULL;
1007
1008         return 0;
1009 }
1010
1011 int
1012 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
1013 {
1014         struct rte_port_in *port_prev, *port_next, *port_first, *port_last;
1015         uint64_t port_mask;
1016         uint32_t port_prev_id, port_next_id, port_first_id, port_last_id;
1017
1018         /* Check input arguments */
1019         if (p == NULL) {
1020                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1021                 __func__);
1022                 return -EINVAL;
1023         }
1024
1025         if (port_id >= p->num_ports_in) {
1026                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
1027                         __func__, port_id);
1028                 return -EINVAL;
1029         }
1030
1031         /* Return if current input port is already disabled */
1032         port_mask = 1LLU << port_id;
1033         if ((p->enabled_port_in_mask & port_mask) == 0)
1034                 return 0;
1035
1036         /* Return if no other enabled ports */
1037         if (__builtin_popcountll(p->enabled_port_in_mask) == 1) {
1038                 p->enabled_port_in_mask &= ~port_mask;
1039                 p->port_in_first = NULL;
1040
1041                 return 0;
1042         }
1043
1044         /* Add current input port to the pipeline chain of enabled ports */
1045         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1046         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1047
1048         port_prev = &p->ports_in[port_prev_id];
1049         port_next = &p->ports_in[port_next_id];
1050
1051         port_prev->next = port_next;
1052         p->enabled_port_in_mask &= ~port_mask;
1053
1054         /* Update the first and last input ports in the chain */
1055         port_first_id = __builtin_ctzll(p->enabled_port_in_mask);
1056         port_last_id = 63 - __builtin_clzll(p->enabled_port_in_mask);
1057
1058         port_first = &p->ports_in[port_first_id];
1059         port_last = &p->ports_in[port_last_id];
1060
1061         p->port_in_first = port_first;
1062         port_last->next = NULL;
1063
1064         return 0;
1065 }
1066
1067 /*
1068  * Pipeline run-time
1069  *
1070  */
1071 int
1072 rte_pipeline_check(struct rte_pipeline *p)
1073 {
1074         uint32_t port_in_id;
1075
1076         /* Check input arguments */
1077         if (p == NULL) {
1078                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1079                         __func__);
1080                 return -EINVAL;
1081         }
1082
1083         /* Check that pipeline has at least one input port, one table and one
1084         output port */
1085         if (p->num_ports_in == 0) {
1086                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1087                         __func__);
1088                 return -EINVAL;
1089         }
1090         if (p->num_tables == 0) {
1091                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1092                         __func__);
1093                 return -EINVAL;
1094         }
1095         if (p->num_ports_out == 0) {
1096                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1097                         __func__);
1098                 return -EINVAL;
1099         }
1100
1101         /* Check that all input ports are connected */
1102         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1103                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1104
1105                 if (port_in->table_id == RTE_TABLE_INVALID) {
1106                         RTE_LOG(ERR, PIPELINE,
1107                                 "%s: Port IN ID %u is not connected\n",
1108                                 __func__, port_in_id);
1109                         return -EINVAL;
1110                 }
1111         }
1112
1113         return 0;
1114 }
1115
1116 static inline void
1117 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1118 {
1119         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1120         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1121         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1122         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1123
1124         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1125                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1126                 uint32_t i;
1127
1128                 for (i = 0; i < n_pkts; i++) {
1129                         uint64_t pkt_mask = 1LLU << i;
1130                         uint32_t pos = p->entries[i]->action;
1131
1132                         p->action_mask1[pos] |= pkt_mask;
1133                 }
1134         } else {
1135                 uint32_t i;
1136
1137                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1138                         uint64_t pkt_mask = 1LLU << i;
1139                         uint32_t pos;
1140
1141                         if ((pkt_mask & pkts_mask) == 0)
1142                                 continue;
1143
1144                         pos = p->entries[i]->action;
1145                         p->action_mask1[pos] |= pkt_mask;
1146                 }
1147         }
1148 }
1149
1150 static inline void
1151 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1152                 uint64_t pkts_mask, uint32_t port_id)
1153 {
1154         struct rte_port_out *port_out = &p->ports_out[port_id];
1155
1156         /* Output port user actions */
1157         if (port_out->f_action_bulk != NULL) {
1158                 uint64_t mask = pkts_mask;
1159
1160                 port_out->f_action_bulk(p->pkts, &pkts_mask, port_out->arg_ah);
1161                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask ^  mask;
1162                 RTE_PIPELINE_STATS_ADD_M(port_out->n_pkts_dropped_by_ah,
1163                                 pkts_mask ^  mask);
1164         }
1165
1166         /* Output port TX */
1167         if (pkts_mask != 0)
1168                 port_out->ops.f_tx_bulk(port_out->h_port, p->pkts, pkts_mask);
1169 }
1170
1171 static inline void
1172 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1173 {
1174         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1175                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1176                 uint32_t i;
1177
1178                 for (i = 0; i < n_pkts; i++) {
1179                         struct rte_mbuf *pkt = p->pkts[i];
1180                         uint32_t port_out_id = p->entries[i]->port_id;
1181                         struct rte_port_out *port_out =
1182                                 &p->ports_out[port_out_id];
1183
1184                         /* Output port user actions */
1185                         if (port_out->f_action == NULL) /* Output port TX */
1186                                 port_out->ops.f_tx(port_out->h_port, pkt);
1187                         else {
1188                                 uint64_t pkt_mask = 1LLU;
1189
1190                                 port_out->f_action(pkt, &pkt_mask,
1191                                         port_out->arg_ah);
1192                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1193                                         (pkt_mask ^ 1LLU) << i;
1194
1195                                 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1196                                                 pkt_mask ^ 1LLU);
1197
1198                                 /* Output port TX */
1199                                 if (pkt_mask != 0)
1200                                         port_out->ops.f_tx(port_out->h_port,
1201                                                 pkt);
1202                         }
1203                 }
1204         } else {
1205                 uint32_t i;
1206
1207                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1208                         uint64_t pkt_mask = 1LLU << i;
1209                         struct rte_mbuf *pkt;
1210                         struct rte_port_out *port_out;
1211                         uint32_t port_out_id;
1212
1213                         if ((pkt_mask & pkts_mask) == 0)
1214                                 continue;
1215
1216                         pkt = p->pkts[i];
1217                         port_out_id = p->entries[i]->port_id;
1218                         port_out = &p->ports_out[port_out_id];
1219
1220                         /* Output port user actions */
1221                         if (port_out->f_action == NULL) /* Output port TX */
1222                                 port_out->ops.f_tx(port_out->h_port, pkt);
1223                         else {
1224                                 pkt_mask = 1LLU;
1225
1226                                 port_out->f_action(pkt, &pkt_mask,
1227                                         port_out->arg_ah);
1228                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1229                                         (pkt_mask ^ 1LLU) << i;
1230
1231                                 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1232                                                 pkt_mask ^ 1LLU);
1233
1234                                 /* Output port TX */
1235                                 if (pkt_mask != 0)
1236                                         port_out->ops.f_tx(port_out->h_port,
1237                                                 pkt);
1238                         }
1239                 }
1240         }
1241 }
1242
1243 static inline void
1244 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1245         uint64_t pkts_mask)
1246 {
1247         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1248                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1249                 uint32_t i;
1250
1251                 for (i = 0; i < n_pkts; i++) {
1252                         struct rte_mbuf *pkt = p->pkts[i];
1253                         uint32_t port_out_id =
1254                                 RTE_MBUF_METADATA_UINT32(pkt,
1255                                         p->offset_port_id);
1256                         struct rte_port_out *port_out = &p->ports_out[
1257                                 port_out_id];
1258
1259                         /* Output port user actions */
1260                         if (port_out->f_action == NULL) /* Output port TX */
1261                                 port_out->ops.f_tx(port_out->h_port, pkt);
1262                         else {
1263                                 uint64_t pkt_mask = 1LLU;
1264
1265                                 port_out->f_action(pkt, &pkt_mask,
1266                                         port_out->arg_ah);
1267                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1268                                         (pkt_mask ^ 1LLU) << i;
1269
1270                                 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1271                                                 pkt_mask ^ 1ULL);
1272
1273                                 /* Output port TX */
1274                                 if (pkt_mask != 0)
1275                                         port_out->ops.f_tx(port_out->h_port,
1276                                                 pkt);
1277                         }
1278                 }
1279         } else {
1280                 uint32_t i;
1281
1282                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1283                         uint64_t pkt_mask = 1LLU << i;
1284                         struct rte_mbuf *pkt;
1285                         struct rte_port_out *port_out;
1286                         uint32_t port_out_id;
1287
1288                         if ((pkt_mask & pkts_mask) == 0)
1289                                 continue;
1290
1291                         pkt = p->pkts[i];
1292                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1293                                 p->offset_port_id);
1294                         port_out = &p->ports_out[port_out_id];
1295
1296                         /* Output port user actions */
1297                         if (port_out->f_action == NULL) /* Output port TX */
1298                                 port_out->ops.f_tx(port_out->h_port, pkt);
1299                         else {
1300                                 pkt_mask = 1LLU;
1301
1302                                 port_out->f_action(pkt, &pkt_mask,
1303                                         port_out->arg_ah);
1304                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1305                                         (pkt_mask ^ 1LLU) << i;
1306
1307                                 RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah,
1308                                                 pkt_mask ^ 1ULL);
1309
1310                                 /* Output port TX */
1311                                 if (pkt_mask != 0)
1312                                         port_out->ops.f_tx(port_out->h_port,
1313                                                 pkt);
1314                         }
1315                 }
1316         }
1317 }
1318
1319 static inline void
1320 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1321 {
1322         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1323                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1324                 uint32_t i;
1325
1326                 for (i = 0; i < n_pkts; i++)
1327                         rte_pktmbuf_free(p->pkts[i]);
1328         } else {
1329                 uint32_t i;
1330
1331                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1332                         uint64_t pkt_mask = 1LLU << i;
1333
1334                         if ((pkt_mask & pkts_mask) == 0)
1335                                 continue;
1336
1337                         rte_pktmbuf_free(p->pkts[i]);
1338                 }
1339         }
1340 }
1341
1342 int
1343 rte_pipeline_run(struct rte_pipeline *p)
1344 {
1345         struct rte_port_in *port_in;
1346
1347         for (port_in = p->port_in_first; port_in != NULL;
1348                 port_in = port_in->next) {
1349                 uint64_t pkts_mask;
1350                 uint32_t n_pkts, table_id;
1351
1352                 /* Input port RX */
1353                 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1354                         port_in->burst_size);
1355                 if (n_pkts == 0)
1356                         continue;
1357
1358                 pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1359                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1360                 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1361                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1362                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1363
1364                 /* Input port user actions */
1365                 if (port_in->f_action != NULL) {
1366                         uint64_t mask = pkts_mask;
1367
1368                         port_in->f_action(p->pkts, n_pkts, &pkts_mask, port_in->arg_ah);
1369                         mask ^= pkts_mask;
1370                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1371                         RTE_PIPELINE_STATS_ADD_M(port_in->n_pkts_dropped_by_ah, mask);
1372                 }
1373
1374                 /* Table */
1375                 for (table_id = port_in->table_id; pkts_mask != 0; ) {
1376                         struct rte_table *table;
1377                         uint64_t lookup_hit_mask, lookup_miss_mask;
1378
1379                         /* Lookup */
1380                         table = &p->tables[table_id];
1381                         table->ops.f_lookup(table->h_table, p->pkts, pkts_mask,
1382                                         &lookup_hit_mask, (void **) p->entries);
1383                         lookup_miss_mask = pkts_mask & (~lookup_hit_mask);
1384
1385                         /* Lookup miss */
1386                         if (lookup_miss_mask != 0) {
1387                                 struct rte_pipeline_table_entry *default_entry =
1388                                         table->default_entry;
1389
1390                                 /* Table user actions */
1391                                 if (table->f_action_miss != NULL) {
1392                                         uint64_t mask = lookup_miss_mask;
1393
1394                                         table->f_action_miss(p->pkts,
1395                                                 &lookup_miss_mask,
1396                                                 default_entry, table->arg_ah);
1397                                         mask ^= lookup_miss_mask;
1398                                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1399                                         RTE_PIPELINE_STATS_ADD_M(
1400                                                 table->n_pkts_dropped_by_lkp_miss_ah, mask);
1401                                 }
1402
1403                                 /* Table reserved actions */
1404                                 if ((default_entry->action ==
1405                                         RTE_PIPELINE_ACTION_PORT) &&
1406                                         (lookup_miss_mask != 0))
1407                                         rte_pipeline_action_handler_port_bulk(p,
1408                                                 lookup_miss_mask,
1409                                                 default_entry->port_id);
1410                                 else {
1411                                         uint32_t pos = default_entry->action;
1412
1413                                         p->action_mask0[pos] = lookup_miss_mask;
1414                                         if (pos == RTE_PIPELINE_ACTION_DROP) {
1415                                                 RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_miss,
1416                                                         lookup_miss_mask);
1417                                         }
1418                                 }
1419                         }
1420
1421                         /* Lookup hit */
1422                         if (lookup_hit_mask != 0) {
1423                                 /* Table user actions */
1424                                 if (table->f_action_hit != NULL) {
1425                                         uint64_t mask = lookup_hit_mask;
1426
1427                                         table->f_action_hit(p->pkts,
1428                                                 &lookup_hit_mask,
1429                                                 p->entries, table->arg_ah);
1430                                         mask ^= lookup_hit_mask;
1431                                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= mask;
1432                                         RTE_PIPELINE_STATS_ADD_M(
1433                                                 table->n_pkts_dropped_by_lkp_hit_ah, mask);
1434                                 }
1435
1436                                 /* Table reserved actions */
1437                                 rte_pipeline_compute_masks(p, lookup_hit_mask);
1438                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1439                                         p->action_mask1[
1440                                                 RTE_PIPELINE_ACTION_DROP];
1441                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1442                                         p->action_mask1[
1443                                                 RTE_PIPELINE_ACTION_PORT];
1444                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1445                                         p->action_mask1[
1446                                                 RTE_PIPELINE_ACTION_PORT_META];
1447                                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1448                                         p->action_mask1[
1449                                                 RTE_PIPELINE_ACTION_TABLE];
1450
1451                                 RTE_PIPELINE_STATS_ADD_M(table->n_pkts_dropped_lkp_hit,
1452                                                 p->action_mask1[RTE_PIPELINE_ACTION_DROP]);
1453                         }
1454
1455                         /* Prepare for next iteration */
1456                         pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1457                         table_id = table->table_next_id;
1458                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1459                 }
1460
1461                 /* Table reserved action PORT */
1462                 rte_pipeline_action_handler_port(p,
1463                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1464
1465                 /* Table reserved action PORT META */
1466                 rte_pipeline_action_handler_port_meta(p,
1467                                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1468
1469                 /* Table reserved action DROP */
1470                 rte_pipeline_action_handler_drop(p,
1471                                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1472         }
1473
1474         return 0;
1475 }
1476
1477 int
1478 rte_pipeline_flush(struct rte_pipeline *p)
1479 {
1480         uint32_t port_id;
1481
1482         /* Check input arguments */
1483         if (p == NULL) {
1484                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1485                         __func__);
1486                 return -EINVAL;
1487         }
1488
1489         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1490                 struct rte_port_out *port = &p->ports_out[port_id];
1491
1492                 if (port->ops.f_flush != NULL)
1493                         port->ops.f_flush(port->h_port);
1494         }
1495
1496         return 0;
1497 }
1498
1499 int
1500 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1501                 uint32_t port_id, struct rte_mbuf *pkt)
1502 {
1503         struct rte_port_out *port_out = &p->ports_out[port_id];
1504
1505         /* Output port user actions */
1506         if (port_out->f_action == NULL)
1507                 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1508         else {
1509                 uint64_t pkt_mask = 1LLU;
1510
1511                 port_out->f_action(pkt, &pkt_mask, port_out->arg_ah);
1512
1513                 if (pkt_mask != 0) /* Output port TX */
1514                         port_out->ops.f_tx(port_out->h_port, pkt);
1515                 else {
1516                         rte_pktmbuf_free(pkt);
1517                         RTE_PIPELINE_STATS_ADD(port_out->n_pkts_dropped_by_ah, 1);
1518                 }
1519         }
1520
1521         return 0;
1522 }
1523
1524 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1525         struct rte_pipeline_port_in_stats *stats, int clear)
1526 {
1527         struct rte_port_in *port;
1528         int retval;
1529
1530         if (p == NULL) {
1531                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1532                         __func__);
1533                 return -EINVAL;
1534         }
1535
1536         if (port_id >= p->num_ports_in) {
1537                 RTE_LOG(ERR, PIPELINE,
1538                         "%s: port IN ID %u is out of range\n",
1539                         __func__, port_id);
1540                 return -EINVAL;
1541         }
1542
1543         port = &p->ports_in[port_id];
1544
1545         if (port->ops.f_stats != NULL) {
1546                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1547                 if (retval)
1548                         return retval;
1549         } else if (stats != NULL)
1550                 memset(&stats->stats, 0, sizeof(stats->stats));
1551
1552         if (stats != NULL)
1553                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1554
1555         if (clear != 0)
1556                 port->n_pkts_dropped_by_ah = 0;
1557
1558         return 0;
1559 }
1560
1561 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1562         struct rte_pipeline_port_out_stats *stats, int clear)
1563 {
1564         struct rte_port_out *port;
1565         int retval;
1566
1567         if (p == NULL) {
1568                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1569                 return -EINVAL;
1570         }
1571
1572         if (port_id >= p->num_ports_out) {
1573                 RTE_LOG(ERR, PIPELINE,
1574                         "%s: port OUT ID %u is out of range\n", __func__, port_id);
1575                 return -EINVAL;
1576         }
1577
1578         port = &p->ports_out[port_id];
1579         if (port->ops.f_stats != NULL) {
1580                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1581                 if (retval != 0)
1582                         return retval;
1583         } else if (stats != NULL)
1584                 memset(&stats->stats, 0, sizeof(stats->stats));
1585
1586         if (stats != NULL)
1587                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1588
1589         if (clear != 0)
1590                 port->n_pkts_dropped_by_ah = 0;
1591
1592         return 0;
1593 }
1594
1595 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1596         struct rte_pipeline_table_stats *stats, int clear)
1597 {
1598         struct rte_table *table;
1599         int retval;
1600
1601         if (p == NULL) {
1602                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1603                         __func__);
1604                 return -EINVAL;
1605         }
1606
1607         if (table_id >= p->num_tables) {
1608                 RTE_LOG(ERR, PIPELINE,
1609                                 "%s: table %u is out of range\n", __func__, table_id);
1610                 return -EINVAL;
1611         }
1612
1613         table = &p->tables[table_id];
1614         if (table->ops.f_stats != NULL) {
1615                 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1616                 if (retval != 0)
1617                         return retval;
1618         } else if (stats != NULL)
1619                 memset(&stats->stats, 0, sizeof(stats->stats));
1620
1621         if (stats != NULL) {
1622                 stats->n_pkts_dropped_by_lkp_hit_ah =
1623                         table->n_pkts_dropped_by_lkp_hit_ah;
1624                 stats->n_pkts_dropped_by_lkp_miss_ah =
1625                         table->n_pkts_dropped_by_lkp_miss_ah;
1626                 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1627                 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1628         }
1629
1630         if (clear != 0) {
1631                 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1632                 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1633                 table->n_pkts_dropped_lkp_hit = 0;
1634                 table->n_pkts_dropped_lkp_miss = 0;
1635         }
1636
1637         return 0;
1638 }
1639