lib/librte_pipeline/rte_pipeline.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_memory.h>
  39 #include <rte_memzone.h>
  40 #include <rte_cycles.h>
  41 #include <rte_prefetch.h>
  42 #include <rte_branch_prediction.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_malloc.h>
  45 #include <rte_string_fns.h>
  46
  47 #include "rte_pipeline.h"
  48
  49 #define RTE_TABLE_INVALID                                 UINT32_MAX
  50
  51 #ifdef RTE_PIPELINE_STATS_COLLECT
  52
  53 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)                       \
  54         ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
  55
  56 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)                     \
  57         ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
  58
  59 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)                               \
  60         ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
  61
  62 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)                      \
  63 ({                                                                      \
  64         uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP];    \
  65         mask ^= (p)->pkts_drop_mask;                                    \
  66         (counter) += __builtin_popcountll(mask);                        \
  67 })
  68
  69 #else
  70
  71 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
  72 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
  73 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
  74 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
  75
  76 #endif
  77
  78 struct rte_port_in {
  79         /* Input parameters */
  80         struct rte_port_in_ops ops;
  81         rte_pipeline_port_in_action_handler f_action;
  82         void *arg_ah;
  83         uint32_t burst_size;
  84
  85         /* The table to which this port is connected */
  86         uint32_t table_id;
  87
  88         /* Handle to low-level port */
  89         void *h_port;
  90
  91         /* List of enabled ports */
  92         struct rte_port_in *next;
  93
  94         /* Statistics */
  95         uint64_t n_pkts_dropped_by_ah;
  96 };
  97
  98 struct rte_port_out {
  99         /* Input parameters */
 100         struct rte_port_out_ops ops;
 101         rte_pipeline_port_out_action_handler f_action;
 102         void *arg_ah;
 103
 104         /* Handle to low-level port */
 105         void *h_port;
 106
 107         /* Statistics */
 108         uint64_t n_pkts_dropped_by_ah;
 109 };
 110
 111 struct rte_table {
 112         /* Input parameters */
 113         struct rte_table_ops ops;
 114         rte_pipeline_table_action_handler_hit f_action_hit;
 115         rte_pipeline_table_action_handler_miss f_action_miss;
 116         void *arg_ah;
 117         struct rte_pipeline_table_entry *default_entry;
 118         uint32_t entry_size;
 119
 120         uint32_t table_next_id;
 121         uint32_t table_next_id_valid;
 122
 123         /* Handle to the low-level table object */
 124         void *h_table;
 125
 126         /* Statistics */
 127         uint64_t n_pkts_dropped_by_lkp_hit_ah;
 128         uint64_t n_pkts_dropped_by_lkp_miss_ah;
 129         uint64_t n_pkts_dropped_lkp_hit;
 130         uint64_t n_pkts_dropped_lkp_miss;
 131 };
 132
 133 #define RTE_PIPELINE_MAX_NAME_SZ                           124
 134
 135 struct rte_pipeline {
 136         /* Input parameters */
 137         char name[RTE_PIPELINE_MAX_NAME_SZ];
 138         int socket_id;
 139         uint32_t offset_port_id;
 140
 141         /* Internal tables */
 142         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 143         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 144         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 145
 146         /* Occupancy of internal tables */
 147         uint32_t num_ports_in;
 148         uint32_t num_ports_out;
 149         uint32_t num_tables;
 150
 151         /* List of enabled ports */
 152         uint64_t enabled_port_in_mask;
 153         struct rte_port_in *port_in_next;
 154
 155         /* Pipeline run structures */
 156         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 157         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 158         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 159         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 160         uint64_t pkts_mask;
 161         uint64_t n_pkts_ah_drop;
 162         uint64_t pkts_drop_mask;
 163 } __rte_cache_aligned;
 164
 165 static inline uint32_t
 166 rte_mask_get_next(uint64_t mask, uint32_t pos)
 167 {
 168         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 169                         (mask >> ((pos + 1) & 0x3F));
 170         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 171 }
 172
 173 static inline uint32_t
 174 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 175 {
 176         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 177                         (mask << ((64 - pos) & 0x3F));
 178         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 179 }
 180
 181 static void
 182 rte_pipeline_table_free(struct rte_table *table);
 183
 184 static void
 185 rte_pipeline_port_in_free(struct rte_port_in *port);
 186
 187 static void
 188 rte_pipeline_port_out_free(struct rte_port_out *port);
 189
 190 /*
 191  * Pipeline
 192  *
 193  */
 194 static int
 195 rte_pipeline_check_params(struct rte_pipeline_params *params)
 196 {
 197         if (params == NULL) {
 198                 RTE_LOG(ERR, PIPELINE,
 199                         "%s: Incorrect value for parameter params\n", __func__);
 200                 return -EINVAL;
 201         }
 202
 203         /* name */
 204         if (params->name == NULL) {
 205                 RTE_LOG(ERR, PIPELINE,
 206                         "%s: Incorrect value for parameter name\n", __func__);
 207                 return -EINVAL;
 208         }
 209
 210         /* socket */
 211         if ((params->socket_id < 0) ||
 212             (params->socket_id >= RTE_MAX_NUMA_NODES)) {
 213                 RTE_LOG(ERR, PIPELINE,
 214                         "%s: Incorrect value for parameter socket_id\n",
 215                         __func__);
 216                 return -EINVAL;
 217         }
 218
 219         return 0;
 220 }
 221
 222 struct rte_pipeline *
 223 rte_pipeline_create(struct rte_pipeline_params *params)
 224 {
 225         struct rte_pipeline *p;
 226         int status;
 227
 228         /* Check input parameters */
 229         status = rte_pipeline_check_params(params);
 230         if (status != 0) {
 231                 RTE_LOG(ERR, PIPELINE,
 232                         "%s: Pipeline params check failed (%d)\n",
 233                         __func__, status);
 234                 return NULL;
 235         }
 236
 237         /* Allocate memory for the pipeline on requested socket */
 238         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 239                         RTE_CACHE_LINE_SIZE, params->socket_id);
 240
 241         if (p == NULL) {
 242                 RTE_LOG(ERR, PIPELINE,
 243                         "%s: Pipeline memory allocation failed\n", __func__);
 244                 return NULL;
 245         }
 246
 247         /* Save input parameters */
 248         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
 249         p->socket_id = params->socket_id;
 250         p->offset_port_id = params->offset_port_id;
 251
 252         /* Initialize pipeline internal data structure */
 253         p->num_ports_in = 0;
 254         p->num_ports_out = 0;
 255         p->num_tables = 0;
 256         p->enabled_port_in_mask = 0;
 257         p->port_in_next = NULL;
 258         p->pkts_mask = 0;
 259         p->n_pkts_ah_drop = 0;
 260
 261         return p;
 262 }
 263
 264 int
 265 rte_pipeline_free(struct rte_pipeline *p)
 266 {
 267         uint32_t i;
 268
 269         /* Check input parameters */
 270         if (p == NULL) {
 271                 RTE_LOG(ERR, PIPELINE,
 272                         "%s: rte_pipeline parameter is NULL\n", __func__);
 273                 return -EINVAL;
 274         }
 275
 276         /* Free input ports */
 277         for (i = 0; i < p->num_ports_in; i++) {
 278                 struct rte_port_in *port = &p->ports_in[i];
 279
 280                 rte_pipeline_port_in_free(port);
 281         }
 282
 283         /* Free tables */
 284         for (i = 0; i < p->num_tables; i++) {
 285                 struct rte_table *table = &p->tables[i];
 286
 287                 rte_pipeline_table_free(table);
 288         }
 289
 290         /* Free output ports */
 291         for (i = 0; i < p->num_ports_out; i++) {
 292                 struct rte_port_out *port = &p->ports_out[i];
 293
 294                 rte_pipeline_port_out_free(port);
 295         }
 296
 297         /* Free pipeline memory */
 298         rte_free(p);
 299
 300         return 0;
 301 }
 302
 303 /*
 304  * Table
 305  *
 306  */
 307 static int
 308 rte_table_check_params(struct rte_pipeline *p,
 309                 struct rte_pipeline_table_params *params,
 310                 uint32_t *table_id)
 311 {
 312         if (p == NULL) {
 313                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 314                         __func__);
 315                 return -EINVAL;
 316         }
 317         if (params == NULL) {
 318                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 319                         __func__);
 320                 return -EINVAL;
 321         }
 322         if (table_id == NULL) {
 323                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 324                         __func__);
 325                 return -EINVAL;
 326         }
 327
 328         /* ops */
 329         if (params->ops == NULL) {
 330                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 331                         __func__);
 332                 return -EINVAL;
 333         }
 334
 335         if (params->ops->f_create == NULL) {
 336                 RTE_LOG(ERR, PIPELINE,
 337                         "%s: f_create function pointer is NULL\n", __func__);
 338                 return -EINVAL;
 339         }
 340
 341         if (params->ops->f_lookup == NULL) {
 342                 RTE_LOG(ERR, PIPELINE,
 343                         "%s: f_lookup function pointer is NULL\n", __func__);
 344                 return -EINVAL;
 345         }
 346
 347         /* De we have room for one more table? */
 348         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 349                 RTE_LOG(ERR, PIPELINE,
 350                         "%s: Incorrect value for num_tables parameter\n",
 351                         __func__);
 352                 return -EINVAL;
 353         }
 354
 355         return 0;
 356 }
 357
 358 int
 359 rte_pipeline_table_create(struct rte_pipeline *p,
 360                 struct rte_pipeline_table_params *params,
 361                 uint32_t *table_id)
 362 {
 363         struct rte_table *table;
 364         struct rte_pipeline_table_entry *default_entry;
 365         void *h_table;
 366         uint32_t entry_size, id;
 367         int status;
 368
 369         /* Check input arguments */
 370         status = rte_table_check_params(p, params, table_id);
 371         if (status != 0)
 372                 return status;
 373
 374         id = p->num_tables;
 375         table = &p->tables[id];
 376
 377         /* Allocate space for the default table entry */
 378         entry_size = sizeof(struct rte_pipeline_table_entry) +
 379                 params->action_data_size;
 380         default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
 381                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 382         if (default_entry == NULL) {
 383                 RTE_LOG(ERR, PIPELINE,
 384                         "%s: Failed to allocate default entry\n", __func__);
 385                 return -EINVAL;
 386         }
 387
 388         /* Create the table */
 389         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 390                 entry_size);
 391         if (h_table == NULL) {
 392                 rte_free(default_entry);
 393                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 394                 return -EINVAL;
 395         }
 396
 397         /* Commit current table to the pipeline */
 398         p->num_tables++;
 399         *table_id = id;
 400
 401         /* Save input parameters */
 402         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 403         table->f_action_hit = params->f_action_hit;
 404         table->f_action_miss = params->f_action_miss;
 405         table->arg_ah = params->arg_ah;
 406         table->entry_size = entry_size;
 407
 408         /* Clear the lookup miss actions (to be set later through API) */
 409         table->default_entry = default_entry;
 410         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 411
 412         /* Initialize table internal data structure */
 413         table->h_table = h_table;
 414         table->table_next_id = 0;
 415         table->table_next_id_valid = 0;
 416
 417         return 0;
 418 }
 419
 420 void
 421 rte_pipeline_table_free(struct rte_table *table)
 422 {
 423         if (table->ops.f_free != NULL)
 424                 table->ops.f_free(table->h_table);
 425
 426         rte_free(table->default_entry);
 427 }
 428
 429 int
 430 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 431         uint32_t table_id,
 432         struct rte_pipeline_table_entry *default_entry,
 433         struct rte_pipeline_table_entry **default_entry_ptr)
 434 {
 435         struct rte_table *table;
 436
 437         /* Check input arguments */
 438         if (p == NULL) {
 439                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 440                         __func__);
 441                 return -EINVAL;
 442         }
 443
 444         if (default_entry == NULL) {
 445                 RTE_LOG(ERR, PIPELINE,
 446                         "%s: default_entry parameter is NULL\n", __func__);
 447                 return -EINVAL;
 448         }
 449
 450         if (table_id >= p->num_tables) {
 451                 RTE_LOG(ERR, PIPELINE,
 452                         "%s: table_id %d out of range\n", __func__, table_id);
 453                 return -EINVAL;
 454         }
 455
 456         table = &p->tables[table_id];
 457
 458         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 459                 table->table_next_id_valid &&
 460                 (default_entry->table_id != table->table_next_id)) {
 461                 RTE_LOG(ERR, PIPELINE,
 462                         "%s: Tree-like topologies not allowed\n", __func__);
 463                 return -EINVAL;
 464         }
 465
 466         /* Set the lookup miss actions */
 467         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 468                 (table->table_next_id_valid == 0)) {
 469                 table->table_next_id = default_entry->table_id;
 470                 table->table_next_id_valid = 1;
 471         }
 472
 473         memcpy(table->default_entry, default_entry, table->entry_size);
 474
 475         *default_entry_ptr = table->default_entry;
 476         return 0;
 477 }
 478
 479 int
 480 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 481                 uint32_t table_id,
 482                 struct rte_pipeline_table_entry *entry)
 483 {
 484         struct rte_table *table;
 485
 486         /* Check input arguments */
 487         if (p == NULL) {
 488                 RTE_LOG(ERR, PIPELINE,
 489                         "%s: pipeline parameter is NULL\n", __func__);
 490                 return -EINVAL;
 491         }
 492
 493         if (table_id >= p->num_tables) {
 494                 RTE_LOG(ERR, PIPELINE,
 495                         "%s: table_id %d out of range\n", __func__, table_id);
 496                 return -EINVAL;
 497         }
 498
 499         table = &p->tables[table_id];
 500
 501         /* Save the current contents of the default entry */
 502         if (entry)
 503                 memcpy(entry, table->default_entry, table->entry_size);
 504
 505         /* Clear the lookup miss actions */
 506         memset(table->default_entry, 0, table->entry_size);
 507         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 508
 509         return 0;
 510 }
 511
 512 int
 513 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 514                 uint32_t table_id,
 515                 void *key,
 516                 struct rte_pipeline_table_entry *entry,
 517                 int *key_found,
 518                 struct rte_pipeline_table_entry **entry_ptr)
 519 {
 520         struct rte_table *table;
 521
 522         /* Check input arguments */
 523         if (p == NULL) {
 524                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 525                         __func__);
 526                 return -EINVAL;
 527         }
 528
 529         if (key == NULL) {
 530                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 531                 return -EINVAL;
 532         }
 533
 534         if (entry == NULL) {
 535                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 536                         __func__);
 537                 return -EINVAL;
 538         }
 539
 540         if (table_id >= p->num_tables) {
 541                 RTE_LOG(ERR, PIPELINE,
 542                         "%s: table_id %d out of range\n", __func__, table_id);
 543                 return -EINVAL;
 544         }
 545
 546         table = &p->tables[table_id];
 547
 548         if (table->ops.f_add == NULL) {
 549                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 550                         __func__);
 551                 return -EINVAL;
 552         }
 553
 554         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 555                 table->table_next_id_valid &&
 556                 (entry->table_id != table->table_next_id)) {
 557                 RTE_LOG(ERR, PIPELINE,
 558                         "%s: Tree-like topologies not allowed\n", __func__);
 559                 return -EINVAL;
 560         }
 561
 562         /* Add entry */
 563         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 564                 (table->table_next_id_valid == 0)) {
 565                 table->table_next_id = entry->table_id;
 566                 table->table_next_id_valid = 1;
 567         }
 568
 569         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 570                 key_found, (void **) entry_ptr);
 571 }
 572
 573 int
 574 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 575                 uint32_t table_id,
 576                 void *key,
 577                 int *key_found,
 578                 struct rte_pipeline_table_entry *entry)
 579 {
 580         struct rte_table *table;
 581
 582         /* Check input arguments */
 583         if (p == NULL) {
 584                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 585                         __func__);
 586                 return -EINVAL;
 587         }
 588
 589         if (key == NULL) {
 590                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 591                         __func__);
 592                 return -EINVAL;
 593         }
 594
 595         if (table_id >= p->num_tables) {
 596                 RTE_LOG(ERR, PIPELINE,
 597                         "%s: table_id %d out of range\n", __func__, table_id);
 598                 return -EINVAL;
 599         }
 600
 601         table = &p->tables[table_id];
 602
 603         if (table->ops.f_delete == NULL) {
 604                 RTE_LOG(ERR, PIPELINE,
 605                         "%s: f_delete function pointer NULL\n", __func__);
 606                 return -EINVAL;
 607         }
 608
 609         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 610 }
 611
 612 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
 613         uint32_t table_id,
 614         void **keys,
 615         struct rte_pipeline_table_entry **entries,
 616         uint32_t n_keys,
 617         int *key_found,
 618         struct rte_pipeline_table_entry **entries_ptr)
 619 {
 620         struct rte_table *table;
 621         uint32_t i;
 622
 623         /* Check input arguments */
 624         if (p == NULL) {
 625                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 626                         __func__);
 627                 return -EINVAL;
 628         }
 629
 630         if (keys == NULL) {
 631                 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
 632                 return -EINVAL;
 633         }
 634
 635         if (entries == NULL) {
 636                 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
 637                         __func__);
 638                 return -EINVAL;
 639         }
 640
 641         if (table_id >= p->num_tables) {
 642                 RTE_LOG(ERR, PIPELINE,
 643                         "%s: table_id %d out of range\n", __func__, table_id);
 644                 return -EINVAL;
 645         }
 646
 647         table = &p->tables[table_id];
 648
 649         if (table->ops.f_add_bulk == NULL) {
 650                 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
 651                         __func__);
 652                 return -EINVAL;
 653         }
 654
 655         for (i = 0; i < n_keys; i++) {
 656                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 657                         table->table_next_id_valid &&
 658                         (entries[i]->table_id != table->table_next_id)) {
 659                         RTE_LOG(ERR, PIPELINE,
 660                                 "%s: Tree-like topologies not allowed\n", __func__);
 661                         return -EINVAL;
 662                 }
 663         }
 664
 665         /* Add entry */
 666         for (i = 0; i < n_keys; i++) {
 667                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 668                         (table->table_next_id_valid == 0)) {
 669                         table->table_next_id = entries[i]->table_id;
 670                         table->table_next_id_valid = 1;
 671                 }
 672         }
 673
 674         return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
 675                 n_keys, key_found, (void **) entries_ptr);
 676 }
 677
 678 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
 679         uint32_t table_id,
 680         void **keys,
 681         uint32_t n_keys,
 682         int *key_found,
 683         struct rte_pipeline_table_entry **entries)
 684 {
 685         struct rte_table *table;
 686
 687         /* Check input arguments */
 688         if (p == NULL) {
 689                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 690                         __func__);
 691                 return -EINVAL;
 692         }
 693
 694         if (keys == NULL) {
 695                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 696                         __func__);
 697                 return -EINVAL;
 698         }
 699
 700         if (table_id >= p->num_tables) {
 701                 RTE_LOG(ERR, PIPELINE,
 702                         "%s: table_id %d out of range\n", __func__, table_id);
 703                 return -EINVAL;
 704         }
 705
 706         table = &p->tables[table_id];
 707
 708         if (table->ops.f_delete_bulk == NULL) {
 709                 RTE_LOG(ERR, PIPELINE,
 710                         "%s: f_delete function pointer NULL\n", __func__);
 711                 return -EINVAL;
 712         }
 713
 714         return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
 715                         (void **) entries);
 716 }
 717
 718 /*
 719  * Port
 720  *
 721  */
 722 static int
 723 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 724                 struct rte_pipeline_port_in_params *params,
 725                 uint32_t *port_id)
 726 {
 727         if (p == NULL) {
 728                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 729                         __func__);
 730                 return -EINVAL;
 731         }
 732         if (params == NULL) {
 733                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 734                 return -EINVAL;
 735         }
 736         if (port_id == NULL) {
 737                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 738                         __func__);
 739                 return -EINVAL;
 740         }
 741
 742         /* ops */
 743         if (params->ops == NULL) {
 744                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 745                         __func__);
 746                 return -EINVAL;
 747         }
 748
 749         if (params->ops->f_create == NULL) {
 750                 RTE_LOG(ERR, PIPELINE,
 751                         "%s: f_create function pointer NULL\n", __func__);
 752                 return -EINVAL;
 753         }
 754
 755         if (params->ops->f_rx == NULL) {
 756                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 757                         __func__);
 758                 return -EINVAL;
 759         }
 760
 761         /* burst_size */
 762         if ((params->burst_size == 0) ||
 763                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 764                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 765                         __func__);
 766                 return -EINVAL;
 767         }
 768
 769         /* Do we have room for one more port? */
 770         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 771                 RTE_LOG(ERR, PIPELINE,
 772                         "%s: invalid value for num_ports_in\n", __func__);
 773                 return -EINVAL;
 774         }
 775
 776         return 0;
 777 }
 778
 779 static int
 780 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 781                 struct rte_pipeline_port_out_params *params,
 782                 uint32_t *port_id)
 783 {
 784         if (p == NULL) {
 785                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 786                         __func__);
 787                 return -EINVAL;
 788         }
 789
 790         if (params == NULL) {
 791                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 792                 return -EINVAL;
 793         }
 794
 795         if (port_id == NULL) {
 796                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 797                         __func__);
 798                 return -EINVAL;
 799         }
 800
 801         /* ops */
 802         if (params->ops == NULL) {
 803                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 804                         __func__);
 805                 return -EINVAL;
 806         }
 807
 808         if (params->ops->f_create == NULL) {
 809                 RTE_LOG(ERR, PIPELINE,
 810                         "%s: f_create function pointer NULL\n", __func__);
 811                 return -EINVAL;
 812         }
 813
 814         if (params->ops->f_tx == NULL) {
 815                 RTE_LOG(ERR, PIPELINE,
 816                         "%s: f_tx function pointer NULL\n", __func__);
 817                 return -EINVAL;
 818         }
 819
 820         if (params->ops->f_tx_bulk == NULL) {
 821                 RTE_LOG(ERR, PIPELINE,
 822                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 823                 return -EINVAL;
 824         }
 825
 826         /* Do we have room for one more port? */
 827         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 828                 RTE_LOG(ERR, PIPELINE,
 829                         "%s: invalid value for num_ports_out\n", __func__);
 830                 return -EINVAL;
 831         }
 832
 833         return 0;
 834 }
 835
 836 int
 837 rte_pipeline_port_in_create(struct rte_pipeline *p,
 838                 struct rte_pipeline_port_in_params *params,
 839                 uint32_t *port_id)
 840 {
 841         struct rte_port_in *port;
 842         void *h_port;
 843         uint32_t id;
 844         int status;
 845
 846         /* Check input arguments */
 847         status = rte_pipeline_port_in_check_params(p, params, port_id);
 848         if (status != 0)
 849                 return status;
 850
 851         id = p->num_ports_in;
 852         port = &p->ports_in[id];
 853
 854         /* Create the port */
 855         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 856         if (h_port == NULL) {
 857                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 858                 return -EINVAL;
 859         }
 860
 861         /* Commit current table to the pipeline */
 862         p->num_ports_in++;
 863         *port_id = id;
 864
 865         /* Save input parameters */
 866         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 867         port->f_action = params->f_action;
 868         port->arg_ah = params->arg_ah;
 869         port->burst_size = params->burst_size;
 870
 871         /* Initialize port internal data structure */
 872         port->table_id = RTE_TABLE_INVALID;
 873         port->h_port = h_port;
 874         port->next = NULL;
 875
 876         return 0;
 877 }
 878
 879 void
 880 rte_pipeline_port_in_free(struct rte_port_in *port)
 881 {
 882         if (port->ops.f_free != NULL)
 883                 port->ops.f_free(port->h_port);
 884 }
 885
 886 int
 887 rte_pipeline_port_out_create(struct rte_pipeline *p,
 888                 struct rte_pipeline_port_out_params *params,
 889                 uint32_t *port_id)
 890 {
 891         struct rte_port_out *port;
 892         void *h_port;
 893         uint32_t id;
 894         int status;
 895
 896         /* Check input arguments */
 897         status = rte_pipeline_port_out_check_params(p, params, port_id);
 898         if (status != 0)
 899                 return status;
 900
 901         id = p->num_ports_out;
 902         port = &p->ports_out[id];
 903
 904         /* Create the port */
 905         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 906         if (h_port == NULL) {
 907                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 908                 return -EINVAL;
 909         }
 910
 911         /* Commit current table to the pipeline */
 912         p->num_ports_out++;
 913         *port_id = id;
 914
 915         /* Save input parameters */
 916         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 917         port->f_action = params->f_action;
 918         port->arg_ah = params->arg_ah;
 919
 920         /* Initialize port internal data structure */
 921         port->h_port = h_port;
 922
 923         return 0;
 924 }
 925
 926 void
 927 rte_pipeline_port_out_free(struct rte_port_out *port)
 928 {
 929         if (port->ops.f_free != NULL)
 930                 port->ops.f_free(port->h_port);
 931 }
 932
 933 int
 934 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 935                 uint32_t port_id,
 936                 uint32_t table_id)
 937 {
 938         struct rte_port_in *port;
 939
 940         /* Check input arguments */
 941         if (p == NULL) {
 942                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 943                         __func__);
 944                 return -EINVAL;
 945         }
 946
 947         if (port_id >= p->num_ports_in) {
 948                 RTE_LOG(ERR, PIPELINE,
 949                         "%s: port IN ID %u is out of range\n",
 950                         __func__, port_id);
 951                 return -EINVAL;
 952         }
 953
 954         if (table_id >= p->num_tables) {
 955                 RTE_LOG(ERR, PIPELINE,
 956                         "%s: Table ID %u is out of range\n",
 957                         __func__, table_id);
 958                 return -EINVAL;
 959         }
 960
 961         port = &p->ports_in[port_id];
 962         port->table_id = table_id;
 963
 964         return 0;
 965 }
 966
 967 int
 968 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 969 {
 970         struct rte_port_in *port, *port_prev, *port_next;
 971         uint64_t port_mask;
 972         uint32_t port_prev_id, port_next_id;
 973
 974         /* Check input arguments */
 975         if (p == NULL) {
 976                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 977                         __func__);
 978                 return -EINVAL;
 979         }
 980
 981         if (port_id >= p->num_ports_in) {
 982                 RTE_LOG(ERR, PIPELINE,
 983                         "%s: port IN ID %u is out of range\n",
 984                         __func__, port_id);
 985                 return -EINVAL;
 986         }
 987
 988         port = &p->ports_in[port_id];
 989
 990         /* Return if current input port is already enabled */
 991         port_mask = 1LLU << port_id;
 992         if (p->enabled_port_in_mask & port_mask)
 993                 return 0;
 994
 995         p->enabled_port_in_mask |= port_mask;
 996
 997         /* Add current input port to the pipeline chain of enabled ports */
 998         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 999         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1000
1001         port_prev = &p->ports_in[port_prev_id];
1002         port_next = &p->ports_in[port_next_id];
1003
1004         port_prev->next = port;
1005         port->next = port_next;
1006
1007         /* Check if list of enabled ports was previously empty */
1008         if (p->enabled_port_in_mask == port_mask)
1009                 p->port_in_next = port;
1010
1011         return 0;
1012 }
1013
1014 int
1015 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
1016 {
1017         struct rte_port_in *port, *port_prev, *port_next;
1018         uint64_t port_mask;
1019         uint32_t port_prev_id, port_next_id;
1020
1021         /* Check input arguments */
1022         if (p == NULL) {
1023                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1024                 __func__);
1025                 return -EINVAL;
1026         }
1027
1028         if (port_id >= p->num_ports_in) {
1029                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
1030                         __func__, port_id);
1031                 return -EINVAL;
1032         }
1033
1034         port = &p->ports_in[port_id];
1035
1036         /* Return if current input port is already disabled */
1037         port_mask = 1LLU << port_id;
1038         if ((p->enabled_port_in_mask & port_mask) == 0)
1039                 return 0;
1040
1041         p->enabled_port_in_mask &= ~port_mask;
1042
1043         /* Return if no other enabled ports */
1044         if (p->enabled_port_in_mask == 0) {
1045                 p->port_in_next = NULL;
1046
1047                 return 0;
1048         }
1049
1050         /* Add current input port to the pipeline chain of enabled ports */
1051         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1052         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1053
1054         port_prev = &p->ports_in[port_prev_id];
1055         port_next = &p->ports_in[port_next_id];
1056
1057         port_prev->next = port_next;
1058
1059         /* Check if the port which has just been disabled is next to serve */
1060         if (port == p->port_in_next)
1061                 p->port_in_next = port_next;
1062
1063         return 0;
1064 }
1065
1066 /*
1067  * Pipeline run-time
1068  *
1069  */
1070 int
1071 rte_pipeline_check(struct rte_pipeline *p)
1072 {
1073         uint32_t port_in_id;
1074
1075         /* Check input arguments */
1076         if (p == NULL) {
1077                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1078                         __func__);
1079                 return -EINVAL;
1080         }
1081
1082         /* Check that pipeline has at least one input port, one table and one
1083         output port */
1084         if (p->num_ports_in == 0) {
1085                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1086                         __func__);
1087                 return -EINVAL;
1088         }
1089         if (p->num_tables == 0) {
1090                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1091                         __func__);
1092                 return -EINVAL;
1093         }
1094         if (p->num_ports_out == 0) {
1095                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1096                         __func__);
1097                 return -EINVAL;
1098         }
1099
1100         /* Check that all input ports are connected */
1101         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1102                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1103
1104                 if (port_in->table_id == RTE_TABLE_INVALID) {
1105                         RTE_LOG(ERR, PIPELINE,
1106                                 "%s: Port IN ID %u is not connected\n",
1107                                 __func__, port_in_id);
1108                         return -EINVAL;
1109                 }
1110         }
1111
1112         return 0;
1113 }
1114
1115 static inline void
1116 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1117 {
1118         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1119         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1120         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1121         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1122
1123         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1124                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1125                 uint32_t i;
1126
1127                 for (i = 0; i < n_pkts; i++) {
1128                         uint64_t pkt_mask = 1LLU << i;
1129                         uint32_t pos = p->entries[i]->action;
1130
1131                         p->action_mask1[pos] |= pkt_mask;
1132                 }
1133         } else {
1134                 uint32_t i;
1135
1136                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1137                         uint64_t pkt_mask = 1LLU << i;
1138                         uint32_t pos;
1139
1140                         if ((pkt_mask & pkts_mask) == 0)
1141                                 continue;
1142
1143                         pos = p->entries[i]->action;
1144                         p->action_mask1[pos] |= pkt_mask;
1145                 }
1146         }
1147 }
1148
1149 static inline void
1150 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1151         uint64_t pkts_mask, uint32_t port_id)
1152 {
1153         struct rte_port_out *port_out = &p->ports_out[port_id];
1154
1155         p->pkts_mask = pkts_mask;
1156
1157         /* Output port user actions */
1158         if (port_out->f_action != NULL) {
1159                 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1160
1161                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1162                         port_out->n_pkts_dropped_by_ah);
1163         }
1164
1165         /* Output port TX */
1166         if (p->pkts_mask != 0)
1167                 port_out->ops.f_tx_bulk(port_out->h_port,
1168                         p->pkts,
1169                         p->pkts_mask);
1170 }
1171
1172 static inline void
1173 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1174 {
1175         p->pkts_mask = pkts_mask;
1176
1177         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1178                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1179                 uint32_t i;
1180
1181                 for (i = 0; i < n_pkts; i++) {
1182                         struct rte_mbuf *pkt = p->pkts[i];
1183                         uint32_t port_out_id = p->entries[i]->port_id;
1184                         struct rte_port_out *port_out =
1185                                 &p->ports_out[port_out_id];
1186
1187                         /* Output port user actions */
1188                         if (port_out->f_action == NULL) /* Output port TX */
1189                                 port_out->ops.f_tx(port_out->h_port, pkt);
1190                         else {
1191                                 uint64_t pkt_mask = 1LLU << i;
1192
1193                                 port_out->f_action(p,
1194                                         p->pkts,
1195                                         pkt_mask,
1196                                         port_out->arg_ah);
1197
1198                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1199                                         port_out->n_pkts_dropped_by_ah);
1200
1201                                 /* Output port TX */
1202                                 if (pkt_mask & p->pkts_mask)
1203                                         port_out->ops.f_tx(port_out->h_port,
1204                                                 pkt);
1205                         }
1206                 }
1207         } else {
1208                 uint32_t i;
1209
1210                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1211                         uint64_t pkt_mask = 1LLU << i;
1212                         struct rte_mbuf *pkt;
1213                         struct rte_port_out *port_out;
1214                         uint32_t port_out_id;
1215
1216                         if ((pkt_mask & pkts_mask) == 0)
1217                                 continue;
1218
1219                         pkt = p->pkts[i];
1220                         port_out_id = p->entries[i]->port_id;
1221                         port_out = &p->ports_out[port_out_id];
1222
1223                         /* Output port user actions */
1224                         if (port_out->f_action == NULL) /* Output port TX */
1225                                 port_out->ops.f_tx(port_out->h_port, pkt);
1226                         else {
1227                                 port_out->f_action(p,
1228                                         p->pkts,
1229                                         pkt_mask,
1230                                         port_out->arg_ah);
1231
1232                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1233                                         port_out->n_pkts_dropped_by_ah);
1234
1235                                 /* Output port TX */
1236                                 if (pkt_mask & p->pkts_mask)
1237                                         port_out->ops.f_tx(port_out->h_port,
1238                                                 pkt);
1239                         }
1240                 }
1241         }
1242 }
1243
1244 static inline void
1245 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1246         uint64_t pkts_mask)
1247 {
1248         p->pkts_mask = pkts_mask;
1249
1250         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1251                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1252                 uint32_t i;
1253
1254                 for (i = 0; i < n_pkts; i++) {
1255                         struct rte_mbuf *pkt = p->pkts[i];
1256                         uint32_t port_out_id =
1257                                 RTE_MBUF_METADATA_UINT32(pkt,
1258                                         p->offset_port_id);
1259                         struct rte_port_out *port_out = &p->ports_out[
1260                                 port_out_id];
1261
1262                         /* Output port user actions */
1263                         if (port_out->f_action == NULL) /* Output port TX */
1264                                 port_out->ops.f_tx(port_out->h_port, pkt);
1265                         else {
1266                                 uint64_t pkt_mask = 1LLU << i;
1267
1268                                 port_out->f_action(p,
1269                                         p->pkts,
1270                                         pkt_mask,
1271                                         port_out->arg_ah);
1272
1273                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1274                                         port_out->n_pkts_dropped_by_ah);
1275
1276                                 /* Output port TX */
1277                                 if (pkt_mask & p->pkts_mask)
1278                                         port_out->ops.f_tx(port_out->h_port,
1279                                                 pkt);
1280                         }
1281                 }
1282         } else {
1283                 uint32_t i;
1284
1285                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1286                         uint64_t pkt_mask = 1LLU << i;
1287                         struct rte_mbuf *pkt;
1288                         struct rte_port_out *port_out;
1289                         uint32_t port_out_id;
1290
1291                         if ((pkt_mask & pkts_mask) == 0)
1292                                 continue;
1293
1294                         pkt = p->pkts[i];
1295                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1296                                 p->offset_port_id);
1297                         port_out = &p->ports_out[port_out_id];
1298
1299                         /* Output port user actions */
1300                         if (port_out->f_action == NULL) /* Output port TX */
1301                                 port_out->ops.f_tx(port_out->h_port, pkt);
1302                         else {
1303                                 port_out->f_action(p,
1304                                         p->pkts,
1305                                         pkt_mask,
1306                                         port_out->arg_ah);
1307
1308                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1309                                         port_out->n_pkts_dropped_by_ah);
1310
1311                                 /* Output port TX */
1312                                 if (pkt_mask & p->pkts_mask)
1313                                         port_out->ops.f_tx(port_out->h_port,
1314                                                 pkt);
1315                         }
1316                 }
1317         }
1318 }
1319
1320 static inline void
1321 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1322 {
1323         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1324                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1325                 uint32_t i;
1326
1327                 for (i = 0; i < n_pkts; i++)
1328                         rte_pktmbuf_free(p->pkts[i]);
1329         } else {
1330                 uint32_t i;
1331
1332                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1333                         uint64_t pkt_mask = 1LLU << i;
1334
1335                         if ((pkt_mask & pkts_mask) == 0)
1336                                 continue;
1337
1338                         rte_pktmbuf_free(p->pkts[i]);
1339                 }
1340         }
1341 }
1342
1343 int
1344 rte_pipeline_run(struct rte_pipeline *p)
1345 {
1346         struct rte_port_in *port_in = p->port_in_next;
1347         uint32_t n_pkts, table_id;
1348
1349         if (port_in == NULL)
1350                 return 0;
1351
1352         /* Input port RX */
1353         n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1354                 port_in->burst_size);
1355         if (n_pkts == 0) {
1356                 p->port_in_next = port_in->next;
1357                 return 0;
1358         }
1359
1360         p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1361         p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1362         p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1363         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1364         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1365
1366         /* Input port user actions */
1367         if (port_in->f_action != NULL) {
1368                 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1369
1370                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1371                         port_in->n_pkts_dropped_by_ah);
1372         }
1373
1374         /* Table */
1375         for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1376                 struct rte_table *table;
1377                 uint64_t lookup_hit_mask, lookup_miss_mask;
1378
1379                 /* Lookup */
1380                 table = &p->tables[table_id];
1381                 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1382                         &lookup_hit_mask, (void **) p->entries);
1383                 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1384
1385                 /* Lookup miss */
1386                 if (lookup_miss_mask != 0) {
1387                         struct rte_pipeline_table_entry *default_entry =
1388                                 table->default_entry;
1389
1390                         p->pkts_mask = lookup_miss_mask;
1391
1392                         /* Table user actions */
1393                         if (table->f_action_miss != NULL) {
1394                                 table->f_action_miss(p,
1395                                         p->pkts,
1396                                         lookup_miss_mask,
1397                                         default_entry,
1398                                         table->arg_ah);
1399
1400                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1401                                         table->n_pkts_dropped_by_lkp_miss_ah);
1402                         }
1403
1404                         /* Table reserved actions */
1405                         if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1406                                 (p->pkts_mask != 0))
1407                                 rte_pipeline_action_handler_port_bulk(p,
1408                                         p->pkts_mask,
1409                                         default_entry->port_id);
1410                         else {
1411                                 uint32_t pos = default_entry->action;
1412
1413                                 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1414
1415                                 p->action_mask0[pos] |= p->pkts_mask;
1416
1417                                 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1418                                         table->n_pkts_dropped_lkp_miss);
1419                         }
1420                 }
1421
1422                 /* Lookup hit */
1423                 if (lookup_hit_mask != 0) {
1424                         p->pkts_mask = lookup_hit_mask;
1425
1426                         /* Table user actions */
1427                         if (table->f_action_hit != NULL) {
1428                                 table->f_action_hit(p,
1429                                         p->pkts,
1430                                         lookup_hit_mask,
1431                                         p->entries,
1432                                         table->arg_ah);
1433
1434                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1435                                         table->n_pkts_dropped_by_lkp_hit_ah);
1436                         }
1437
1438                         /* Table reserved actions */
1439                         RTE_PIPELINE_STATS_TABLE_DROP0(p);
1440                         rte_pipeline_compute_masks(p, p->pkts_mask);
1441                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1442                                 p->action_mask1[
1443                                         RTE_PIPELINE_ACTION_DROP];
1444                         p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1445                                 p->action_mask1[
1446                                         RTE_PIPELINE_ACTION_PORT];
1447                         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1448                                 p->action_mask1[
1449                                         RTE_PIPELINE_ACTION_PORT_META];
1450                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1451                                 p->action_mask1[
1452                                         RTE_PIPELINE_ACTION_TABLE];
1453
1454                         RTE_PIPELINE_STATS_TABLE_DROP1(p,
1455                                 table->n_pkts_dropped_lkp_hit);
1456                 }
1457
1458                 /* Prepare for next iteration */
1459                 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1460                 table_id = table->table_next_id;
1461                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1462         }
1463
1464         /* Table reserved action PORT */
1465         rte_pipeline_action_handler_port(p,
1466                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1467
1468         /* Table reserved action PORT META */
1469         rte_pipeline_action_handler_port_meta(p,
1470                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1471
1472         /* Table reserved action DROP */
1473         rte_pipeline_action_handler_drop(p,
1474                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1475
1476         /* Pick candidate for next port IN to serve */
1477         p->port_in_next = port_in->next;
1478
1479         return (int) n_pkts;
1480 }
1481
1482 int
1483 rte_pipeline_flush(struct rte_pipeline *p)
1484 {
1485         uint32_t port_id;
1486
1487         /* Check input arguments */
1488         if (p == NULL) {
1489                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1490                         __func__);
1491                 return -EINVAL;
1492         }
1493
1494         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1495                 struct rte_port_out *port = &p->ports_out[port_id];
1496
1497                 if (port->ops.f_flush != NULL)
1498                         port->ops.f_flush(port->h_port);
1499         }
1500
1501         return 0;
1502 }
1503
1504 int
1505 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1506         uint32_t port_id, struct rte_mbuf *pkt)
1507 {
1508         struct rte_port_out *port_out = &p->ports_out[port_id];
1509
1510         port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1511
1512         return 0;
1513 }
1514
1515 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1516         uint64_t pkts_mask)
1517 {
1518         pkts_mask &= p->pkts_mask;
1519         p->pkts_mask &= ~pkts_mask;
1520
1521         return 0;
1522 }
1523
1524 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1525         uint64_t pkts_mask)
1526 {
1527         pkts_mask &= p->pkts_mask;
1528         p->pkts_mask &= ~pkts_mask;
1529         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1530
1531         RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1532         return 0;
1533 }
1534
1535 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1536         struct rte_pipeline_port_in_stats *stats, int clear)
1537 {
1538         struct rte_port_in *port;
1539         int retval;
1540
1541         if (p == NULL) {
1542                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1543                         __func__);
1544                 return -EINVAL;
1545         }
1546
1547         if (port_id >= p->num_ports_in) {
1548                 RTE_LOG(ERR, PIPELINE,
1549                         "%s: port IN ID %u is out of range\n",
1550                         __func__, port_id);
1551                 return -EINVAL;
1552         }
1553
1554         port = &p->ports_in[port_id];
1555
1556         if (port->ops.f_stats != NULL) {
1557                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1558                 if (retval)
1559                         return retval;
1560         } else if (stats != NULL)
1561                 memset(&stats->stats, 0, sizeof(stats->stats));
1562
1563         if (stats != NULL)
1564                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1565
1566         if (clear != 0)
1567                 port->n_pkts_dropped_by_ah = 0;
1568
1569         return 0;
1570 }
1571
1572 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1573         struct rte_pipeline_port_out_stats *stats, int clear)
1574 {
1575         struct rte_port_out *port;
1576         int retval;
1577
1578         if (p == NULL) {
1579                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1580                 return -EINVAL;
1581         }
1582
1583         if (port_id >= p->num_ports_out) {
1584                 RTE_LOG(ERR, PIPELINE,
1585                         "%s: port OUT ID %u is out of range\n", __func__, port_id);
1586                 return -EINVAL;
1587         }
1588
1589         port = &p->ports_out[port_id];
1590         if (port->ops.f_stats != NULL) {
1591                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1592                 if (retval != 0)
1593                         return retval;
1594         } else if (stats != NULL)
1595                 memset(&stats->stats, 0, sizeof(stats->stats));
1596
1597         if (stats != NULL)
1598                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1599
1600         if (clear != 0)
1601                 port->n_pkts_dropped_by_ah = 0;
1602
1603         return 0;
1604 }
1605
1606 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1607         struct rte_pipeline_table_stats *stats, int clear)
1608 {
1609         struct rte_table *table;
1610         int retval;
1611
1612         if (p == NULL) {
1613                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1614                         __func__);
1615                 return -EINVAL;
1616         }
1617
1618         if (table_id >= p->num_tables) {
1619                 RTE_LOG(ERR, PIPELINE,
1620                                 "%s: table %u is out of range\n", __func__, table_id);
1621                 return -EINVAL;
1622         }
1623
1624         table = &p->tables[table_id];
1625         if (table->ops.f_stats != NULL) {
1626                 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1627                 if (retval != 0)
1628                         return retval;
1629         } else if (stats != NULL)
1630                 memset(&stats->stats, 0, sizeof(stats->stats));
1631
1632         if (stats != NULL) {
1633                 stats->n_pkts_dropped_by_lkp_hit_ah =
1634                         table->n_pkts_dropped_by_lkp_hit_ah;
1635                 stats->n_pkts_dropped_by_lkp_miss_ah =
1636                         table->n_pkts_dropped_by_lkp_miss_ah;
1637                 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1638                 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1639         }
1640
1641         if (clear != 0) {
1642                 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1643                 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1644                 table->n_pkts_dropped_lkp_hit = 0;
1645                 table->n_pkts_dropped_lkp_miss = 0;
1646         }
1647
1648         return 0;
1649 }