lib/librte_pipeline/rte_pipeline.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_memory.h>
  39 #include <rte_cycles.h>
  40 #include <rte_prefetch.h>
  41 #include <rte_branch_prediction.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_malloc.h>
  44 #include <rte_string_fns.h>
  45
  46 #include "rte_pipeline.h"
  47
  48 #define RTE_TABLE_INVALID                                 UINT32_MAX
  49
  50 #ifdef RTE_PIPELINE_STATS_COLLECT
  51
  52 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)                       \
  53         ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
  54
  55 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)                     \
  56         ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
  57
  58 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)                               \
  59         ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
  60
  61 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)                      \
  62 ({                                                                      \
  63         uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP];    \
  64         mask ^= (p)->pkts_drop_mask;                                    \
  65         (counter) += __builtin_popcountll(mask);                        \
  66 })
  67
  68 #else
  69
  70 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
  71 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
  72 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
  73 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
  74
  75 #endif
  76
  77 struct rte_port_in {
  78         /* Input parameters */
  79         struct rte_port_in_ops ops;
  80         rte_pipeline_port_in_action_handler f_action;
  81         void *arg_ah;
  82         uint32_t burst_size;
  83
  84         /* The table to which this port is connected */
  85         uint32_t table_id;
  86
  87         /* Handle to low-level port */
  88         void *h_port;
  89
  90         /* List of enabled ports */
  91         struct rte_port_in *next;
  92
  93         /* Statistics */
  94         uint64_t n_pkts_dropped_by_ah;
  95 };
  96
  97 struct rte_port_out {
  98         /* Input parameters */
  99         struct rte_port_out_ops ops;
 100         rte_pipeline_port_out_action_handler f_action;
 101         void *arg_ah;
 102
 103         /* Handle to low-level port */
 104         void *h_port;
 105
 106         /* Statistics */
 107         uint64_t n_pkts_dropped_by_ah;
 108 };
 109
 110 struct rte_table {
 111         /* Input parameters */
 112         struct rte_table_ops ops;
 113         rte_pipeline_table_action_handler_hit f_action_hit;
 114         rte_pipeline_table_action_handler_miss f_action_miss;
 115         void *arg_ah;
 116         struct rte_pipeline_table_entry *default_entry;
 117         uint32_t entry_size;
 118
 119         uint32_t table_next_id;
 120         uint32_t table_next_id_valid;
 121
 122         /* Handle to the low-level table object */
 123         void *h_table;
 124
 125         /* Statistics */
 126         uint64_t n_pkts_dropped_by_lkp_hit_ah;
 127         uint64_t n_pkts_dropped_by_lkp_miss_ah;
 128         uint64_t n_pkts_dropped_lkp_hit;
 129         uint64_t n_pkts_dropped_lkp_miss;
 130 };
 131
 132 #define RTE_PIPELINE_MAX_NAME_SZ                           124
 133
 134 struct rte_pipeline {
 135         /* Input parameters */
 136         char name[RTE_PIPELINE_MAX_NAME_SZ];
 137         int socket_id;
 138         uint32_t offset_port_id;
 139
 140         /* Internal tables */
 141         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 142         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 143         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 144
 145         /* Occupancy of internal tables */
 146         uint32_t num_ports_in;
 147         uint32_t num_ports_out;
 148         uint32_t num_tables;
 149
 150         /* List of enabled ports */
 151         uint64_t enabled_port_in_mask;
 152         struct rte_port_in *port_in_next;
 153
 154         /* Pipeline run structures */
 155         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 156         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 157         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 158         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 159         uint64_t pkts_mask;
 160         uint64_t n_pkts_ah_drop;
 161         uint64_t pkts_drop_mask;
 162 } __rte_cache_aligned;
 163
 164 static inline uint32_t
 165 rte_mask_get_next(uint64_t mask, uint32_t pos)
 166 {
 167         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 168                         (mask >> ((pos + 1) & 0x3F));
 169         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 170 }
 171
 172 static inline uint32_t
 173 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 174 {
 175         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 176                         (mask << ((64 - pos) & 0x3F));
 177         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 178 }
 179
 180 static void
 181 rte_pipeline_table_free(struct rte_table *table);
 182
 183 static void
 184 rte_pipeline_port_in_free(struct rte_port_in *port);
 185
 186 static void
 187 rte_pipeline_port_out_free(struct rte_port_out *port);
 188
 189 /*
 190  * Pipeline
 191  *
 192  */
 193 static int
 194 rte_pipeline_check_params(struct rte_pipeline_params *params)
 195 {
 196         if (params == NULL) {
 197                 RTE_LOG(ERR, PIPELINE,
 198                         "%s: Incorrect value for parameter params\n", __func__);
 199                 return -EINVAL;
 200         }
 201
 202         /* name */
 203         if (params->name == NULL) {
 204                 RTE_LOG(ERR, PIPELINE,
 205                         "%s: Incorrect value for parameter name\n", __func__);
 206                 return -EINVAL;
 207         }
 208
 209         /* socket */
 210         if ((params->socket_id < 0) ||
 211             (params->socket_id >= RTE_MAX_NUMA_NODES)) {
 212                 RTE_LOG(ERR, PIPELINE,
 213                         "%s: Incorrect value for parameter socket_id\n",
 214                         __func__);
 215                 return -EINVAL;
 216         }
 217
 218         return 0;
 219 }
 220
 221 struct rte_pipeline *
 222 rte_pipeline_create(struct rte_pipeline_params *params)
 223 {
 224         struct rte_pipeline *p;
 225         int status;
 226
 227         /* Check input parameters */
 228         status = rte_pipeline_check_params(params);
 229         if (status != 0) {
 230                 RTE_LOG(ERR, PIPELINE,
 231                         "%s: Pipeline params check failed (%d)\n",
 232                         __func__, status);
 233                 return NULL;
 234         }
 235
 236         /* Allocate memory for the pipeline on requested socket */
 237         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 238                         RTE_CACHE_LINE_SIZE, params->socket_id);
 239
 240         if (p == NULL) {
 241                 RTE_LOG(ERR, PIPELINE,
 242                         "%s: Pipeline memory allocation failed\n", __func__);
 243                 return NULL;
 244         }
 245
 246         /* Save input parameters */
 247         snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
 248         p->socket_id = params->socket_id;
 249         p->offset_port_id = params->offset_port_id;
 250
 251         /* Initialize pipeline internal data structure */
 252         p->num_ports_in = 0;
 253         p->num_ports_out = 0;
 254         p->num_tables = 0;
 255         p->enabled_port_in_mask = 0;
 256         p->port_in_next = NULL;
 257         p->pkts_mask = 0;
 258         p->n_pkts_ah_drop = 0;
 259
 260         return p;
 261 }
 262
 263 int
 264 rte_pipeline_free(struct rte_pipeline *p)
 265 {
 266         uint32_t i;
 267
 268         /* Check input parameters */
 269         if (p == NULL) {
 270                 RTE_LOG(ERR, PIPELINE,
 271                         "%s: rte_pipeline parameter is NULL\n", __func__);
 272                 return -EINVAL;
 273         }
 274
 275         /* Free input ports */
 276         for (i = 0; i < p->num_ports_in; i++) {
 277                 struct rte_port_in *port = &p->ports_in[i];
 278
 279                 rte_pipeline_port_in_free(port);
 280         }
 281
 282         /* Free tables */
 283         for (i = 0; i < p->num_tables; i++) {
 284                 struct rte_table *table = &p->tables[i];
 285
 286                 rte_pipeline_table_free(table);
 287         }
 288
 289         /* Free output ports */
 290         for (i = 0; i < p->num_ports_out; i++) {
 291                 struct rte_port_out *port = &p->ports_out[i];
 292
 293                 rte_pipeline_port_out_free(port);
 294         }
 295
 296         /* Free pipeline memory */
 297         rte_free(p);
 298
 299         return 0;
 300 }
 301
 302 /*
 303  * Table
 304  *
 305  */
 306 static int
 307 rte_table_check_params(struct rte_pipeline *p,
 308                 struct rte_pipeline_table_params *params,
 309                 uint32_t *table_id)
 310 {
 311         if (p == NULL) {
 312                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 313                         __func__);
 314                 return -EINVAL;
 315         }
 316         if (params == NULL) {
 317                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 318                         __func__);
 319                 return -EINVAL;
 320         }
 321         if (table_id == NULL) {
 322                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 323                         __func__);
 324                 return -EINVAL;
 325         }
 326
 327         /* ops */
 328         if (params->ops == NULL) {
 329                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 330                         __func__);
 331                 return -EINVAL;
 332         }
 333
 334         if (params->ops->f_create == NULL) {
 335                 RTE_LOG(ERR, PIPELINE,
 336                         "%s: f_create function pointer is NULL\n", __func__);
 337                 return -EINVAL;
 338         }
 339
 340         if (params->ops->f_lookup == NULL) {
 341                 RTE_LOG(ERR, PIPELINE,
 342                         "%s: f_lookup function pointer is NULL\n", __func__);
 343                 return -EINVAL;
 344         }
 345
 346         /* De we have room for one more table? */
 347         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 348                 RTE_LOG(ERR, PIPELINE,
 349                         "%s: Incorrect value for num_tables parameter\n",
 350                         __func__);
 351                 return -EINVAL;
 352         }
 353
 354         return 0;
 355 }
 356
 357 int
 358 rte_pipeline_table_create(struct rte_pipeline *p,
 359                 struct rte_pipeline_table_params *params,
 360                 uint32_t *table_id)
 361 {
 362         struct rte_table *table;
 363         struct rte_pipeline_table_entry *default_entry;
 364         void *h_table;
 365         uint32_t entry_size, id;
 366         int status;
 367
 368         /* Check input arguments */
 369         status = rte_table_check_params(p, params, table_id);
 370         if (status != 0)
 371                 return status;
 372
 373         id = p->num_tables;
 374         table = &p->tables[id];
 375
 376         /* Allocate space for the default table entry */
 377         entry_size = sizeof(struct rte_pipeline_table_entry) +
 378                 params->action_data_size;
 379         default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
 380                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 381         if (default_entry == NULL) {
 382                 RTE_LOG(ERR, PIPELINE,
 383                         "%s: Failed to allocate default entry\n", __func__);
 384                 return -EINVAL;
 385         }
 386
 387         /* Create the table */
 388         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 389                 entry_size);
 390         if (h_table == NULL) {
 391                 rte_free(default_entry);
 392                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 393                 return -EINVAL;
 394         }
 395
 396         /* Commit current table to the pipeline */
 397         p->num_tables++;
 398         *table_id = id;
 399
 400         /* Save input parameters */
 401         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 402         table->f_action_hit = params->f_action_hit;
 403         table->f_action_miss = params->f_action_miss;
 404         table->arg_ah = params->arg_ah;
 405         table->entry_size = entry_size;
 406
 407         /* Clear the lookup miss actions (to be set later through API) */
 408         table->default_entry = default_entry;
 409         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 410
 411         /* Initialize table internal data structure */
 412         table->h_table = h_table;
 413         table->table_next_id = 0;
 414         table->table_next_id_valid = 0;
 415
 416         return 0;
 417 }
 418
 419 void
 420 rte_pipeline_table_free(struct rte_table *table)
 421 {
 422         if (table->ops.f_free != NULL)
 423                 table->ops.f_free(table->h_table);
 424
 425         rte_free(table->default_entry);
 426 }
 427
 428 int
 429 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 430         uint32_t table_id,
 431         struct rte_pipeline_table_entry *default_entry,
 432         struct rte_pipeline_table_entry **default_entry_ptr)
 433 {
 434         struct rte_table *table;
 435
 436         /* Check input arguments */
 437         if (p == NULL) {
 438                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 439                         __func__);
 440                 return -EINVAL;
 441         }
 442
 443         if (default_entry == NULL) {
 444                 RTE_LOG(ERR, PIPELINE,
 445                         "%s: default_entry parameter is NULL\n", __func__);
 446                 return -EINVAL;
 447         }
 448
 449         if (table_id >= p->num_tables) {
 450                 RTE_LOG(ERR, PIPELINE,
 451                         "%s: table_id %d out of range\n", __func__, table_id);
 452                 return -EINVAL;
 453         }
 454
 455         table = &p->tables[table_id];
 456
 457         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 458                 table->table_next_id_valid &&
 459                 (default_entry->table_id != table->table_next_id)) {
 460                 RTE_LOG(ERR, PIPELINE,
 461                         "%s: Tree-like topologies not allowed\n", __func__);
 462                 return -EINVAL;
 463         }
 464
 465         /* Set the lookup miss actions */
 466         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 467                 (table->table_next_id_valid == 0)) {
 468                 table->table_next_id = default_entry->table_id;
 469                 table->table_next_id_valid = 1;
 470         }
 471
 472         memcpy(table->default_entry, default_entry, table->entry_size);
 473
 474         *default_entry_ptr = table->default_entry;
 475         return 0;
 476 }
 477
 478 int
 479 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 480                 uint32_t table_id,
 481                 struct rte_pipeline_table_entry *entry)
 482 {
 483         struct rte_table *table;
 484
 485         /* Check input arguments */
 486         if (p == NULL) {
 487                 RTE_LOG(ERR, PIPELINE,
 488                         "%s: pipeline parameter is NULL\n", __func__);
 489                 return -EINVAL;
 490         }
 491
 492         if (table_id >= p->num_tables) {
 493                 RTE_LOG(ERR, PIPELINE,
 494                         "%s: table_id %d out of range\n", __func__, table_id);
 495                 return -EINVAL;
 496         }
 497
 498         table = &p->tables[table_id];
 499
 500         /* Save the current contents of the default entry */
 501         if (entry)
 502                 memcpy(entry, table->default_entry, table->entry_size);
 503
 504         /* Clear the lookup miss actions */
 505         memset(table->default_entry, 0, table->entry_size);
 506         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 507
 508         return 0;
 509 }
 510
 511 int
 512 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 513                 uint32_t table_id,
 514                 void *key,
 515                 struct rte_pipeline_table_entry *entry,
 516                 int *key_found,
 517                 struct rte_pipeline_table_entry **entry_ptr)
 518 {
 519         struct rte_table *table;
 520
 521         /* Check input arguments */
 522         if (p == NULL) {
 523                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 524                         __func__);
 525                 return -EINVAL;
 526         }
 527
 528         if (key == NULL) {
 529                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 530                 return -EINVAL;
 531         }
 532
 533         if (entry == NULL) {
 534                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 535                         __func__);
 536                 return -EINVAL;
 537         }
 538
 539         if (table_id >= p->num_tables) {
 540                 RTE_LOG(ERR, PIPELINE,
 541                         "%s: table_id %d out of range\n", __func__, table_id);
 542                 return -EINVAL;
 543         }
 544
 545         table = &p->tables[table_id];
 546
 547         if (table->ops.f_add == NULL) {
 548                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 549                         __func__);
 550                 return -EINVAL;
 551         }
 552
 553         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 554                 table->table_next_id_valid &&
 555                 (entry->table_id != table->table_next_id)) {
 556                 RTE_LOG(ERR, PIPELINE,
 557                         "%s: Tree-like topologies not allowed\n", __func__);
 558                 return -EINVAL;
 559         }
 560
 561         /* Add entry */
 562         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 563                 (table->table_next_id_valid == 0)) {
 564                 table->table_next_id = entry->table_id;
 565                 table->table_next_id_valid = 1;
 566         }
 567
 568         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 569                 key_found, (void **) entry_ptr);
 570 }
 571
 572 int
 573 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 574                 uint32_t table_id,
 575                 void *key,
 576                 int *key_found,
 577                 struct rte_pipeline_table_entry *entry)
 578 {
 579         struct rte_table *table;
 580
 581         /* Check input arguments */
 582         if (p == NULL) {
 583                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 584                         __func__);
 585                 return -EINVAL;
 586         }
 587
 588         if (key == NULL) {
 589                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 590                         __func__);
 591                 return -EINVAL;
 592         }
 593
 594         if (table_id >= p->num_tables) {
 595                 RTE_LOG(ERR, PIPELINE,
 596                         "%s: table_id %d out of range\n", __func__, table_id);
 597                 return -EINVAL;
 598         }
 599
 600         table = &p->tables[table_id];
 601
 602         if (table->ops.f_delete == NULL) {
 603                 RTE_LOG(ERR, PIPELINE,
 604                         "%s: f_delete function pointer NULL\n", __func__);
 605                 return -EINVAL;
 606         }
 607
 608         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 609 }
 610
 611 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
 612         uint32_t table_id,
 613         void **keys,
 614         struct rte_pipeline_table_entry **entries,
 615         uint32_t n_keys,
 616         int *key_found,
 617         struct rte_pipeline_table_entry **entries_ptr)
 618 {
 619         struct rte_table *table;
 620         uint32_t i;
 621
 622         /* Check input arguments */
 623         if (p == NULL) {
 624                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 625                         __func__);
 626                 return -EINVAL;
 627         }
 628
 629         if (keys == NULL) {
 630                 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
 631                 return -EINVAL;
 632         }
 633
 634         if (entries == NULL) {
 635                 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
 636                         __func__);
 637                 return -EINVAL;
 638         }
 639
 640         if (table_id >= p->num_tables) {
 641                 RTE_LOG(ERR, PIPELINE,
 642                         "%s: table_id %d out of range\n", __func__, table_id);
 643                 return -EINVAL;
 644         }
 645
 646         table = &p->tables[table_id];
 647
 648         if (table->ops.f_add_bulk == NULL) {
 649                 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
 650                         __func__);
 651                 return -EINVAL;
 652         }
 653
 654         for (i = 0; i < n_keys; i++) {
 655                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 656                         table->table_next_id_valid &&
 657                         (entries[i]->table_id != table->table_next_id)) {
 658                         RTE_LOG(ERR, PIPELINE,
 659                                 "%s: Tree-like topologies not allowed\n", __func__);
 660                         return -EINVAL;
 661                 }
 662         }
 663
 664         /* Add entry */
 665         for (i = 0; i < n_keys; i++) {
 666                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 667                         (table->table_next_id_valid == 0)) {
 668                         table->table_next_id = entries[i]->table_id;
 669                         table->table_next_id_valid = 1;
 670                 }
 671         }
 672
 673         return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
 674                 n_keys, key_found, (void **) entries_ptr);
 675 }
 676
 677 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
 678         uint32_t table_id,
 679         void **keys,
 680         uint32_t n_keys,
 681         int *key_found,
 682         struct rte_pipeline_table_entry **entries)
 683 {
 684         struct rte_table *table;
 685
 686         /* Check input arguments */
 687         if (p == NULL) {
 688                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 689                         __func__);
 690                 return -EINVAL;
 691         }
 692
 693         if (keys == NULL) {
 694                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 695                         __func__);
 696                 return -EINVAL;
 697         }
 698
 699         if (table_id >= p->num_tables) {
 700                 RTE_LOG(ERR, PIPELINE,
 701                         "%s: table_id %d out of range\n", __func__, table_id);
 702                 return -EINVAL;
 703         }
 704
 705         table = &p->tables[table_id];
 706
 707         if (table->ops.f_delete_bulk == NULL) {
 708                 RTE_LOG(ERR, PIPELINE,
 709                         "%s: f_delete function pointer NULL\n", __func__);
 710                 return -EINVAL;
 711         }
 712
 713         return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
 714                         (void **) entries);
 715 }
 716
 717 /*
 718  * Port
 719  *
 720  */
 721 static int
 722 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 723                 struct rte_pipeline_port_in_params *params,
 724                 uint32_t *port_id)
 725 {
 726         if (p == NULL) {
 727                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 728                         __func__);
 729                 return -EINVAL;
 730         }
 731         if (params == NULL) {
 732                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 733                 return -EINVAL;
 734         }
 735         if (port_id == NULL) {
 736                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 737                         __func__);
 738                 return -EINVAL;
 739         }
 740
 741         /* ops */
 742         if (params->ops == NULL) {
 743                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 744                         __func__);
 745                 return -EINVAL;
 746         }
 747
 748         if (params->ops->f_create == NULL) {
 749                 RTE_LOG(ERR, PIPELINE,
 750                         "%s: f_create function pointer NULL\n", __func__);
 751                 return -EINVAL;
 752         }
 753
 754         if (params->ops->f_rx == NULL) {
 755                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 756                         __func__);
 757                 return -EINVAL;
 758         }
 759
 760         /* burst_size */
 761         if ((params->burst_size == 0) ||
 762                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 763                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 764                         __func__);
 765                 return -EINVAL;
 766         }
 767
 768         /* Do we have room for one more port? */
 769         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 770                 RTE_LOG(ERR, PIPELINE,
 771                         "%s: invalid value for num_ports_in\n", __func__);
 772                 return -EINVAL;
 773         }
 774
 775         return 0;
 776 }
 777
 778 static int
 779 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 780                 struct rte_pipeline_port_out_params *params,
 781                 uint32_t *port_id)
 782 {
 783         if (p == NULL) {
 784                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 785                         __func__);
 786                 return -EINVAL;
 787         }
 788
 789         if (params == NULL) {
 790                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 791                 return -EINVAL;
 792         }
 793
 794         if (port_id == NULL) {
 795                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 796                         __func__);
 797                 return -EINVAL;
 798         }
 799
 800         /* ops */
 801         if (params->ops == NULL) {
 802                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 803                         __func__);
 804                 return -EINVAL;
 805         }
 806
 807         if (params->ops->f_create == NULL) {
 808                 RTE_LOG(ERR, PIPELINE,
 809                         "%s: f_create function pointer NULL\n", __func__);
 810                 return -EINVAL;
 811         }
 812
 813         if (params->ops->f_tx == NULL) {
 814                 RTE_LOG(ERR, PIPELINE,
 815                         "%s: f_tx function pointer NULL\n", __func__);
 816                 return -EINVAL;
 817         }
 818
 819         if (params->ops->f_tx_bulk == NULL) {
 820                 RTE_LOG(ERR, PIPELINE,
 821                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 822                 return -EINVAL;
 823         }
 824
 825         /* Do we have room for one more port? */
 826         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 827                 RTE_LOG(ERR, PIPELINE,
 828                         "%s: invalid value for num_ports_out\n", __func__);
 829                 return -EINVAL;
 830         }
 831
 832         return 0;
 833 }
 834
 835 int
 836 rte_pipeline_port_in_create(struct rte_pipeline *p,
 837                 struct rte_pipeline_port_in_params *params,
 838                 uint32_t *port_id)
 839 {
 840         struct rte_port_in *port;
 841         void *h_port;
 842         uint32_t id;
 843         int status;
 844
 845         /* Check input arguments */
 846         status = rte_pipeline_port_in_check_params(p, params, port_id);
 847         if (status != 0)
 848                 return status;
 849
 850         id = p->num_ports_in;
 851         port = &p->ports_in[id];
 852
 853         /* Create the port */
 854         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 855         if (h_port == NULL) {
 856                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 857                 return -EINVAL;
 858         }
 859
 860         /* Commit current table to the pipeline */
 861         p->num_ports_in++;
 862         *port_id = id;
 863
 864         /* Save input parameters */
 865         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 866         port->f_action = params->f_action;
 867         port->arg_ah = params->arg_ah;
 868         port->burst_size = params->burst_size;
 869
 870         /* Initialize port internal data structure */
 871         port->table_id = RTE_TABLE_INVALID;
 872         port->h_port = h_port;
 873         port->next = NULL;
 874
 875         return 0;
 876 }
 877
 878 void
 879 rte_pipeline_port_in_free(struct rte_port_in *port)
 880 {
 881         if (port->ops.f_free != NULL)
 882                 port->ops.f_free(port->h_port);
 883 }
 884
 885 int
 886 rte_pipeline_port_out_create(struct rte_pipeline *p,
 887                 struct rte_pipeline_port_out_params *params,
 888                 uint32_t *port_id)
 889 {
 890         struct rte_port_out *port;
 891         void *h_port;
 892         uint32_t id;
 893         int status;
 894
 895         /* Check input arguments */
 896         status = rte_pipeline_port_out_check_params(p, params, port_id);
 897         if (status != 0)
 898                 return status;
 899
 900         id = p->num_ports_out;
 901         port = &p->ports_out[id];
 902
 903         /* Create the port */
 904         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 905         if (h_port == NULL) {
 906                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 907                 return -EINVAL;
 908         }
 909
 910         /* Commit current table to the pipeline */
 911         p->num_ports_out++;
 912         *port_id = id;
 913
 914         /* Save input parameters */
 915         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 916         port->f_action = params->f_action;
 917         port->arg_ah = params->arg_ah;
 918
 919         /* Initialize port internal data structure */
 920         port->h_port = h_port;
 921
 922         return 0;
 923 }
 924
 925 void
 926 rte_pipeline_port_out_free(struct rte_port_out *port)
 927 {
 928         if (port->ops.f_free != NULL)
 929                 port->ops.f_free(port->h_port);
 930 }
 931
 932 int
 933 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 934                 uint32_t port_id,
 935                 uint32_t table_id)
 936 {
 937         struct rte_port_in *port;
 938
 939         /* Check input arguments */
 940         if (p == NULL) {
 941                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 942                         __func__);
 943                 return -EINVAL;
 944         }
 945
 946         if (port_id >= p->num_ports_in) {
 947                 RTE_LOG(ERR, PIPELINE,
 948                         "%s: port IN ID %u is out of range\n",
 949                         __func__, port_id);
 950                 return -EINVAL;
 951         }
 952
 953         if (table_id >= p->num_tables) {
 954                 RTE_LOG(ERR, PIPELINE,
 955                         "%s: Table ID %u is out of range\n",
 956                         __func__, table_id);
 957                 return -EINVAL;
 958         }
 959
 960         port = &p->ports_in[port_id];
 961         port->table_id = table_id;
 962
 963         return 0;
 964 }
 965
 966 int
 967 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 968 {
 969         struct rte_port_in *port, *port_prev, *port_next;
 970         uint64_t port_mask;
 971         uint32_t port_prev_id, port_next_id;
 972
 973         /* Check input arguments */
 974         if (p == NULL) {
 975                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 976                         __func__);
 977                 return -EINVAL;
 978         }
 979
 980         if (port_id >= p->num_ports_in) {
 981                 RTE_LOG(ERR, PIPELINE,
 982                         "%s: port IN ID %u is out of range\n",
 983                         __func__, port_id);
 984                 return -EINVAL;
 985         }
 986
 987         port = &p->ports_in[port_id];
 988
 989         /* Return if current input port is already enabled */
 990         port_mask = 1LLU << port_id;
 991         if (p->enabled_port_in_mask & port_mask)
 992                 return 0;
 993
 994         p->enabled_port_in_mask |= port_mask;
 995
 996         /* Add current input port to the pipeline chain of enabled ports */
 997         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 998         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 999
1000         port_prev = &p->ports_in[port_prev_id];
1001         port_next = &p->ports_in[port_next_id];
1002
1003         port_prev->next = port;
1004         port->next = port_next;
1005
1006         /* Check if list of enabled ports was previously empty */
1007         if (p->enabled_port_in_mask == port_mask)
1008                 p->port_in_next = port;
1009
1010         return 0;
1011 }
1012
1013 int
1014 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
1015 {
1016         struct rte_port_in *port, *port_prev, *port_next;
1017         uint64_t port_mask;
1018         uint32_t port_prev_id, port_next_id;
1019
1020         /* Check input arguments */
1021         if (p == NULL) {
1022                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1023                 __func__);
1024                 return -EINVAL;
1025         }
1026
1027         if (port_id >= p->num_ports_in) {
1028                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
1029                         __func__, port_id);
1030                 return -EINVAL;
1031         }
1032
1033         port = &p->ports_in[port_id];
1034
1035         /* Return if current input port is already disabled */
1036         port_mask = 1LLU << port_id;
1037         if ((p->enabled_port_in_mask & port_mask) == 0)
1038                 return 0;
1039
1040         p->enabled_port_in_mask &= ~port_mask;
1041
1042         /* Return if no other enabled ports */
1043         if (p->enabled_port_in_mask == 0) {
1044                 p->port_in_next = NULL;
1045
1046                 return 0;
1047         }
1048
1049         /* Add current input port to the pipeline chain of enabled ports */
1050         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1051         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1052
1053         port_prev = &p->ports_in[port_prev_id];
1054         port_next = &p->ports_in[port_next_id];
1055
1056         port_prev->next = port_next;
1057
1058         /* Check if the port which has just been disabled is next to serve */
1059         if (port == p->port_in_next)
1060                 p->port_in_next = port_next;
1061
1062         return 0;
1063 }
1064
1065 /*
1066  * Pipeline run-time
1067  *
1068  */
1069 int
1070 rte_pipeline_check(struct rte_pipeline *p)
1071 {
1072         uint32_t port_in_id;
1073
1074         /* Check input arguments */
1075         if (p == NULL) {
1076                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1077                         __func__);
1078                 return -EINVAL;
1079         }
1080
1081         /* Check that pipeline has at least one input port, one table and one
1082         output port */
1083         if (p->num_ports_in == 0) {
1084                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1085                         __func__);
1086                 return -EINVAL;
1087         }
1088         if (p->num_tables == 0) {
1089                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1090                         __func__);
1091                 return -EINVAL;
1092         }
1093         if (p->num_ports_out == 0) {
1094                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1095                         __func__);
1096                 return -EINVAL;
1097         }
1098
1099         /* Check that all input ports are connected */
1100         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1101                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1102
1103                 if (port_in->table_id == RTE_TABLE_INVALID) {
1104                         RTE_LOG(ERR, PIPELINE,
1105                                 "%s: Port IN ID %u is not connected\n",
1106                                 __func__, port_in_id);
1107                         return -EINVAL;
1108                 }
1109         }
1110
1111         return 0;
1112 }
1113
1114 static inline void
1115 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1116 {
1117         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1118         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1119         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1120         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1121
1122         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1123                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1124                 uint32_t i;
1125
1126                 for (i = 0; i < n_pkts; i++) {
1127                         uint64_t pkt_mask = 1LLU << i;
1128                         uint32_t pos = p->entries[i]->action;
1129
1130                         p->action_mask1[pos] |= pkt_mask;
1131                 }
1132         } else {
1133                 uint32_t i;
1134
1135                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1136                         uint64_t pkt_mask = 1LLU << i;
1137                         uint32_t pos;
1138
1139                         if ((pkt_mask & pkts_mask) == 0)
1140                                 continue;
1141
1142                         pos = p->entries[i]->action;
1143                         p->action_mask1[pos] |= pkt_mask;
1144                 }
1145         }
1146 }
1147
1148 static inline void
1149 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1150         uint64_t pkts_mask, uint32_t port_id)
1151 {
1152         struct rte_port_out *port_out = &p->ports_out[port_id];
1153
1154         p->pkts_mask = pkts_mask;
1155
1156         /* Output port user actions */
1157         if (port_out->f_action != NULL) {
1158                 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1159
1160                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1161                         port_out->n_pkts_dropped_by_ah);
1162         }
1163
1164         /* Output port TX */
1165         if (p->pkts_mask != 0)
1166                 port_out->ops.f_tx_bulk(port_out->h_port,
1167                         p->pkts,
1168                         p->pkts_mask);
1169 }
1170
1171 static inline void
1172 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1173 {
1174         p->pkts_mask = pkts_mask;
1175
1176         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1177                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1178                 uint32_t i;
1179
1180                 for (i = 0; i < n_pkts; i++) {
1181                         struct rte_mbuf *pkt = p->pkts[i];
1182                         uint32_t port_out_id = p->entries[i]->port_id;
1183                         struct rte_port_out *port_out =
1184                                 &p->ports_out[port_out_id];
1185
1186                         /* Output port user actions */
1187                         if (port_out->f_action == NULL) /* Output port TX */
1188                                 port_out->ops.f_tx(port_out->h_port, pkt);
1189                         else {
1190                                 uint64_t pkt_mask = 1LLU << i;
1191
1192                                 port_out->f_action(p,
1193                                         p->pkts,
1194                                         pkt_mask,
1195                                         port_out->arg_ah);
1196
1197                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1198                                         port_out->n_pkts_dropped_by_ah);
1199
1200                                 /* Output port TX */
1201                                 if (pkt_mask & p->pkts_mask)
1202                                         port_out->ops.f_tx(port_out->h_port,
1203                                                 pkt);
1204                         }
1205                 }
1206         } else {
1207                 uint32_t i;
1208
1209                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1210                         uint64_t pkt_mask = 1LLU << i;
1211                         struct rte_mbuf *pkt;
1212                         struct rte_port_out *port_out;
1213                         uint32_t port_out_id;
1214
1215                         if ((pkt_mask & pkts_mask) == 0)
1216                                 continue;
1217
1218                         pkt = p->pkts[i];
1219                         port_out_id = p->entries[i]->port_id;
1220                         port_out = &p->ports_out[port_out_id];
1221
1222                         /* Output port user actions */
1223                         if (port_out->f_action == NULL) /* Output port TX */
1224                                 port_out->ops.f_tx(port_out->h_port, pkt);
1225                         else {
1226                                 port_out->f_action(p,
1227                                         p->pkts,
1228                                         pkt_mask,
1229                                         port_out->arg_ah);
1230
1231                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1232                                         port_out->n_pkts_dropped_by_ah);
1233
1234                                 /* Output port TX */
1235                                 if (pkt_mask & p->pkts_mask)
1236                                         port_out->ops.f_tx(port_out->h_port,
1237                                                 pkt);
1238                         }
1239                 }
1240         }
1241 }
1242
1243 static inline void
1244 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1245         uint64_t pkts_mask)
1246 {
1247         p->pkts_mask = pkts_mask;
1248
1249         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1250                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1251                 uint32_t i;
1252
1253                 for (i = 0; i < n_pkts; i++) {
1254                         struct rte_mbuf *pkt = p->pkts[i];
1255                         uint32_t port_out_id =
1256                                 RTE_MBUF_METADATA_UINT32(pkt,
1257                                         p->offset_port_id);
1258                         struct rte_port_out *port_out = &p->ports_out[
1259                                 port_out_id];
1260
1261                         /* Output port user actions */
1262                         if (port_out->f_action == NULL) /* Output port TX */
1263                                 port_out->ops.f_tx(port_out->h_port, pkt);
1264                         else {
1265                                 uint64_t pkt_mask = 1LLU << i;
1266
1267                                 port_out->f_action(p,
1268                                         p->pkts,
1269                                         pkt_mask,
1270                                         port_out->arg_ah);
1271
1272                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1273                                         port_out->n_pkts_dropped_by_ah);
1274
1275                                 /* Output port TX */
1276                                 if (pkt_mask & p->pkts_mask)
1277                                         port_out->ops.f_tx(port_out->h_port,
1278                                                 pkt);
1279                         }
1280                 }
1281         } else {
1282                 uint32_t i;
1283
1284                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1285                         uint64_t pkt_mask = 1LLU << i;
1286                         struct rte_mbuf *pkt;
1287                         struct rte_port_out *port_out;
1288                         uint32_t port_out_id;
1289
1290                         if ((pkt_mask & pkts_mask) == 0)
1291                                 continue;
1292
1293                         pkt = p->pkts[i];
1294                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1295                                 p->offset_port_id);
1296                         port_out = &p->ports_out[port_out_id];
1297
1298                         /* Output port user actions */
1299                         if (port_out->f_action == NULL) /* Output port TX */
1300                                 port_out->ops.f_tx(port_out->h_port, pkt);
1301                         else {
1302                                 port_out->f_action(p,
1303                                         p->pkts,
1304                                         pkt_mask,
1305                                         port_out->arg_ah);
1306
1307                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1308                                         port_out->n_pkts_dropped_by_ah);
1309
1310                                 /* Output port TX */
1311                                 if (pkt_mask & p->pkts_mask)
1312                                         port_out->ops.f_tx(port_out->h_port,
1313                                                 pkt);
1314                         }
1315                 }
1316         }
1317 }
1318
1319 static inline void
1320 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1321 {
1322         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1323                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1324                 uint32_t i;
1325
1326                 for (i = 0; i < n_pkts; i++)
1327                         rte_pktmbuf_free(p->pkts[i]);
1328         } else {
1329                 uint32_t i;
1330
1331                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1332                         uint64_t pkt_mask = 1LLU << i;
1333
1334                         if ((pkt_mask & pkts_mask) == 0)
1335                                 continue;
1336
1337                         rte_pktmbuf_free(p->pkts[i]);
1338                 }
1339         }
1340 }
1341
1342 int
1343 rte_pipeline_run(struct rte_pipeline *p)
1344 {
1345         struct rte_port_in *port_in = p->port_in_next;
1346         uint32_t n_pkts, table_id;
1347
1348         if (port_in == NULL)
1349                 return 0;
1350
1351         /* Input port RX */
1352         n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1353                 port_in->burst_size);
1354         if (n_pkts == 0) {
1355                 p->port_in_next = port_in->next;
1356                 return 0;
1357         }
1358
1359         p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1360         p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1361         p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1362         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1363         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1364
1365         /* Input port user actions */
1366         if (port_in->f_action != NULL) {
1367                 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1368
1369                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1370                         port_in->n_pkts_dropped_by_ah);
1371         }
1372
1373         /* Table */
1374         for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1375                 struct rte_table *table;
1376                 uint64_t lookup_hit_mask, lookup_miss_mask;
1377
1378                 /* Lookup */
1379                 table = &p->tables[table_id];
1380                 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1381                         &lookup_hit_mask, (void **) p->entries);
1382                 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1383
1384                 /* Lookup miss */
1385                 if (lookup_miss_mask != 0) {
1386                         struct rte_pipeline_table_entry *default_entry =
1387                                 table->default_entry;
1388
1389                         p->pkts_mask = lookup_miss_mask;
1390
1391                         /* Table user actions */
1392                         if (table->f_action_miss != NULL) {
1393                                 table->f_action_miss(p,
1394                                         p->pkts,
1395                                         lookup_miss_mask,
1396                                         default_entry,
1397                                         table->arg_ah);
1398
1399                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1400                                         table->n_pkts_dropped_by_lkp_miss_ah);
1401                         }
1402
1403                         /* Table reserved actions */
1404                         if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1405                                 (p->pkts_mask != 0))
1406                                 rte_pipeline_action_handler_port_bulk(p,
1407                                         p->pkts_mask,
1408                                         default_entry->port_id);
1409                         else {
1410                                 uint32_t pos = default_entry->action;
1411
1412                                 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1413
1414                                 p->action_mask0[pos] |= p->pkts_mask;
1415
1416                                 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1417                                         table->n_pkts_dropped_lkp_miss);
1418                         }
1419                 }
1420
1421                 /* Lookup hit */
1422                 if (lookup_hit_mask != 0) {
1423                         p->pkts_mask = lookup_hit_mask;
1424
1425                         /* Table user actions */
1426                         if (table->f_action_hit != NULL) {
1427                                 table->f_action_hit(p,
1428                                         p->pkts,
1429                                         lookup_hit_mask,
1430                                         p->entries,
1431                                         table->arg_ah);
1432
1433                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1434                                         table->n_pkts_dropped_by_lkp_hit_ah);
1435                         }
1436
1437                         /* Table reserved actions */
1438                         RTE_PIPELINE_STATS_TABLE_DROP0(p);
1439                         rte_pipeline_compute_masks(p, p->pkts_mask);
1440                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1441                                 p->action_mask1[
1442                                         RTE_PIPELINE_ACTION_DROP];
1443                         p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1444                                 p->action_mask1[
1445                                         RTE_PIPELINE_ACTION_PORT];
1446                         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1447                                 p->action_mask1[
1448                                         RTE_PIPELINE_ACTION_PORT_META];
1449                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1450                                 p->action_mask1[
1451                                         RTE_PIPELINE_ACTION_TABLE];
1452
1453                         RTE_PIPELINE_STATS_TABLE_DROP1(p,
1454                                 table->n_pkts_dropped_lkp_hit);
1455                 }
1456
1457                 /* Prepare for next iteration */
1458                 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1459                 table_id = table->table_next_id;
1460                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1461         }
1462
1463         /* Table reserved action PORT */
1464         rte_pipeline_action_handler_port(p,
1465                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1466
1467         /* Table reserved action PORT META */
1468         rte_pipeline_action_handler_port_meta(p,
1469                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1470
1471         /* Table reserved action DROP */
1472         rte_pipeline_action_handler_drop(p,
1473                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1474
1475         /* Pick candidate for next port IN to serve */
1476         p->port_in_next = port_in->next;
1477
1478         return (int) n_pkts;
1479 }
1480
1481 int
1482 rte_pipeline_flush(struct rte_pipeline *p)
1483 {
1484         uint32_t port_id;
1485
1486         /* Check input arguments */
1487         if (p == NULL) {
1488                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1489                         __func__);
1490                 return -EINVAL;
1491         }
1492
1493         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1494                 struct rte_port_out *port = &p->ports_out[port_id];
1495
1496                 if (port->ops.f_flush != NULL)
1497                         port->ops.f_flush(port->h_port);
1498         }
1499
1500         return 0;
1501 }
1502
1503 int
1504 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1505         uint32_t port_id, struct rte_mbuf *pkt)
1506 {
1507         struct rte_port_out *port_out = &p->ports_out[port_id];
1508
1509         port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1510
1511         return 0;
1512 }
1513
1514 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1515         uint64_t pkts_mask)
1516 {
1517         pkts_mask &= p->pkts_mask;
1518         p->pkts_mask &= ~pkts_mask;
1519
1520         return 0;
1521 }
1522
1523 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1524         uint64_t pkts_mask)
1525 {
1526         pkts_mask &= p->pkts_mask;
1527         p->pkts_mask &= ~pkts_mask;
1528         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1529
1530         RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1531         return 0;
1532 }
1533
1534 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1535         struct rte_pipeline_port_in_stats *stats, int clear)
1536 {
1537         struct rte_port_in *port;
1538         int retval;
1539
1540         if (p == NULL) {
1541                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1542                         __func__);
1543                 return -EINVAL;
1544         }
1545
1546         if (port_id >= p->num_ports_in) {
1547                 RTE_LOG(ERR, PIPELINE,
1548                         "%s: port IN ID %u is out of range\n",
1549                         __func__, port_id);
1550                 return -EINVAL;
1551         }
1552
1553         port = &p->ports_in[port_id];
1554
1555         if (port->ops.f_stats != NULL) {
1556                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1557                 if (retval)
1558                         return retval;
1559         } else if (stats != NULL)
1560                 memset(&stats->stats, 0, sizeof(stats->stats));
1561
1562         if (stats != NULL)
1563                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1564
1565         if (clear != 0)
1566                 port->n_pkts_dropped_by_ah = 0;
1567
1568         return 0;
1569 }
1570
1571 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1572         struct rte_pipeline_port_out_stats *stats, int clear)
1573 {
1574         struct rte_port_out *port;
1575         int retval;
1576
1577         if (p == NULL) {
1578                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1579                 return -EINVAL;
1580         }
1581
1582         if (port_id >= p->num_ports_out) {
1583                 RTE_LOG(ERR, PIPELINE,
1584                         "%s: port OUT ID %u is out of range\n", __func__, port_id);
1585                 return -EINVAL;
1586         }
1587
1588         port = &p->ports_out[port_id];
1589         if (port->ops.f_stats != NULL) {
1590                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1591                 if (retval != 0)
1592                         return retval;
1593         } else if (stats != NULL)
1594                 memset(&stats->stats, 0, sizeof(stats->stats));
1595
1596         if (stats != NULL)
1597                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1598
1599         if (clear != 0)
1600                 port->n_pkts_dropped_by_ah = 0;
1601
1602         return 0;
1603 }
1604
1605 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1606         struct rte_pipeline_table_stats *stats, int clear)
1607 {
1608         struct rte_table *table;
1609         int retval;
1610
1611         if (p == NULL) {
1612                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1613                         __func__);
1614                 return -EINVAL;
1615         }
1616
1617         if (table_id >= p->num_tables) {
1618                 RTE_LOG(ERR, PIPELINE,
1619                                 "%s: table %u is out of range\n", __func__, table_id);
1620                 return -EINVAL;
1621         }
1622
1623         table = &p->tables[table_id];
1624         if (table->ops.f_stats != NULL) {
1625                 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1626                 if (retval != 0)
1627                         return retval;
1628         } else if (stats != NULL)
1629                 memset(&stats->stats, 0, sizeof(stats->stats));
1630
1631         if (stats != NULL) {
1632                 stats->n_pkts_dropped_by_lkp_hit_ah =
1633                         table->n_pkts_dropped_by_lkp_hit_ah;
1634                 stats->n_pkts_dropped_by_lkp_miss_ah =
1635                         table->n_pkts_dropped_by_lkp_miss_ah;
1636                 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1637                 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1638         }
1639
1640         if (clear != 0) {
1641                 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1642                 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1643                 table->n_pkts_dropped_lkp_hit = 0;
1644                 table->n_pkts_dropped_lkp_miss = 0;
1645         }
1646
1647         return 0;
1648 }