lib/pipeline/rte_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6 #include <stdio.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_mbuf.h>
  10 #include <rte_malloc.h>
  11 #include <rte_string_fns.h>
  12
  13 #include "rte_pipeline.h"
  14
  15 #define RTE_TABLE_INVALID                                 UINT32_MAX
  16
  17 #ifdef RTE_PIPELINE_STATS_COLLECT
  18
  19 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)                       \
  20         ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
  21
  22 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)                     \
  23         ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
  24
  25 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)                               \
  26         ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
  27
  28 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)                      \
  29 ({                                                                      \
  30         uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP];    \
  31         mask ^= (p)->pkts_drop_mask;                                    \
  32         (counter) += __builtin_popcountll(mask);                        \
  33 })
  34
  35 #else
  36
  37 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
  38 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
  39 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
  40 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
  41
  42 #endif
  43
  44 struct rte_port_in {
  45         /* Input parameters */
  46         struct rte_port_in_ops ops;
  47         rte_pipeline_port_in_action_handler f_action;
  48         void *arg_ah;
  49         uint32_t burst_size;
  50
  51         /* The table to which this port is connected */
  52         uint32_t table_id;
  53
  54         /* Handle to low-level port */
  55         void *h_port;
  56
  57         /* List of enabled ports */
  58         struct rte_port_in *next;
  59
  60         /* Statistics */
  61         uint64_t n_pkts_dropped_by_ah;
  62 };
  63
  64 struct rte_port_out {
  65         /* Input parameters */
  66         struct rte_port_out_ops ops;
  67         rte_pipeline_port_out_action_handler f_action;
  68         void *arg_ah;
  69
  70         /* Handle to low-level port */
  71         void *h_port;
  72
  73         /* Statistics */
  74         uint64_t n_pkts_dropped_by_ah;
  75 };
  76
  77 struct rte_table {
  78         /* Input parameters */
  79         struct rte_table_ops ops;
  80         rte_pipeline_table_action_handler_hit f_action_hit;
  81         rte_pipeline_table_action_handler_miss f_action_miss;
  82         void *arg_ah;
  83         struct rte_pipeline_table_entry *default_entry;
  84         uint32_t entry_size;
  85
  86         uint32_t table_next_id;
  87         uint32_t table_next_id_valid;
  88
  89         /* Handle to the low-level table object */
  90         void *h_table;
  91
  92         /* Statistics */
  93         uint64_t n_pkts_dropped_by_lkp_hit_ah;
  94         uint64_t n_pkts_dropped_by_lkp_miss_ah;
  95         uint64_t n_pkts_dropped_lkp_hit;
  96         uint64_t n_pkts_dropped_lkp_miss;
  97 };
  98
  99 #define RTE_PIPELINE_MAX_NAME_SZ                           124
 100
 101 struct rte_pipeline {
 102         /* Input parameters */
 103         char name[RTE_PIPELINE_MAX_NAME_SZ];
 104         int socket_id;
 105         uint32_t offset_port_id;
 106
 107         /* Internal tables */
 108         struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
 109         struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
 110         struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
 111
 112         /* Occupancy of internal tables */
 113         uint32_t num_ports_in;
 114         uint32_t num_ports_out;
 115         uint32_t num_tables;
 116
 117         /* List of enabled ports */
 118         uint64_t enabled_port_in_mask;
 119         struct rte_port_in *port_in_next;
 120
 121         /* Pipeline run structures */
 122         struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
 123         struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
 124         uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
 125         uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
 126         uint64_t pkts_mask;
 127         uint64_t n_pkts_ah_drop;
 128         uint64_t pkts_drop_mask;
 129 } __rte_cache_aligned;
 130
 131 static inline uint32_t
 132 rte_mask_get_next(uint64_t mask, uint32_t pos)
 133 {
 134         uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
 135                         (mask >> ((pos + 1) & 0x3F));
 136         return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
 137 }
 138
 139 static inline uint32_t
 140 rte_mask_get_prev(uint64_t mask, uint32_t pos)
 141 {
 142         uint64_t mask_rot = (mask >> (pos & 0x3F)) |
 143                         (mask << ((64 - pos) & 0x3F));
 144         return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
 145 }
 146
 147 static void
 148 rte_pipeline_table_free(struct rte_table *table);
 149
 150 static void
 151 rte_pipeline_port_in_free(struct rte_port_in *port);
 152
 153 static void
 154 rte_pipeline_port_out_free(struct rte_port_out *port);
 155
 156 /*
 157  * Pipeline
 158  *
 159  */
 160 static int
 161 rte_pipeline_check_params(struct rte_pipeline_params *params)
 162 {
 163         if (params == NULL) {
 164                 RTE_LOG(ERR, PIPELINE,
 165                         "%s: Incorrect value for parameter params\n", __func__);
 166                 return -EINVAL;
 167         }
 168
 169         /* name */
 170         if (params->name == NULL) {
 171                 RTE_LOG(ERR, PIPELINE,
 172                         "%s: Incorrect value for parameter name\n", __func__);
 173                 return -EINVAL;
 174         }
 175
 176         /* socket */
 177         if (params->socket_id < 0) {
 178                 RTE_LOG(ERR, PIPELINE,
 179                         "%s: Incorrect value for parameter socket_id\n",
 180                         __func__);
 181                 return -EINVAL;
 182         }
 183
 184         return 0;
 185 }
 186
 187 struct rte_pipeline *
 188 rte_pipeline_create(struct rte_pipeline_params *params)
 189 {
 190         struct rte_pipeline *p;
 191         int status;
 192
 193         /* Check input parameters */
 194         status = rte_pipeline_check_params(params);
 195         if (status != 0) {
 196                 RTE_LOG(ERR, PIPELINE,
 197                         "%s: Pipeline params check failed (%d)\n",
 198                         __func__, status);
 199                 return NULL;
 200         }
 201
 202         /* Allocate memory for the pipeline on requested socket */
 203         p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
 204                         RTE_CACHE_LINE_SIZE, params->socket_id);
 205
 206         if (p == NULL) {
 207                 RTE_LOG(ERR, PIPELINE,
 208                         "%s: Pipeline memory allocation failed\n", __func__);
 209                 return NULL;
 210         }
 211
 212         /* Save input parameters */
 213         strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ);
 214         p->socket_id = params->socket_id;
 215         p->offset_port_id = params->offset_port_id;
 216
 217         /* Initialize pipeline internal data structure */
 218         p->num_ports_in = 0;
 219         p->num_ports_out = 0;
 220         p->num_tables = 0;
 221         p->enabled_port_in_mask = 0;
 222         p->port_in_next = NULL;
 223         p->pkts_mask = 0;
 224         p->n_pkts_ah_drop = 0;
 225
 226         return p;
 227 }
 228
 229 int
 230 rte_pipeline_free(struct rte_pipeline *p)
 231 {
 232         uint32_t i;
 233
 234         /* Check input parameters */
 235         if (p == NULL) {
 236                 RTE_LOG(ERR, PIPELINE,
 237                         "%s: rte_pipeline parameter is NULL\n", __func__);
 238                 return -EINVAL;
 239         }
 240
 241         /* Free input ports */
 242         for (i = 0; i < p->num_ports_in; i++) {
 243                 struct rte_port_in *port = &p->ports_in[i];
 244
 245                 rte_pipeline_port_in_free(port);
 246         }
 247
 248         /* Free tables */
 249         for (i = 0; i < p->num_tables; i++) {
 250                 struct rte_table *table = &p->tables[i];
 251
 252                 rte_pipeline_table_free(table);
 253         }
 254
 255         /* Free output ports */
 256         for (i = 0; i < p->num_ports_out; i++) {
 257                 struct rte_port_out *port = &p->ports_out[i];
 258
 259                 rte_pipeline_port_out_free(port);
 260         }
 261
 262         /* Free pipeline memory */
 263         rte_free(p);
 264
 265         return 0;
 266 }
 267
 268 /*
 269  * Table
 270  *
 271  */
 272 static int
 273 rte_table_check_params(struct rte_pipeline *p,
 274                 struct rte_pipeline_table_params *params,
 275                 uint32_t *table_id)
 276 {
 277         if (p == NULL) {
 278                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 279                         __func__);
 280                 return -EINVAL;
 281         }
 282         if (params == NULL) {
 283                 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
 284                         __func__);
 285                 return -EINVAL;
 286         }
 287         if (table_id == NULL) {
 288                 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
 289                         __func__);
 290                 return -EINVAL;
 291         }
 292
 293         /* ops */
 294         if (params->ops == NULL) {
 295                 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
 296                         __func__);
 297                 return -EINVAL;
 298         }
 299
 300         if (params->ops->f_create == NULL) {
 301                 RTE_LOG(ERR, PIPELINE,
 302                         "%s: f_create function pointer is NULL\n", __func__);
 303                 return -EINVAL;
 304         }
 305
 306         if (params->ops->f_lookup == NULL) {
 307                 RTE_LOG(ERR, PIPELINE,
 308                         "%s: f_lookup function pointer is NULL\n", __func__);
 309                 return -EINVAL;
 310         }
 311
 312         /* De we have room for one more table? */
 313         if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
 314                 RTE_LOG(ERR, PIPELINE,
 315                         "%s: Incorrect value for num_tables parameter\n",
 316                         __func__);
 317                 return -EINVAL;
 318         }
 319
 320         return 0;
 321 }
 322
 323 int
 324 rte_pipeline_table_create(struct rte_pipeline *p,
 325                 struct rte_pipeline_table_params *params,
 326                 uint32_t *table_id)
 327 {
 328         struct rte_table *table;
 329         struct rte_pipeline_table_entry *default_entry;
 330         void *h_table;
 331         uint32_t entry_size, id;
 332         int status;
 333
 334         /* Check input arguments */
 335         status = rte_table_check_params(p, params, table_id);
 336         if (status != 0)
 337                 return status;
 338
 339         id = p->num_tables;
 340         table = &p->tables[id];
 341
 342         /* Allocate space for the default table entry */
 343         entry_size = sizeof(struct rte_pipeline_table_entry) +
 344                 params->action_data_size;
 345         default_entry = rte_zmalloc_socket(
 346                 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
 347         if (default_entry == NULL) {
 348                 RTE_LOG(ERR, PIPELINE,
 349                         "%s: Failed to allocate default entry\n", __func__);
 350                 return -EINVAL;
 351         }
 352
 353         /* Create the table */
 354         h_table = params->ops->f_create(params->arg_create, p->socket_id,
 355                 entry_size);
 356         if (h_table == NULL) {
 357                 rte_free(default_entry);
 358                 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
 359                 return -EINVAL;
 360         }
 361
 362         /* Commit current table to the pipeline */
 363         p->num_tables++;
 364         *table_id = id;
 365
 366         /* Save input parameters */
 367         memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
 368         table->f_action_hit = params->f_action_hit;
 369         table->f_action_miss = params->f_action_miss;
 370         table->arg_ah = params->arg_ah;
 371         table->entry_size = entry_size;
 372
 373         /* Clear the lookup miss actions (to be set later through API) */
 374         table->default_entry = default_entry;
 375         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 376
 377         /* Initialize table internal data structure */
 378         table->h_table = h_table;
 379         table->table_next_id = 0;
 380         table->table_next_id_valid = 0;
 381
 382         return 0;
 383 }
 384
 385 void
 386 rte_pipeline_table_free(struct rte_table *table)
 387 {
 388         if (table->ops.f_free != NULL)
 389                 table->ops.f_free(table->h_table);
 390
 391         rte_free(table->default_entry);
 392 }
 393
 394 int
 395 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
 396         uint32_t table_id,
 397         struct rte_pipeline_table_entry *default_entry,
 398         struct rte_pipeline_table_entry **default_entry_ptr)
 399 {
 400         struct rte_table *table;
 401
 402         /* Check input arguments */
 403         if (p == NULL) {
 404                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 405                         __func__);
 406                 return -EINVAL;
 407         }
 408
 409         if (default_entry == NULL) {
 410                 RTE_LOG(ERR, PIPELINE,
 411                         "%s: default_entry parameter is NULL\n", __func__);
 412                 return -EINVAL;
 413         }
 414
 415         if (table_id >= p->num_tables) {
 416                 RTE_LOG(ERR, PIPELINE,
 417                         "%s: table_id %d out of range\n", __func__, table_id);
 418                 return -EINVAL;
 419         }
 420
 421         table = &p->tables[table_id];
 422
 423         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 424                 table->table_next_id_valid &&
 425                 (default_entry->table_id != table->table_next_id)) {
 426                 RTE_LOG(ERR, PIPELINE,
 427                         "%s: Tree-like topologies not allowed\n", __func__);
 428                 return -EINVAL;
 429         }
 430
 431         /* Set the lookup miss actions */
 432         if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 433                 (table->table_next_id_valid == 0)) {
 434                 table->table_next_id = default_entry->table_id;
 435                 table->table_next_id_valid = 1;
 436         }
 437
 438         memcpy(table->default_entry, default_entry, table->entry_size);
 439
 440         *default_entry_ptr = table->default_entry;
 441         return 0;
 442 }
 443
 444 int
 445 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
 446                 uint32_t table_id,
 447                 struct rte_pipeline_table_entry *entry)
 448 {
 449         struct rte_table *table;
 450
 451         /* Check input arguments */
 452         if (p == NULL) {
 453                 RTE_LOG(ERR, PIPELINE,
 454                         "%s: pipeline parameter is NULL\n", __func__);
 455                 return -EINVAL;
 456         }
 457
 458         if (table_id >= p->num_tables) {
 459                 RTE_LOG(ERR, PIPELINE,
 460                         "%s: table_id %d out of range\n", __func__, table_id);
 461                 return -EINVAL;
 462         }
 463
 464         table = &p->tables[table_id];
 465
 466         /* Save the current contents of the default entry */
 467         if (entry)
 468                 memcpy(entry, table->default_entry, table->entry_size);
 469
 470         /* Clear the lookup miss actions */
 471         memset(table->default_entry, 0, table->entry_size);
 472         table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
 473
 474         return 0;
 475 }
 476
 477 int
 478 rte_pipeline_table_entry_add(struct rte_pipeline *p,
 479                 uint32_t table_id,
 480                 void *key,
 481                 struct rte_pipeline_table_entry *entry,
 482                 int *key_found,
 483                 struct rte_pipeline_table_entry **entry_ptr)
 484 {
 485         struct rte_table *table;
 486
 487         /* Check input arguments */
 488         if (p == NULL) {
 489                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 490                         __func__);
 491                 return -EINVAL;
 492         }
 493
 494         if (key == NULL) {
 495                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
 496                 return -EINVAL;
 497         }
 498
 499         if (entry == NULL) {
 500                 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
 501                         __func__);
 502                 return -EINVAL;
 503         }
 504
 505         if (table_id >= p->num_tables) {
 506                 RTE_LOG(ERR, PIPELINE,
 507                         "%s: table_id %d out of range\n", __func__, table_id);
 508                 return -EINVAL;
 509         }
 510
 511         table = &p->tables[table_id];
 512
 513         if (table->ops.f_add == NULL) {
 514                 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
 515                         __func__);
 516                 return -EINVAL;
 517         }
 518
 519         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 520                 table->table_next_id_valid &&
 521                 (entry->table_id != table->table_next_id)) {
 522                 RTE_LOG(ERR, PIPELINE,
 523                         "%s: Tree-like topologies not allowed\n", __func__);
 524                 return -EINVAL;
 525         }
 526
 527         /* Add entry */
 528         if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
 529                 (table->table_next_id_valid == 0)) {
 530                 table->table_next_id = entry->table_id;
 531                 table->table_next_id_valid = 1;
 532         }
 533
 534         return (table->ops.f_add)(table->h_table, key, (void *) entry,
 535                 key_found, (void **) entry_ptr);
 536 }
 537
 538 int
 539 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
 540                 uint32_t table_id,
 541                 void *key,
 542                 int *key_found,
 543                 struct rte_pipeline_table_entry *entry)
 544 {
 545         struct rte_table *table;
 546
 547         /* Check input arguments */
 548         if (p == NULL) {
 549                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 550                         __func__);
 551                 return -EINVAL;
 552         }
 553
 554         if (key == NULL) {
 555                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 556                         __func__);
 557                 return -EINVAL;
 558         }
 559
 560         if (table_id >= p->num_tables) {
 561                 RTE_LOG(ERR, PIPELINE,
 562                         "%s: table_id %d out of range\n", __func__, table_id);
 563                 return -EINVAL;
 564         }
 565
 566         table = &p->tables[table_id];
 567
 568         if (table->ops.f_delete == NULL) {
 569                 RTE_LOG(ERR, PIPELINE,
 570                         "%s: f_delete function pointer NULL\n", __func__);
 571                 return -EINVAL;
 572         }
 573
 574         return (table->ops.f_delete)(table->h_table, key, key_found, entry);
 575 }
 576
 577 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
 578         uint32_t table_id,
 579         void **keys,
 580         struct rte_pipeline_table_entry **entries,
 581         uint32_t n_keys,
 582         int *key_found,
 583         struct rte_pipeline_table_entry **entries_ptr)
 584 {
 585         struct rte_table *table;
 586         uint32_t i;
 587
 588         /* Check input arguments */
 589         if (p == NULL) {
 590                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
 591                         __func__);
 592                 return -EINVAL;
 593         }
 594
 595         if (keys == NULL) {
 596                 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
 597                 return -EINVAL;
 598         }
 599
 600         if (entries == NULL) {
 601                 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
 602                         __func__);
 603                 return -EINVAL;
 604         }
 605
 606         if (table_id >= p->num_tables) {
 607                 RTE_LOG(ERR, PIPELINE,
 608                         "%s: table_id %d out of range\n", __func__, table_id);
 609                 return -EINVAL;
 610         }
 611
 612         table = &p->tables[table_id];
 613
 614         if (table->ops.f_add_bulk == NULL) {
 615                 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
 616                         __func__);
 617                 return -EINVAL;
 618         }
 619
 620         for (i = 0; i < n_keys; i++) {
 621                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 622                         table->table_next_id_valid &&
 623                         (entries[i]->table_id != table->table_next_id)) {
 624                         RTE_LOG(ERR, PIPELINE,
 625                                 "%s: Tree-like topologies not allowed\n", __func__);
 626                         return -EINVAL;
 627                 }
 628         }
 629
 630         /* Add entry */
 631         for (i = 0; i < n_keys; i++) {
 632                 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
 633                         (table->table_next_id_valid == 0)) {
 634                         table->table_next_id = entries[i]->table_id;
 635                         table->table_next_id_valid = 1;
 636                 }
 637         }
 638
 639         return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
 640                 n_keys, key_found, (void **) entries_ptr);
 641 }
 642
 643 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
 644         uint32_t table_id,
 645         void **keys,
 646         uint32_t n_keys,
 647         int *key_found,
 648         struct rte_pipeline_table_entry **entries)
 649 {
 650         struct rte_table *table;
 651
 652         /* Check input arguments */
 653         if (p == NULL) {
 654                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 655                         __func__);
 656                 return -EINVAL;
 657         }
 658
 659         if (keys == NULL) {
 660                 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
 661                         __func__);
 662                 return -EINVAL;
 663         }
 664
 665         if (table_id >= p->num_tables) {
 666                 RTE_LOG(ERR, PIPELINE,
 667                         "%s: table_id %d out of range\n", __func__, table_id);
 668                 return -EINVAL;
 669         }
 670
 671         table = &p->tables[table_id];
 672
 673         if (table->ops.f_delete_bulk == NULL) {
 674                 RTE_LOG(ERR, PIPELINE,
 675                         "%s: f_delete function pointer NULL\n", __func__);
 676                 return -EINVAL;
 677         }
 678
 679         return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
 680                         (void **) entries);
 681 }
 682
 683 /*
 684  * Port
 685  *
 686  */
 687 static int
 688 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
 689                 struct rte_pipeline_port_in_params *params,
 690                 uint32_t *port_id)
 691 {
 692         if (p == NULL) {
 693                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 694                         __func__);
 695                 return -EINVAL;
 696         }
 697         if (params == NULL) {
 698                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 699                 return -EINVAL;
 700         }
 701         if (port_id == NULL) {
 702                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 703                         __func__);
 704                 return -EINVAL;
 705         }
 706
 707         /* ops */
 708         if (params->ops == NULL) {
 709                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 710                         __func__);
 711                 return -EINVAL;
 712         }
 713
 714         if (params->ops->f_create == NULL) {
 715                 RTE_LOG(ERR, PIPELINE,
 716                         "%s: f_create function pointer NULL\n", __func__);
 717                 return -EINVAL;
 718         }
 719
 720         if (params->ops->f_rx == NULL) {
 721                 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
 722                         __func__);
 723                 return -EINVAL;
 724         }
 725
 726         /* burst_size */
 727         if ((params->burst_size == 0) ||
 728                 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
 729                 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
 730                         __func__);
 731                 return -EINVAL;
 732         }
 733
 734         /* Do we have room for one more port? */
 735         if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
 736                 RTE_LOG(ERR, PIPELINE,
 737                         "%s: invalid value for num_ports_in\n", __func__);
 738                 return -EINVAL;
 739         }
 740
 741         return 0;
 742 }
 743
 744 static int
 745 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
 746                 struct rte_pipeline_port_out_params *params,
 747                 uint32_t *port_id)
 748 {
 749         if (p == NULL) {
 750                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 751                         __func__);
 752                 return -EINVAL;
 753         }
 754
 755         if (params == NULL) {
 756                 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
 757                 return -EINVAL;
 758         }
 759
 760         if (port_id == NULL) {
 761                 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
 762                         __func__);
 763                 return -EINVAL;
 764         }
 765
 766         /* ops */
 767         if (params->ops == NULL) {
 768                 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
 769                         __func__);
 770                 return -EINVAL;
 771         }
 772
 773         if (params->ops->f_create == NULL) {
 774                 RTE_LOG(ERR, PIPELINE,
 775                         "%s: f_create function pointer NULL\n", __func__);
 776                 return -EINVAL;
 777         }
 778
 779         if (params->ops->f_tx == NULL) {
 780                 RTE_LOG(ERR, PIPELINE,
 781                         "%s: f_tx function pointer NULL\n", __func__);
 782                 return -EINVAL;
 783         }
 784
 785         if (params->ops->f_tx_bulk == NULL) {
 786                 RTE_LOG(ERR, PIPELINE,
 787                         "%s: f_tx_bulk function pointer NULL\n", __func__);
 788                 return -EINVAL;
 789         }
 790
 791         /* Do we have room for one more port? */
 792         if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
 793                 RTE_LOG(ERR, PIPELINE,
 794                         "%s: invalid value for num_ports_out\n", __func__);
 795                 return -EINVAL;
 796         }
 797
 798         return 0;
 799 }
 800
 801 int
 802 rte_pipeline_port_in_create(struct rte_pipeline *p,
 803                 struct rte_pipeline_port_in_params *params,
 804                 uint32_t *port_id)
 805 {
 806         struct rte_port_in *port;
 807         void *h_port;
 808         uint32_t id;
 809         int status;
 810
 811         /* Check input arguments */
 812         status = rte_pipeline_port_in_check_params(p, params, port_id);
 813         if (status != 0)
 814                 return status;
 815
 816         id = p->num_ports_in;
 817         port = &p->ports_in[id];
 818
 819         /* Create the port */
 820         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 821         if (h_port == NULL) {
 822                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 823                 return -EINVAL;
 824         }
 825
 826         /* Commit current table to the pipeline */
 827         p->num_ports_in++;
 828         *port_id = id;
 829
 830         /* Save input parameters */
 831         memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
 832         port->f_action = params->f_action;
 833         port->arg_ah = params->arg_ah;
 834         port->burst_size = params->burst_size;
 835
 836         /* Initialize port internal data structure */
 837         port->table_id = RTE_TABLE_INVALID;
 838         port->h_port = h_port;
 839         port->next = NULL;
 840
 841         return 0;
 842 }
 843
 844 void
 845 rte_pipeline_port_in_free(struct rte_port_in *port)
 846 {
 847         if (port->ops.f_free != NULL)
 848                 port->ops.f_free(port->h_port);
 849 }
 850
 851 int
 852 rte_pipeline_port_out_create(struct rte_pipeline *p,
 853                 struct rte_pipeline_port_out_params *params,
 854                 uint32_t *port_id)
 855 {
 856         struct rte_port_out *port;
 857         void *h_port;
 858         uint32_t id;
 859         int status;
 860
 861         /* Check input arguments */
 862         status = rte_pipeline_port_out_check_params(p, params, port_id);
 863         if (status != 0)
 864                 return status;
 865
 866         id = p->num_ports_out;
 867         port = &p->ports_out[id];
 868
 869         /* Create the port */
 870         h_port = params->ops->f_create(params->arg_create, p->socket_id);
 871         if (h_port == NULL) {
 872                 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
 873                 return -EINVAL;
 874         }
 875
 876         /* Commit current table to the pipeline */
 877         p->num_ports_out++;
 878         *port_id = id;
 879
 880         /* Save input parameters */
 881         memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
 882         port->f_action = params->f_action;
 883         port->arg_ah = params->arg_ah;
 884
 885         /* Initialize port internal data structure */
 886         port->h_port = h_port;
 887
 888         return 0;
 889 }
 890
 891 void
 892 rte_pipeline_port_out_free(struct rte_port_out *port)
 893 {
 894         if (port->ops.f_free != NULL)
 895                 port->ops.f_free(port->h_port);
 896 }
 897
 898 int
 899 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
 900                 uint32_t port_id,
 901                 uint32_t table_id)
 902 {
 903         struct rte_port_in *port;
 904
 905         /* Check input arguments */
 906         if (p == NULL) {
 907                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 908                         __func__);
 909                 return -EINVAL;
 910         }
 911
 912         if (port_id >= p->num_ports_in) {
 913                 RTE_LOG(ERR, PIPELINE,
 914                         "%s: port IN ID %u is out of range\n",
 915                         __func__, port_id);
 916                 return -EINVAL;
 917         }
 918
 919         if (table_id >= p->num_tables) {
 920                 RTE_LOG(ERR, PIPELINE,
 921                         "%s: Table ID %u is out of range\n",
 922                         __func__, table_id);
 923                 return -EINVAL;
 924         }
 925
 926         port = &p->ports_in[port_id];
 927         port->table_id = table_id;
 928
 929         return 0;
 930 }
 931
 932 int
 933 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
 934 {
 935         struct rte_port_in *port, *port_prev, *port_next;
 936         uint64_t port_mask;
 937         uint32_t port_prev_id, port_next_id;
 938
 939         /* Check input arguments */
 940         if (p == NULL) {
 941                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 942                         __func__);
 943                 return -EINVAL;
 944         }
 945
 946         if (port_id >= p->num_ports_in) {
 947                 RTE_LOG(ERR, PIPELINE,
 948                         "%s: port IN ID %u is out of range\n",
 949                         __func__, port_id);
 950                 return -EINVAL;
 951         }
 952
 953         port = &p->ports_in[port_id];
 954
 955         /* Return if current input port is already enabled */
 956         port_mask = 1LLU << port_id;
 957         if (p->enabled_port_in_mask & port_mask)
 958                 return 0;
 959
 960         p->enabled_port_in_mask |= port_mask;
 961
 962         /* Add current input port to the pipeline chain of enabled ports */
 963         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
 964         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
 965
 966         port_prev = &p->ports_in[port_prev_id];
 967         port_next = &p->ports_in[port_next_id];
 968
 969         port_prev->next = port;
 970         port->next = port_next;
 971
 972         /* Check if list of enabled ports was previously empty */
 973         if (p->enabled_port_in_mask == port_mask)
 974                 p->port_in_next = port;
 975
 976         return 0;
 977 }
 978
 979 int
 980 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
 981 {
 982         struct rte_port_in *port, *port_prev, *port_next;
 983         uint64_t port_mask;
 984         uint32_t port_prev_id, port_next_id;
 985
 986         /* Check input arguments */
 987         if (p == NULL) {
 988                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
 989                 __func__);
 990                 return -EINVAL;
 991         }
 992
 993         if (port_id >= p->num_ports_in) {
 994                 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
 995                         __func__, port_id);
 996                 return -EINVAL;
 997         }
 998
 999         port = &p->ports_in[port_id];
1000
1001         /* Return if current input port is already disabled */
1002         port_mask = 1LLU << port_id;
1003         if ((p->enabled_port_in_mask & port_mask) == 0)
1004                 return 0;
1005
1006         p->enabled_port_in_mask &= ~port_mask;
1007
1008         /* Return if no other enabled ports */
1009         if (p->enabled_port_in_mask == 0) {
1010                 p->port_in_next = NULL;
1011
1012                 return 0;
1013         }
1014
1015         /* Add current input port to the pipeline chain of enabled ports */
1016         port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1017         port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1018
1019         port_prev = &p->ports_in[port_prev_id];
1020         port_next = &p->ports_in[port_next_id];
1021
1022         port_prev->next = port_next;
1023
1024         /* Check if the port which has just been disabled is next to serve */
1025         if (port == p->port_in_next)
1026                 p->port_in_next = port_next;
1027
1028         return 0;
1029 }
1030
1031 /*
1032  * Pipeline run-time
1033  *
1034  */
1035 int
1036 rte_pipeline_check(struct rte_pipeline *p)
1037 {
1038         uint32_t port_in_id;
1039
1040         /* Check input arguments */
1041         if (p == NULL) {
1042                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1043                         __func__);
1044                 return -EINVAL;
1045         }
1046
1047         /* Check that pipeline has at least one input port, one table and one
1048         output port */
1049         if (p->num_ports_in == 0) {
1050                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1051                         __func__);
1052                 return -EINVAL;
1053         }
1054         if (p->num_tables == 0) {
1055                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1056                         __func__);
1057                 return -EINVAL;
1058         }
1059         if (p->num_ports_out == 0) {
1060                 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1061                         __func__);
1062                 return -EINVAL;
1063         }
1064
1065         /* Check that all input ports are connected */
1066         for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1067                 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1068
1069                 if (port_in->table_id == RTE_TABLE_INVALID) {
1070                         RTE_LOG(ERR, PIPELINE,
1071                                 "%s: Port IN ID %u is not connected\n",
1072                                 __func__, port_in_id);
1073                         return -EINVAL;
1074                 }
1075         }
1076
1077         return 0;
1078 }
1079
1080 static inline void
1081 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1082 {
1083         p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1084         p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1085         p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1086         p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1087
1088         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1089                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1090                 uint32_t i;
1091
1092                 for (i = 0; i < n_pkts; i++) {
1093                         uint64_t pkt_mask = 1LLU << i;
1094                         uint32_t pos = p->entries[i]->action;
1095
1096                         p->action_mask1[pos] |= pkt_mask;
1097                 }
1098         } else {
1099                 uint32_t i;
1100
1101                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1102                         uint64_t pkt_mask = 1LLU << i;
1103                         uint32_t pos;
1104
1105                         if ((pkt_mask & pkts_mask) == 0)
1106                                 continue;
1107
1108                         pos = p->entries[i]->action;
1109                         p->action_mask1[pos] |= pkt_mask;
1110                 }
1111         }
1112 }
1113
1114 static inline void
1115 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1116         uint64_t pkts_mask, uint32_t port_id)
1117 {
1118         struct rte_port_out *port_out = &p->ports_out[port_id];
1119
1120         p->pkts_mask = pkts_mask;
1121
1122         /* Output port user actions */
1123         if (port_out->f_action != NULL) {
1124                 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1125
1126                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1127                         port_out->n_pkts_dropped_by_ah);
1128         }
1129
1130         /* Output port TX */
1131         if (p->pkts_mask != 0)
1132                 port_out->ops.f_tx_bulk(port_out->h_port,
1133                         p->pkts,
1134                         p->pkts_mask);
1135 }
1136
1137 static inline void
1138 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1139 {
1140         p->pkts_mask = pkts_mask;
1141
1142         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1143                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1144                 uint32_t i;
1145
1146                 for (i = 0; i < n_pkts; i++) {
1147                         struct rte_mbuf *pkt = p->pkts[i];
1148                         uint32_t port_out_id = p->entries[i]->port_id;
1149                         struct rte_port_out *port_out =
1150                                 &p->ports_out[port_out_id];
1151
1152                         /* Output port user actions */
1153                         if (port_out->f_action == NULL) /* Output port TX */
1154                                 port_out->ops.f_tx(port_out->h_port, pkt);
1155                         else {
1156                                 uint64_t pkt_mask = 1LLU << i;
1157
1158                                 port_out->f_action(p,
1159                                         p->pkts,
1160                                         pkt_mask,
1161                                         port_out->arg_ah);
1162
1163                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1164                                         port_out->n_pkts_dropped_by_ah);
1165
1166                                 /* Output port TX */
1167                                 if (pkt_mask & p->pkts_mask)
1168                                         port_out->ops.f_tx(port_out->h_port,
1169                                                 pkt);
1170                         }
1171                 }
1172         } else {
1173                 uint32_t i;
1174
1175                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1176                         uint64_t pkt_mask = 1LLU << i;
1177                         struct rte_mbuf *pkt;
1178                         struct rte_port_out *port_out;
1179                         uint32_t port_out_id;
1180
1181                         if ((pkt_mask & pkts_mask) == 0)
1182                                 continue;
1183
1184                         pkt = p->pkts[i];
1185                         port_out_id = p->entries[i]->port_id;
1186                         port_out = &p->ports_out[port_out_id];
1187
1188                         /* Output port user actions */
1189                         if (port_out->f_action == NULL) /* Output port TX */
1190                                 port_out->ops.f_tx(port_out->h_port, pkt);
1191                         else {
1192                                 port_out->f_action(p,
1193                                         p->pkts,
1194                                         pkt_mask,
1195                                         port_out->arg_ah);
1196
1197                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1198                                         port_out->n_pkts_dropped_by_ah);
1199
1200                                 /* Output port TX */
1201                                 if (pkt_mask & p->pkts_mask)
1202                                         port_out->ops.f_tx(port_out->h_port,
1203                                                 pkt);
1204                         }
1205                 }
1206         }
1207 }
1208
1209 static inline void
1210 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1211         uint64_t pkts_mask)
1212 {
1213         p->pkts_mask = pkts_mask;
1214
1215         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1216                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1217                 uint32_t i;
1218
1219                 for (i = 0; i < n_pkts; i++) {
1220                         struct rte_mbuf *pkt = p->pkts[i];
1221                         uint32_t port_out_id =
1222                                 RTE_MBUF_METADATA_UINT32(pkt,
1223                                         p->offset_port_id);
1224                         struct rte_port_out *port_out = &p->ports_out[
1225                                 port_out_id];
1226
1227                         /* Output port user actions */
1228                         if (port_out->f_action == NULL) /* Output port TX */
1229                                 port_out->ops.f_tx(port_out->h_port, pkt);
1230                         else {
1231                                 uint64_t pkt_mask = 1LLU << i;
1232
1233                                 port_out->f_action(p,
1234                                         p->pkts,
1235                                         pkt_mask,
1236                                         port_out->arg_ah);
1237
1238                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1239                                         port_out->n_pkts_dropped_by_ah);
1240
1241                                 /* Output port TX */
1242                                 if (pkt_mask & p->pkts_mask)
1243                                         port_out->ops.f_tx(port_out->h_port,
1244                                                 pkt);
1245                         }
1246                 }
1247         } else {
1248                 uint32_t i;
1249
1250                 for (i = 0;  i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1251                         uint64_t pkt_mask = 1LLU << i;
1252                         struct rte_mbuf *pkt;
1253                         struct rte_port_out *port_out;
1254                         uint32_t port_out_id;
1255
1256                         if ((pkt_mask & pkts_mask) == 0)
1257                                 continue;
1258
1259                         pkt = p->pkts[i];
1260                         port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1261                                 p->offset_port_id);
1262                         port_out = &p->ports_out[port_out_id];
1263
1264                         /* Output port user actions */
1265                         if (port_out->f_action == NULL) /* Output port TX */
1266                                 port_out->ops.f_tx(port_out->h_port, pkt);
1267                         else {
1268                                 port_out->f_action(p,
1269                                         p->pkts,
1270                                         pkt_mask,
1271                                         port_out->arg_ah);
1272
1273                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1274                                         port_out->n_pkts_dropped_by_ah);
1275
1276                                 /* Output port TX */
1277                                 if (pkt_mask & p->pkts_mask)
1278                                         port_out->ops.f_tx(port_out->h_port,
1279                                                 pkt);
1280                         }
1281                 }
1282         }
1283 }
1284
1285 static inline void
1286 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1287 {
1288         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1289                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1290                 uint32_t i;
1291
1292                 for (i = 0; i < n_pkts; i++)
1293                         rte_pktmbuf_free(p->pkts[i]);
1294         } else {
1295                 uint32_t i;
1296
1297                 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1298                         uint64_t pkt_mask = 1LLU << i;
1299
1300                         if ((pkt_mask & pkts_mask) == 0)
1301                                 continue;
1302
1303                         rte_pktmbuf_free(p->pkts[i]);
1304                 }
1305         }
1306 }
1307
1308 int
1309 rte_pipeline_run(struct rte_pipeline *p)
1310 {
1311         struct rte_port_in *port_in = p->port_in_next;
1312         uint32_t n_pkts, table_id;
1313
1314         if (port_in == NULL)
1315                 return 0;
1316
1317         /* Input port RX */
1318         n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1319                 port_in->burst_size);
1320         if (n_pkts == 0) {
1321                 p->port_in_next = port_in->next;
1322                 return 0;
1323         }
1324
1325         p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1326         p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1327         p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1328         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1329         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1330
1331         /* Input port user actions */
1332         if (port_in->f_action != NULL) {
1333                 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1334
1335                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1336                         port_in->n_pkts_dropped_by_ah);
1337         }
1338
1339         /* Table */
1340         for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1341                 struct rte_table *table;
1342                 uint64_t lookup_hit_mask, lookup_miss_mask;
1343
1344                 /* Lookup */
1345                 table = &p->tables[table_id];
1346                 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1347                         &lookup_hit_mask, (void **) p->entries);
1348                 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1349
1350                 /* Lookup miss */
1351                 if (lookup_miss_mask != 0) {
1352                         struct rte_pipeline_table_entry *default_entry =
1353                                 table->default_entry;
1354
1355                         p->pkts_mask = lookup_miss_mask;
1356
1357                         /* Table user actions */
1358                         if (table->f_action_miss != NULL) {
1359                                 table->f_action_miss(p,
1360                                         p->pkts,
1361                                         lookup_miss_mask,
1362                                         default_entry,
1363                                         table->arg_ah);
1364
1365                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1366                                         table->n_pkts_dropped_by_lkp_miss_ah);
1367                         }
1368
1369                         /* Table reserved actions */
1370                         if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1371                                 (p->pkts_mask != 0))
1372                                 rte_pipeline_action_handler_port_bulk(p,
1373                                         p->pkts_mask,
1374                                         default_entry->port_id);
1375                         else {
1376                                 uint32_t pos = default_entry->action;
1377
1378                                 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1379
1380                                 p->action_mask0[pos] |= p->pkts_mask;
1381
1382                                 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1383                                         table->n_pkts_dropped_lkp_miss);
1384                         }
1385                 }
1386
1387                 /* Lookup hit */
1388                 if (lookup_hit_mask != 0) {
1389                         p->pkts_mask = lookup_hit_mask;
1390
1391                         /* Table user actions */
1392                         if (table->f_action_hit != NULL) {
1393                                 table->f_action_hit(p,
1394                                         p->pkts,
1395                                         lookup_hit_mask,
1396                                         p->entries,
1397                                         table->arg_ah);
1398
1399                                 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1400                                         table->n_pkts_dropped_by_lkp_hit_ah);
1401                         }
1402
1403                         /* Table reserved actions */
1404                         RTE_PIPELINE_STATS_TABLE_DROP0(p);
1405                         rte_pipeline_compute_masks(p, p->pkts_mask);
1406                         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1407                                 p->action_mask1[
1408                                         RTE_PIPELINE_ACTION_DROP];
1409                         p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1410                                 p->action_mask1[
1411                                         RTE_PIPELINE_ACTION_PORT];
1412                         p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1413                                 p->action_mask1[
1414                                         RTE_PIPELINE_ACTION_PORT_META];
1415                         p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1416                                 p->action_mask1[
1417                                         RTE_PIPELINE_ACTION_TABLE];
1418
1419                         RTE_PIPELINE_STATS_TABLE_DROP1(p,
1420                                 table->n_pkts_dropped_lkp_hit);
1421                 }
1422
1423                 /* Prepare for next iteration */
1424                 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1425                 table_id = table->table_next_id;
1426                 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1427         }
1428
1429         /* Table reserved action PORT */
1430         rte_pipeline_action_handler_port(p,
1431                 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1432
1433         /* Table reserved action PORT META */
1434         rte_pipeline_action_handler_port_meta(p,
1435                 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1436
1437         /* Table reserved action DROP */
1438         rte_pipeline_action_handler_drop(p,
1439                 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1440
1441         /* Pick candidate for next port IN to serve */
1442         p->port_in_next = port_in->next;
1443
1444         return (int) n_pkts;
1445 }
1446
1447 int
1448 rte_pipeline_flush(struct rte_pipeline *p)
1449 {
1450         uint32_t port_id;
1451
1452         /* Check input arguments */
1453         if (p == NULL) {
1454                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1455                         __func__);
1456                 return -EINVAL;
1457         }
1458
1459         for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1460                 struct rte_port_out *port = &p->ports_out[port_id];
1461
1462                 if (port->ops.f_flush != NULL)
1463                         port->ops.f_flush(port->h_port);
1464         }
1465
1466         return 0;
1467 }
1468
1469 int
1470 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1471         uint32_t port_id, struct rte_mbuf *pkt)
1472 {
1473         struct rte_port_out *port_out = &p->ports_out[port_id];
1474
1475         port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1476
1477         return 0;
1478 }
1479
1480 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1481         uint64_t pkts_mask)
1482 {
1483         pkts_mask &= p->pkts_mask;
1484         p->pkts_mask &= ~pkts_mask;
1485
1486         return 0;
1487 }
1488
1489 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1490         uint64_t pkts_mask)
1491 {
1492         pkts_mask &= p->pkts_mask;
1493         p->pkts_mask &= ~pkts_mask;
1494         p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1495
1496         RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1497         return 0;
1498 }
1499
1500 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1501         struct rte_pipeline_port_in_stats *stats, int clear)
1502 {
1503         struct rte_port_in *port;
1504         int retval;
1505
1506         if (p == NULL) {
1507                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1508                         __func__);
1509                 return -EINVAL;
1510         }
1511
1512         if (port_id >= p->num_ports_in) {
1513                 RTE_LOG(ERR, PIPELINE,
1514                         "%s: port IN ID %u is out of range\n",
1515                         __func__, port_id);
1516                 return -EINVAL;
1517         }
1518
1519         port = &p->ports_in[port_id];
1520
1521         if (port->ops.f_stats != NULL) {
1522                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1523                 if (retval)
1524                         return retval;
1525         } else if (stats != NULL)
1526                 memset(&stats->stats, 0, sizeof(stats->stats));
1527
1528         if (stats != NULL)
1529                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1530
1531         if (clear != 0)
1532                 port->n_pkts_dropped_by_ah = 0;
1533
1534         return 0;
1535 }
1536
1537 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1538         struct rte_pipeline_port_out_stats *stats, int clear)
1539 {
1540         struct rte_port_out *port;
1541         int retval;
1542
1543         if (p == NULL) {
1544                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1545                 return -EINVAL;
1546         }
1547
1548         if (port_id >= p->num_ports_out) {
1549                 RTE_LOG(ERR, PIPELINE,
1550                         "%s: port OUT ID %u is out of range\n", __func__, port_id);
1551                 return -EINVAL;
1552         }
1553
1554         port = &p->ports_out[port_id];
1555         if (port->ops.f_stats != NULL) {
1556                 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1557                 if (retval != 0)
1558                         return retval;
1559         } else if (stats != NULL)
1560                 memset(&stats->stats, 0, sizeof(stats->stats));
1561
1562         if (stats != NULL)
1563                 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1564
1565         if (clear != 0)
1566                 port->n_pkts_dropped_by_ah = 0;
1567
1568         return 0;
1569 }
1570
1571 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1572         struct rte_pipeline_table_stats *stats, int clear)
1573 {
1574         struct rte_table *table;
1575         int retval;
1576
1577         if (p == NULL) {
1578                 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1579                         __func__);
1580                 return -EINVAL;
1581         }
1582
1583         if (table_id >= p->num_tables) {
1584                 RTE_LOG(ERR, PIPELINE,
1585                                 "%s: table %u is out of range\n", __func__, table_id);
1586                 return -EINVAL;
1587         }
1588
1589         table = &p->tables[table_id];
1590         if (table->ops.f_stats != NULL) {
1591                 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1592                 if (retval != 0)
1593                         return retval;
1594         } else if (stats != NULL)
1595                 memset(&stats->stats, 0, sizeof(stats->stats));
1596
1597         if (stats != NULL) {
1598                 stats->n_pkts_dropped_by_lkp_hit_ah =
1599                         table->n_pkts_dropped_by_lkp_hit_ah;
1600                 stats->n_pkts_dropped_by_lkp_miss_ah =
1601                         table->n_pkts_dropped_by_lkp_miss_ah;
1602                 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1603                 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1604         }
1605
1606         if (clear != 0) {
1607                 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1608                 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1609                 table->n_pkts_dropped_lkp_hit = 0;
1610                 table->n_pkts_dropped_lkp_miss = 0;
1611         }
1612
1613         return 0;
1614 }