examples/ip_pipeline/thread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdlib.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_cycles.h>
   9 #include <rte_lcore.h>
  10 #include <rte_ring.h>
  11
  12 #include <rte_table_acl.h>
  13 #include <rte_table_array.h>
  14 #include <rte_table_hash.h>
  15 #include <rte_table_lpm.h>
  16 #include <rte_table_lpm_ipv6.h>
  17
  18 #include "common.h"
  19 #include "thread.h"
  20 #include "pipeline.h"
  21
  22 #ifndef THREAD_PIPELINES_MAX
  23 #define THREAD_PIPELINES_MAX                               256
  24 #endif
  25
  26 #ifndef THREAD_MSGQ_SIZE
  27 #define THREAD_MSGQ_SIZE                                   64
  28 #endif
  29
  30 #ifndef THREAD_TIMER_PERIOD_MS
  31 #define THREAD_TIMER_PERIOD_MS                             100
  32 #endif
  33
  34 /**
  35  * Master thead: data plane thread context
  36  */
  37 struct thread {
  38         struct rte_ring *msgq_req;
  39         struct rte_ring *msgq_rsp;
  40
  41         uint32_t enabled;
  42 };
  43
  44 static struct thread thread[RTE_MAX_LCORE];
  45
  46 /**
  47  * Data plane threads: context
  48  */
  49 struct table_data {
  50         struct rte_table_action *a;
  51 };
  52
  53 struct pipeline_data {
  54         struct rte_pipeline *p;
  55         struct table_data table_data[RTE_PIPELINE_TABLE_MAX];
  56         uint32_t n_tables;
  57
  58         struct rte_ring *msgq_req;
  59         struct rte_ring *msgq_rsp;
  60         uint64_t timer_period; /* Measured in CPU cycles. */
  61         uint64_t time_next;
  62
  63         uint8_t buffer[TABLE_RULE_ACTION_SIZE_MAX];
  64 };
  65
  66 struct thread_data {
  67         struct rte_pipeline *p[THREAD_PIPELINES_MAX];
  68         uint32_t n_pipelines;
  69
  70         struct pipeline_data pipeline_data[THREAD_PIPELINES_MAX];
  71         struct rte_ring *msgq_req;
  72         struct rte_ring *msgq_rsp;
  73         uint64_t timer_period; /* Measured in CPU cycles. */
  74         uint64_t time_next;
  75         uint64_t time_next_min;
  76 } __rte_cache_aligned;
  77
  78 static struct thread_data thread_data[RTE_MAX_LCORE];
  79
  80 /**
  81  * Master thread: data plane thread init
  82  */
  83 static void
  84 thread_free(void)
  85 {
  86         uint32_t i;
  87
  88         for (i = 0; i < RTE_MAX_LCORE; i++) {
  89                 struct thread *t = &thread[i];
  90
  91                 if (!rte_lcore_is_enabled(i))
  92                         continue;
  93
  94                 /* MSGQs */
  95                 if (t->msgq_req)
  96                         rte_ring_free(t->msgq_req);
  97
  98                 if (t->msgq_rsp)
  99                         rte_ring_free(t->msgq_rsp);
 100         }
 101 }
 102
 103 int
 104 thread_init(void)
 105 {
 106         uint32_t i;
 107
 108         RTE_LCORE_FOREACH_SLAVE(i) {
 109                 char name[NAME_MAX];
 110                 struct rte_ring *msgq_req, *msgq_rsp;
 111                 struct thread *t = &thread[i];
 112                 struct thread_data *t_data = &thread_data[i];
 113                 uint32_t cpu_id = rte_lcore_to_socket_id(i);
 114
 115                 /* MSGQs */
 116                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-REQ", i);
 117
 118                 msgq_req = rte_ring_create(name,
 119                         THREAD_MSGQ_SIZE,
 120                         cpu_id,
 121                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 122
 123                 if (msgq_req == NULL) {
 124                         thread_free();
 125                         return -1;
 126                 }
 127
 128                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-RSP", i);
 129
 130                 msgq_rsp = rte_ring_create(name,
 131                         THREAD_MSGQ_SIZE,
 132                         cpu_id,
 133                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 134
 135                 if (msgq_rsp == NULL) {
 136                         thread_free();
 137                         return -1;
 138                 }
 139
 140                 /* Master thread records */
 141                 t->msgq_req = msgq_req;
 142                 t->msgq_rsp = msgq_rsp;
 143                 t->enabled = 1;
 144
 145                 /* Data plane thread records */
 146                 t_data->n_pipelines = 0;
 147                 t_data->msgq_req = msgq_req;
 148                 t_data->msgq_rsp = msgq_rsp;
 149                 t_data->timer_period =
 150                         (rte_get_tsc_hz() * THREAD_TIMER_PERIOD_MS) / 1000;
 151                 t_data->time_next = rte_get_tsc_cycles() + t_data->timer_period;
 152                 t_data->time_next_min = t_data->time_next;
 153         }
 154
 155         return 0;
 156 }
 157
 158 static inline int
 159 thread_is_running(uint32_t thread_id)
 160 {
 161         enum rte_lcore_state_t thread_state;
 162
 163         thread_state = rte_eal_get_lcore_state(thread_id);
 164         return (thread_state == RUNNING) ? 1 : 0;
 165 }
 166
 167 /**
 168  * Pipeline is running when:
 169  *    (A) Pipeline is mapped to a data plane thread AND
 170  *    (B) Its data plane thread is in RUNNING state.
 171  */
 172 static inline int
 173 pipeline_is_running(struct pipeline *p)
 174 {
 175         if (p->enabled == 0)
 176                 return 0;
 177
 178         return thread_is_running(p->thread_id);
 179 }
 180
 181 /**
 182  * Master thread & data plane threads: message passing
 183  */
 184 enum thread_req_type {
 185         THREAD_REQ_PIPELINE_ENABLE = 0,
 186         THREAD_REQ_PIPELINE_DISABLE,
 187         THREAD_REQ_MAX
 188 };
 189
 190 struct thread_msg_req {
 191         enum thread_req_type type;
 192
 193         union {
 194                 struct {
 195                         struct rte_pipeline *p;
 196                         struct {
 197                                 struct rte_table_action *a;
 198                         } table[RTE_PIPELINE_TABLE_MAX];
 199                         struct rte_ring *msgq_req;
 200                         struct rte_ring *msgq_rsp;
 201                         uint32_t timer_period_ms;
 202                         uint32_t n_tables;
 203                 } pipeline_enable;
 204
 205                 struct {
 206                         struct rte_pipeline *p;
 207                 } pipeline_disable;
 208         };
 209 };
 210
 211 struct thread_msg_rsp {
 212         int status;
 213 };
 214
 215 /**
 216  * Master thread
 217  */
 218 static struct thread_msg_req *
 219 thread_msg_alloc(void)
 220 {
 221         size_t size = RTE_MAX(sizeof(struct thread_msg_req),
 222                 sizeof(struct thread_msg_rsp));
 223
 224         return calloc(1, size);
 225 }
 226
 227 static void
 228 thread_msg_free(struct thread_msg_rsp *rsp)
 229 {
 230         free(rsp);
 231 }
 232
 233 static struct thread_msg_rsp *
 234 thread_msg_send_recv(uint32_t thread_id,
 235         struct thread_msg_req *req)
 236 {
 237         struct thread *t = &thread[thread_id];
 238         struct rte_ring *msgq_req = t->msgq_req;
 239         struct rte_ring *msgq_rsp = t->msgq_rsp;
 240         struct thread_msg_rsp *rsp;
 241         int status;
 242
 243         /* send */
 244         do {
 245                 status = rte_ring_sp_enqueue(msgq_req, req);
 246         } while (status == -ENOBUFS);
 247
 248         /* recv */
 249         do {
 250                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 251         } while (status != 0);
 252
 253         return rsp;
 254 }
 255
 256 int
 257 thread_pipeline_enable(uint32_t thread_id,
 258         const char *pipeline_name)
 259 {
 260         struct pipeline *p = pipeline_find(pipeline_name);
 261         struct thread *t;
 262         struct thread_msg_req *req;
 263         struct thread_msg_rsp *rsp;
 264         uint32_t i;
 265         int status;
 266
 267         /* Check input params */
 268         if ((thread_id >= RTE_MAX_LCORE) ||
 269                 (p == NULL) ||
 270                 (p->n_ports_in == 0) ||
 271                 (p->n_ports_out == 0) ||
 272                 (p->n_tables == 0))
 273                 return -1;
 274
 275         t = &thread[thread_id];
 276         if ((t->enabled == 0) ||
 277                 p->enabled)
 278                 return -1;
 279
 280         if (!thread_is_running(thread_id)) {
 281                 struct thread_data *td = &thread_data[thread_id];
 282                 struct pipeline_data *tdp = &td->pipeline_data[td->n_pipelines];
 283
 284                 if (td->n_pipelines >= THREAD_PIPELINES_MAX)
 285                         return -1;
 286
 287                 /* Data plane thread */
 288                 td->p[td->n_pipelines] = p->p;
 289
 290                 tdp->p = p->p;
 291                 for (i = 0; i < p->n_tables; i++)
 292                         tdp->table_data[i].a = p->table[i].a;
 293
 294                 tdp->n_tables = p->n_tables;
 295
 296                 tdp->msgq_req = p->msgq_req;
 297                 tdp->msgq_rsp = p->msgq_rsp;
 298                 tdp->timer_period = (rte_get_tsc_hz() * p->timer_period_ms) / 1000;
 299                 tdp->time_next = rte_get_tsc_cycles() + tdp->timer_period;
 300
 301                 td->n_pipelines++;
 302
 303                 /* Pipeline */
 304                 p->thread_id = thread_id;
 305                 p->enabled = 1;
 306
 307                 return 0;
 308         }
 309
 310         /* Allocate request */
 311         req = thread_msg_alloc();
 312         if (req == NULL)
 313                 return -1;
 314
 315         /* Write request */
 316         req->type = THREAD_REQ_PIPELINE_ENABLE;
 317         req->pipeline_enable.p = p->p;
 318         for (i = 0; i < p->n_tables; i++)
 319                 req->pipeline_enable.table[i].a =
 320                         p->table[i].a;
 321         req->pipeline_enable.msgq_req = p->msgq_req;
 322         req->pipeline_enable.msgq_rsp = p->msgq_rsp;
 323         req->pipeline_enable.timer_period_ms = p->timer_period_ms;
 324         req->pipeline_enable.n_tables = p->n_tables;
 325
 326         /* Send request and wait for response */
 327         rsp = thread_msg_send_recv(thread_id, req);
 328         if (rsp == NULL)
 329                 return -1;
 330
 331         /* Read response */
 332         status = rsp->status;
 333
 334         /* Free response */
 335         thread_msg_free(rsp);
 336
 337         /* Request completion */
 338         if (status)
 339                 return status;
 340
 341         p->thread_id = thread_id;
 342         p->enabled = 1;
 343
 344         return 0;
 345 }
 346
 347 int
 348 thread_pipeline_disable(uint32_t thread_id,
 349         const char *pipeline_name)
 350 {
 351         struct pipeline *p = pipeline_find(pipeline_name);
 352         struct thread *t;
 353         struct thread_msg_req *req;
 354         struct thread_msg_rsp *rsp;
 355         int status;
 356
 357         /* Check input params */
 358         if ((thread_id >= RTE_MAX_LCORE) ||
 359                 (p == NULL))
 360                 return -1;
 361
 362         t = &thread[thread_id];
 363         if (t->enabled == 0)
 364                 return -1;
 365
 366         if (p->enabled == 0)
 367                 return 0;
 368
 369         if (p->thread_id != thread_id)
 370                 return -1;
 371
 372         if (!thread_is_running(thread_id)) {
 373                 struct thread_data *td = &thread_data[thread_id];
 374                 uint32_t i;
 375
 376                 for (i = 0; i < td->n_pipelines; i++) {
 377                         struct pipeline_data *tdp = &td->pipeline_data[i];
 378
 379                         if (tdp->p != p->p)
 380                                 continue;
 381
 382                         /* Data plane thread */
 383                         if (i < td->n_pipelines - 1) {
 384                                 struct rte_pipeline *pipeline_last =
 385                                         td->p[td->n_pipelines - 1];
 386                                 struct pipeline_data *tdp_last =
 387                                         &td->pipeline_data[td->n_pipelines - 1];
 388
 389                                 td->p[i] = pipeline_last;
 390                                 memcpy(tdp, tdp_last, sizeof(*tdp));
 391                         }
 392
 393                         td->n_pipelines--;
 394
 395                         /* Pipeline */
 396                         p->enabled = 0;
 397
 398                         break;
 399                 }
 400
 401                 return 0;
 402         }
 403
 404         /* Allocate request */
 405         req = thread_msg_alloc();
 406         if (req == NULL)
 407                 return -1;
 408
 409         /* Write request */
 410         req->type = THREAD_REQ_PIPELINE_DISABLE;
 411         req->pipeline_disable.p = p->p;
 412
 413         /* Send request and wait for response */
 414         rsp = thread_msg_send_recv(thread_id, req);
 415         if (rsp == NULL)
 416                 return -1;
 417
 418         /* Read response */
 419         status = rsp->status;
 420
 421         /* Free response */
 422         thread_msg_free(rsp);
 423
 424         /* Request completion */
 425         if (status)
 426                 return status;
 427
 428         p->enabled = 0;
 429
 430         return 0;
 431 }
 432
 433 /**
 434  * Data plane threads: message handling
 435  */
 436 static inline struct thread_msg_req *
 437 thread_msg_recv(struct rte_ring *msgq_req)
 438 {
 439         struct thread_msg_req *req;
 440
 441         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
 442
 443         if (status != 0)
 444                 return NULL;
 445
 446         return req;
 447 }
 448
 449 static inline void
 450 thread_msg_send(struct rte_ring *msgq_rsp,
 451         struct thread_msg_rsp *rsp)
 452 {
 453         int status;
 454
 455         do {
 456                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
 457         } while (status == -ENOBUFS);
 458 }
 459
 460 static struct thread_msg_rsp *
 461 thread_msg_handle_pipeline_enable(struct thread_data *t,
 462         struct thread_msg_req *req)
 463 {
 464         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 465         struct pipeline_data *p = &t->pipeline_data[t->n_pipelines];
 466         uint32_t i;
 467
 468         /* Request */
 469         if (t->n_pipelines >= THREAD_PIPELINES_MAX) {
 470                 rsp->status = -1;
 471                 return rsp;
 472         }
 473
 474         t->p[t->n_pipelines] = req->pipeline_enable.p;
 475
 476         p->p = req->pipeline_enable.p;
 477         for (i = 0; i < req->pipeline_enable.n_tables; i++)
 478                 p->table_data[i].a =
 479                         req->pipeline_enable.table[i].a;
 480
 481         p->n_tables = req->pipeline_enable.n_tables;
 482
 483         p->msgq_req = req->pipeline_enable.msgq_req;
 484         p->msgq_rsp = req->pipeline_enable.msgq_rsp;
 485         p->timer_period =
 486                 (rte_get_tsc_hz() * req->pipeline_enable.timer_period_ms) / 1000;
 487         p->time_next = rte_get_tsc_cycles() + p->timer_period;
 488
 489         t->n_pipelines++;
 490
 491         /* Response */
 492         rsp->status = 0;
 493         return rsp;
 494 }
 495
 496 static struct thread_msg_rsp *
 497 thread_msg_handle_pipeline_disable(struct thread_data *t,
 498         struct thread_msg_req *req)
 499 {
 500         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 501         uint32_t n_pipelines = t->n_pipelines;
 502         struct rte_pipeline *pipeline = req->pipeline_disable.p;
 503         uint32_t i;
 504
 505         /* find pipeline */
 506         for (i = 0; i < n_pipelines; i++) {
 507                 struct pipeline_data *p = &t->pipeline_data[i];
 508
 509                 if (p->p != pipeline)
 510                         continue;
 511
 512                 if (i < n_pipelines - 1) {
 513                         struct rte_pipeline *pipeline_last =
 514                                 t->p[n_pipelines - 1];
 515                         struct pipeline_data *p_last =
 516                                 &t->pipeline_data[n_pipelines - 1];
 517
 518                         t->p[i] = pipeline_last;
 519                         memcpy(p, p_last, sizeof(*p));
 520                 }
 521
 522                 t->n_pipelines--;
 523
 524                 rsp->status = 0;
 525                 return rsp;
 526         }
 527
 528         /* should not get here */
 529         rsp->status = 0;
 530         return rsp;
 531 }
 532
 533 static void
 534 thread_msg_handle(struct thread_data *t)
 535 {
 536         for ( ; ; ) {
 537                 struct thread_msg_req *req;
 538                 struct thread_msg_rsp *rsp;
 539
 540                 req = thread_msg_recv(t->msgq_req);
 541                 if (req == NULL)
 542                         break;
 543
 544                 switch (req->type) {
 545                 case THREAD_REQ_PIPELINE_ENABLE:
 546                         rsp = thread_msg_handle_pipeline_enable(t, req);
 547                         break;
 548
 549                 case THREAD_REQ_PIPELINE_DISABLE:
 550                         rsp = thread_msg_handle_pipeline_disable(t, req);
 551                         break;
 552
 553                 default:
 554                         rsp = (struct thread_msg_rsp *) req;
 555                         rsp->status = -1;
 556                 }
 557
 558                 thread_msg_send(t->msgq_rsp, rsp);
 559         }
 560 }
 561
 562 /**
 563  * Master thread & data plane threads: message passing
 564  */
 565 enum pipeline_req_type {
 566         /* Port IN */
 567         PIPELINE_REQ_PORT_IN_STATS_READ,
 568         PIPELINE_REQ_PORT_IN_ENABLE,
 569         PIPELINE_REQ_PORT_IN_DISABLE,
 570
 571         /* Port OUT */
 572         PIPELINE_REQ_PORT_OUT_STATS_READ,
 573
 574         /* Table */
 575         PIPELINE_REQ_TABLE_STATS_READ,
 576         PIPELINE_REQ_TABLE_RULE_ADD,
 577         PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT,
 578         PIPELINE_REQ_TABLE_RULE_ADD_BULK,
 579         PIPELINE_REQ_TABLE_RULE_DELETE,
 580         PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT,
 581         PIPELINE_REQ_TABLE_RULE_STATS_READ,
 582         PIPELINE_REQ_TABLE_MTR_PROFILE_ADD,
 583         PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE,
 584         PIPELINE_REQ_TABLE_RULE_MTR_READ,
 585         PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE,
 586         PIPELINE_REQ_TABLE_RULE_TTL_READ,
 587         PIPELINE_REQ_MAX
 588 };
 589
 590 struct pipeline_msg_req_port_in_stats_read {
 591         int clear;
 592 };
 593
 594 struct pipeline_msg_req_port_out_stats_read {
 595         int clear;
 596 };
 597
 598 struct pipeline_msg_req_table_stats_read {
 599         int clear;
 600 };
 601
 602 struct pipeline_msg_req_table_rule_add {
 603         struct table_rule_match match;
 604         struct table_rule_action action;
 605 };
 606
 607 struct pipeline_msg_req_table_rule_add_default {
 608         struct table_rule_action action;
 609 };
 610
 611 struct pipeline_msg_req_table_rule_add_bulk {
 612         struct table_rule_list *list;
 613         int bulk;
 614 };
 615
 616 struct pipeline_msg_req_table_rule_delete {
 617         struct table_rule_match match;
 618 };
 619
 620 struct pipeline_msg_req_table_rule_stats_read {
 621         void *data;
 622         int clear;
 623 };
 624
 625 struct pipeline_msg_req_table_mtr_profile_add {
 626         uint32_t meter_profile_id;
 627         struct rte_table_action_meter_profile profile;
 628 };
 629
 630 struct pipeline_msg_req_table_mtr_profile_delete {
 631         uint32_t meter_profile_id;
 632 };
 633
 634 struct pipeline_msg_req_table_rule_mtr_read {
 635         void *data;
 636         uint32_t tc_mask;
 637         int clear;
 638 };
 639
 640 struct pipeline_msg_req_table_dscp_table_update {
 641         uint64_t dscp_mask;
 642         struct rte_table_action_dscp_table dscp_table;
 643 };
 644
 645 struct pipeline_msg_req_table_rule_ttl_read {
 646         void *data;
 647         int clear;
 648 };
 649
 650 struct pipeline_msg_req {
 651         enum pipeline_req_type type;
 652         uint32_t id; /* Port IN, port OUT or table ID */
 653
 654         RTE_STD_C11
 655         union {
 656                 struct pipeline_msg_req_port_in_stats_read port_in_stats_read;
 657                 struct pipeline_msg_req_port_out_stats_read port_out_stats_read;
 658                 struct pipeline_msg_req_table_stats_read table_stats_read;
 659                 struct pipeline_msg_req_table_rule_add table_rule_add;
 660                 struct pipeline_msg_req_table_rule_add_default table_rule_add_default;
 661                 struct pipeline_msg_req_table_rule_add_bulk table_rule_add_bulk;
 662                 struct pipeline_msg_req_table_rule_delete table_rule_delete;
 663                 struct pipeline_msg_req_table_rule_stats_read table_rule_stats_read;
 664                 struct pipeline_msg_req_table_mtr_profile_add table_mtr_profile_add;
 665                 struct pipeline_msg_req_table_mtr_profile_delete table_mtr_profile_delete;
 666                 struct pipeline_msg_req_table_rule_mtr_read table_rule_mtr_read;
 667                 struct pipeline_msg_req_table_dscp_table_update table_dscp_table_update;
 668                 struct pipeline_msg_req_table_rule_ttl_read table_rule_ttl_read;
 669         };
 670 };
 671
 672 struct pipeline_msg_rsp_port_in_stats_read {
 673         struct rte_pipeline_port_in_stats stats;
 674 };
 675
 676 struct pipeline_msg_rsp_port_out_stats_read {
 677         struct rte_pipeline_port_out_stats stats;
 678 };
 679
 680 struct pipeline_msg_rsp_table_stats_read {
 681         struct rte_pipeline_table_stats stats;
 682 };
 683
 684 struct pipeline_msg_rsp_table_rule_add {
 685         void *data;
 686 };
 687
 688 struct pipeline_msg_rsp_table_rule_add_default {
 689         void *data;
 690 };
 691
 692 struct pipeline_msg_rsp_table_rule_add_bulk {
 693         uint32_t n_rules;
 694 };
 695
 696 struct pipeline_msg_rsp_table_rule_stats_read {
 697         struct rte_table_action_stats_counters stats;
 698 };
 699
 700 struct pipeline_msg_rsp_table_rule_mtr_read {
 701         struct rte_table_action_mtr_counters stats;
 702 };
 703
 704 struct pipeline_msg_rsp_table_rule_ttl_read {
 705         struct rte_table_action_ttl_counters stats;
 706 };
 707
 708 struct pipeline_msg_rsp {
 709         int status;
 710
 711         RTE_STD_C11
 712         union {
 713                 struct pipeline_msg_rsp_port_in_stats_read port_in_stats_read;
 714                 struct pipeline_msg_rsp_port_out_stats_read port_out_stats_read;
 715                 struct pipeline_msg_rsp_table_stats_read table_stats_read;
 716                 struct pipeline_msg_rsp_table_rule_add table_rule_add;
 717                 struct pipeline_msg_rsp_table_rule_add_default table_rule_add_default;
 718                 struct pipeline_msg_rsp_table_rule_add_bulk table_rule_add_bulk;
 719                 struct pipeline_msg_rsp_table_rule_stats_read table_rule_stats_read;
 720                 struct pipeline_msg_rsp_table_rule_mtr_read table_rule_mtr_read;
 721                 struct pipeline_msg_rsp_table_rule_ttl_read table_rule_ttl_read;
 722         };
 723 };
 724
 725 /**
 726  * Master thread
 727  */
 728 static struct pipeline_msg_req *
 729 pipeline_msg_alloc(void)
 730 {
 731         size_t size = RTE_MAX(sizeof(struct pipeline_msg_req),
 732                 sizeof(struct pipeline_msg_rsp));
 733
 734         return calloc(1, size);
 735 }
 736
 737 static void
 738 pipeline_msg_free(struct pipeline_msg_rsp *rsp)
 739 {
 740         free(rsp);
 741 }
 742
 743 static struct pipeline_msg_rsp *
 744 pipeline_msg_send_recv(struct pipeline *p,
 745         struct pipeline_msg_req *req)
 746 {
 747         struct rte_ring *msgq_req = p->msgq_req;
 748         struct rte_ring *msgq_rsp = p->msgq_rsp;
 749         struct pipeline_msg_rsp *rsp;
 750         int status;
 751
 752         /* send */
 753         do {
 754                 status = rte_ring_sp_enqueue(msgq_req, req);
 755         } while (status == -ENOBUFS);
 756
 757         /* recv */
 758         do {
 759                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 760         } while (status != 0);
 761
 762         return rsp;
 763 }
 764
 765 int
 766 pipeline_port_in_stats_read(const char *pipeline_name,
 767         uint32_t port_id,
 768         struct rte_pipeline_port_in_stats *stats,
 769         int clear)
 770 {
 771         struct pipeline *p;
 772         struct pipeline_msg_req *req;
 773         struct pipeline_msg_rsp *rsp;
 774         int status;
 775
 776         /* Check input params */
 777         if ((pipeline_name == NULL) ||
 778                 (stats == NULL))
 779                 return -1;
 780
 781         p = pipeline_find(pipeline_name);
 782         if ((p == NULL) ||
 783                 (port_id >= p->n_ports_in))
 784                 return -1;
 785
 786         if (!pipeline_is_running(p)) {
 787                 status = rte_pipeline_port_in_stats_read(p->p,
 788                         port_id,
 789                         stats,
 790                         clear);
 791
 792                 return status;
 793         }
 794
 795         /* Allocate request */
 796         req = pipeline_msg_alloc();
 797         if (req == NULL)
 798                 return -1;
 799
 800         /* Write request */
 801         req->type = PIPELINE_REQ_PORT_IN_STATS_READ;
 802         req->id = port_id;
 803         req->port_in_stats_read.clear = clear;
 804
 805         /* Send request and wait for response */
 806         rsp = pipeline_msg_send_recv(p, req);
 807         if (rsp == NULL)
 808                 return -1;
 809
 810         /* Read response */
 811         status = rsp->status;
 812         if (status)
 813                 memcpy(stats, &rsp->port_in_stats_read.stats, sizeof(*stats));
 814
 815         /* Free response */
 816         pipeline_msg_free(rsp);
 817
 818         return status;
 819 }
 820
 821 int
 822 pipeline_port_in_enable(const char *pipeline_name,
 823         uint32_t port_id)
 824 {
 825         struct pipeline *p;
 826         struct pipeline_msg_req *req;
 827         struct pipeline_msg_rsp *rsp;
 828         int status;
 829
 830         /* Check input params */
 831         if (pipeline_name == NULL)
 832                 return -1;
 833
 834         p = pipeline_find(pipeline_name);
 835         if ((p == NULL) ||
 836                 (port_id >= p->n_ports_in))
 837                 return -1;
 838
 839         if (!pipeline_is_running(p)) {
 840                 status = rte_pipeline_port_in_enable(p->p, port_id);
 841                 return status;
 842         }
 843
 844         /* Allocate request */
 845         req = pipeline_msg_alloc();
 846         if (req == NULL)
 847                 return -1;
 848
 849         /* Write request */
 850         req->type = PIPELINE_REQ_PORT_IN_ENABLE;
 851         req->id = port_id;
 852
 853         /* Send request and wait for response */
 854         rsp = pipeline_msg_send_recv(p, req);
 855         if (rsp == NULL)
 856                 return -1;
 857
 858         /* Read response */
 859         status = rsp->status;
 860
 861         /* Free response */
 862         pipeline_msg_free(rsp);
 863
 864         return status;
 865 }
 866
 867 int
 868 pipeline_port_in_disable(const char *pipeline_name,
 869         uint32_t port_id)
 870 {
 871         struct pipeline *p;
 872         struct pipeline_msg_req *req;
 873         struct pipeline_msg_rsp *rsp;
 874         int status;
 875
 876         /* Check input params */
 877         if (pipeline_name == NULL)
 878                 return -1;
 879
 880         p = pipeline_find(pipeline_name);
 881         if ((p == NULL) ||
 882                 (port_id >= p->n_ports_in))
 883                 return -1;
 884
 885         if (!pipeline_is_running(p)) {
 886                 status = rte_pipeline_port_in_disable(p->p, port_id);
 887                 return status;
 888         }
 889
 890         /* Allocate request */
 891         req = pipeline_msg_alloc();
 892         if (req == NULL)
 893                 return -1;
 894
 895         /* Write request */
 896         req->type = PIPELINE_REQ_PORT_IN_DISABLE;
 897         req->id = port_id;
 898
 899         /* Send request and wait for response */
 900         rsp = pipeline_msg_send_recv(p, req);
 901         if (rsp == NULL)
 902                 return -1;
 903
 904         /* Read response */
 905         status = rsp->status;
 906
 907         /* Free response */
 908         pipeline_msg_free(rsp);
 909
 910         return status;
 911 }
 912
 913 int
 914 pipeline_port_out_stats_read(const char *pipeline_name,
 915         uint32_t port_id,
 916         struct rte_pipeline_port_out_stats *stats,
 917         int clear)
 918 {
 919         struct pipeline *p;
 920         struct pipeline_msg_req *req;
 921         struct pipeline_msg_rsp *rsp;
 922         int status;
 923
 924         /* Check input params */
 925         if ((pipeline_name == NULL) ||
 926                 (stats == NULL))
 927                 return -1;
 928
 929         p = pipeline_find(pipeline_name);
 930         if ((p == NULL) ||
 931                 (port_id >= p->n_ports_out))
 932                 return -1;
 933
 934         if (!pipeline_is_running(p)) {
 935                 status = rte_pipeline_port_out_stats_read(p->p,
 936                         port_id,
 937                         stats,
 938                         clear);
 939
 940                 return status;
 941         }
 942
 943         /* Allocate request */
 944         req = pipeline_msg_alloc();
 945         if (req == NULL)
 946                 return -1;
 947
 948         /* Write request */
 949         req->type = PIPELINE_REQ_PORT_OUT_STATS_READ;
 950         req->id = port_id;
 951         req->port_out_stats_read.clear = clear;
 952
 953         /* Send request and wait for response */
 954         rsp = pipeline_msg_send_recv(p, req);
 955         if (rsp == NULL)
 956                 return -1;
 957
 958         /* Read response */
 959         status = rsp->status;
 960         if (status)
 961                 memcpy(stats, &rsp->port_out_stats_read.stats, sizeof(*stats));
 962
 963         /* Free response */
 964         pipeline_msg_free(rsp);
 965
 966         return status;
 967 }
 968
 969 int
 970 pipeline_table_stats_read(const char *pipeline_name,
 971         uint32_t table_id,
 972         struct rte_pipeline_table_stats *stats,
 973         int clear)
 974 {
 975         struct pipeline *p;
 976         struct pipeline_msg_req *req;
 977         struct pipeline_msg_rsp *rsp;
 978         int status;
 979
 980         /* Check input params */
 981         if ((pipeline_name == NULL) ||
 982                 (stats == NULL))
 983                 return -1;
 984
 985         p = pipeline_find(pipeline_name);
 986         if ((p == NULL) ||
 987                 (table_id >= p->n_tables))
 988                 return -1;
 989
 990         if (!pipeline_is_running(p)) {
 991                 status = rte_pipeline_table_stats_read(p->p,
 992                         table_id,
 993                         stats,
 994                         clear);
 995
 996                 return status;
 997         }
 998
 999         /* Allocate request */
1000         req = pipeline_msg_alloc();
1001         if (req == NULL)
1002                 return -1;
1003
1004         /* Write request */
1005         req->type = PIPELINE_REQ_TABLE_STATS_READ;
1006         req->id = table_id;
1007         req->table_stats_read.clear = clear;
1008
1009         /* Send request and wait for response */
1010         rsp = pipeline_msg_send_recv(p, req);
1011         if (rsp == NULL)
1012                 return -1;
1013
1014         /* Read response */
1015         status = rsp->status;
1016         if (status)
1017                 memcpy(stats, &rsp->table_stats_read.stats, sizeof(*stats));
1018
1019         /* Free response */
1020         pipeline_msg_free(rsp);
1021
1022         return status;
1023 }
1024
1025 static int
1026 match_check(struct table_rule_match *match,
1027         struct pipeline *p,
1028         uint32_t table_id)
1029 {
1030         struct table *table;
1031
1032         if ((match == NULL) ||
1033                 (p == NULL) ||
1034                 (table_id >= p->n_tables))
1035                 return -1;
1036
1037         table = &p->table[table_id];
1038         if (match->match_type != table->params.match_type)
1039                 return -1;
1040
1041         switch (match->match_type) {
1042         case TABLE_ACL:
1043         {
1044                 struct table_acl_params *t = &table->params.match.acl;
1045                 struct table_rule_match_acl *r = &match->match.acl;
1046
1047                 if ((r->ip_version && (t->ip_version == 0)) ||
1048                         ((r->ip_version == 0) && t->ip_version))
1049                         return -1;
1050
1051                 if (r->ip_version) {
1052                         if ((r->sa_depth > 32) ||
1053                                 (r->da_depth > 32))
1054                                 return -1;
1055                 } else {
1056                         if ((r->sa_depth > 128) ||
1057                                 (r->da_depth > 128))
1058                                 return -1;
1059                 }
1060                 return 0;
1061         }
1062
1063         case TABLE_ARRAY:
1064                 return 0;
1065
1066         case TABLE_HASH:
1067                 return 0;
1068
1069         case TABLE_LPM:
1070         {
1071                 struct table_lpm_params *t = &table->params.match.lpm;
1072                 struct table_rule_match_lpm *r = &match->match.lpm;
1073
1074                 if ((r->ip_version && (t->key_size != 4)) ||
1075                         ((r->ip_version == 0) && (t->key_size != 16)))
1076                         return -1;
1077
1078                 if (r->ip_version) {
1079                         if (r->depth > 32)
1080                                 return -1;
1081                 } else {
1082                         if (r->depth > 128)
1083                                 return -1;
1084                 }
1085                 return 0;
1086         }
1087
1088         case TABLE_STUB:
1089                 return -1;
1090
1091         default:
1092                 return -1;
1093         }
1094 }
1095
1096 static int
1097 action_check(struct table_rule_action *action,
1098         struct pipeline *p,
1099         uint32_t table_id)
1100 {
1101         struct table_action_profile *ap;
1102
1103         if ((action == NULL) ||
1104                 (p == NULL) ||
1105                 (table_id >= p->n_tables))
1106                 return -1;
1107
1108         ap = p->table[table_id].ap;
1109         if (action->action_mask != ap->params.action_mask)
1110                 return -1;
1111
1112         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1113                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
1114                         (action->fwd.id >= p->n_ports_out))
1115                         return -1;
1116
1117                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
1118                         (action->fwd.id >= p->n_tables))
1119                         return -1;
1120         }
1121
1122         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1123                 uint32_t tc_mask0 = (1 << ap->params.mtr.n_tc) - 1;
1124                 uint32_t tc_mask1 = action->mtr.tc_mask;
1125
1126                 if (tc_mask1 != tc_mask0)
1127                         return -1;
1128         }
1129
1130         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1131                 uint32_t n_subports_per_port =
1132                         ap->params.tm.n_subports_per_port;
1133                 uint32_t n_pipes_per_subport =
1134                         ap->params.tm.n_pipes_per_subport;
1135                 uint32_t subport_id = action->tm.subport_id;
1136                 uint32_t pipe_id = action->tm.pipe_id;
1137
1138                 if ((subport_id >= n_subports_per_port) ||
1139                         (pipe_id >= n_pipes_per_subport))
1140                         return -1;
1141         }
1142
1143         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1144                 uint64_t encap_mask = ap->params.encap.encap_mask;
1145                 enum rte_table_action_encap_type type = action->encap.type;
1146
1147                 if ((encap_mask & (1LLU << type)) == 0)
1148                         return -1;
1149         }
1150
1151         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1152                 int ip_version0 = ap->params.common.ip_version;
1153                 int ip_version1 = action->nat.ip_version;
1154
1155                 if ((ip_version1 && (ip_version0 == 0)) ||
1156                         ((ip_version1 == 0) && ip_version0))
1157                         return -1;
1158         }
1159
1160         return 0;
1161 }
1162
1163 static int
1164 action_default_check(struct table_rule_action *action,
1165         struct pipeline *p,
1166         uint32_t table_id)
1167 {
1168         if ((action == NULL) ||
1169                 (action->action_mask != (1LLU << RTE_TABLE_ACTION_FWD)) ||
1170                 (p == NULL) ||
1171                 (table_id >= p->n_tables))
1172                 return -1;
1173
1174         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1175                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
1176                         (action->fwd.id >= p->n_ports_out))
1177                         return -1;
1178
1179                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
1180                         (action->fwd.id >= p->n_tables))
1181                         return -1;
1182         }
1183
1184         return 0;
1185 }
1186
1187 union table_rule_match_low_level {
1188         struct rte_table_acl_rule_add_params acl_add;
1189         struct rte_table_acl_rule_delete_params acl_delete;
1190         struct rte_table_array_key array;
1191         uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
1192         struct rte_table_lpm_key lpm_ipv4;
1193         struct rte_table_lpm_ipv6_key lpm_ipv6;
1194 };
1195
1196 static int
1197 match_convert(struct table_rule_match *mh,
1198         union table_rule_match_low_level *ml,
1199         int add);
1200
1201 static int
1202 action_convert(struct rte_table_action *a,
1203         struct table_rule_action *action,
1204         struct rte_pipeline_table_entry *data);
1205
1206 struct table_ll {
1207         struct rte_pipeline *p;
1208         int table_id;
1209         struct rte_table_action *a;
1210         int bulk_supported;
1211 };
1212
1213 static int
1214 table_rule_add_bulk_ll(struct table_ll *table,
1215         struct table_rule_list *list,
1216         uint32_t *n_rules)
1217 {
1218         union table_rule_match_low_level *match_ll = NULL;
1219         uint8_t *action_ll = NULL;
1220         void **match_ll_ptr = NULL;
1221         struct rte_pipeline_table_entry **action_ll_ptr = NULL;
1222         struct rte_pipeline_table_entry **entries_ptr = NULL;
1223         int *found = NULL;
1224         struct table_rule *rule;
1225         uint32_t n, i;
1226         int status = 0;
1227
1228         n = 0;
1229         TAILQ_FOREACH(rule, list, node)
1230                 n++;
1231
1232         /* Memory allocation */
1233         match_ll = calloc(n, sizeof(union table_rule_match_low_level));
1234         action_ll = calloc(n, TABLE_RULE_ACTION_SIZE_MAX);
1235
1236         match_ll_ptr = calloc(n, sizeof(void *));
1237         action_ll_ptr = calloc(n, sizeof(struct rte_pipeline_table_entry *));
1238
1239         entries_ptr = calloc(n, sizeof(struct rte_pipeline_table_entry *));
1240         found = calloc(n, sizeof(int));
1241
1242         if (match_ll == NULL ||
1243                 action_ll == NULL ||
1244                 match_ll_ptr == NULL ||
1245                 action_ll_ptr == NULL ||
1246                 entries_ptr == NULL ||
1247                 found == NULL) {
1248                         status = -ENOMEM;
1249                         goto table_rule_add_bulk_ll_free;
1250         }
1251
1252         /* Init */
1253         for (i = 0; i < n; i++) {
1254                 match_ll_ptr[i] = (void *)&match_ll[i];
1255                 action_ll_ptr[i] = (struct rte_pipeline_table_entry *)
1256                         &action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
1257         }
1258
1259         /* Rule (match, action) conversion */
1260         i = 0;
1261         TAILQ_FOREACH(rule, list, node) {
1262                 status = match_convert(&rule->match, match_ll_ptr[i], 1);
1263                 if (status)
1264                         goto table_rule_add_bulk_ll_free;
1265
1266                 status = action_convert(table->a, &rule->action, action_ll_ptr[i]);
1267                 if (status)
1268                         goto table_rule_add_bulk_ll_free;
1269
1270                 i++;
1271         }
1272
1273         /* Add rule (match, action) to table */
1274         if (table->bulk_supported) {
1275                 status = rte_pipeline_table_entry_add_bulk(table->p,
1276                         table->table_id,
1277                         match_ll_ptr,
1278                         action_ll_ptr,
1279                         n,
1280                         found,
1281                         entries_ptr);
1282                 if (status)
1283                         goto table_rule_add_bulk_ll_free;
1284         } else
1285                 for (i = 0; i < n; i++) {
1286                         status = rte_pipeline_table_entry_add(table->p,
1287                                 table->table_id,
1288                                 match_ll_ptr[i],
1289                                 action_ll_ptr[i],
1290                                 &found[i],
1291                                 &entries_ptr[i]);
1292                         if (status) {
1293                                 if (i == 0)
1294                                         goto table_rule_add_bulk_ll_free;
1295
1296                                 /* No roll-back. */
1297                                 status = 0;
1298                                 n = i;
1299                                 break;
1300                         }
1301                 }
1302
1303         /* Write back to the rule list. */
1304         i = 0;
1305         TAILQ_FOREACH(rule, list, node) {
1306                 if (i >= n)
1307                         break;
1308
1309                 rule->data = entries_ptr[i];
1310
1311                 i++;
1312         }
1313
1314         *n_rules = n;
1315
1316         /* Free */
1317 table_rule_add_bulk_ll_free:
1318         free(found);
1319         free(entries_ptr);
1320         free(action_ll_ptr);
1321         free(match_ll_ptr);
1322         free(action_ll);
1323         free(match_ll);
1324
1325         return status;
1326 }
1327
1328 int
1329 pipeline_table_rule_add(const char *pipeline_name,
1330         uint32_t table_id,
1331         struct table_rule_match *match,
1332         struct table_rule_action *action)
1333 {
1334         struct pipeline *p;
1335         struct table *table;
1336         struct pipeline_msg_req *req;
1337         struct pipeline_msg_rsp *rsp;
1338         struct table_rule *rule;
1339         int status;
1340
1341         /* Check input params */
1342         if ((pipeline_name == NULL) ||
1343                 (match == NULL) ||
1344                 (action == NULL))
1345                 return -1;
1346
1347         p = pipeline_find(pipeline_name);
1348         if ((p == NULL) ||
1349                 (table_id >= p->n_tables) ||
1350                 match_check(match, p, table_id) ||
1351                 action_check(action, p, table_id))
1352                 return -1;
1353
1354         table = &p->table[table_id];
1355
1356         rule = calloc(1, sizeof(struct table_rule));
1357         if (rule == NULL)
1358                 return -1;
1359
1360         memcpy(&rule->match, match, sizeof(*match));
1361         memcpy(&rule->action, action, sizeof(*action));
1362
1363         if (!pipeline_is_running(p)) {
1364                 union table_rule_match_low_level match_ll;
1365                 struct rte_pipeline_table_entry *data_in, *data_out;
1366                 int key_found;
1367                 uint8_t *buffer;
1368
1369                 buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
1370                 if (buffer == NULL) {
1371                         free(rule);
1372                         return -1;
1373                 }
1374
1375                 /* Table match-action rule conversion */
1376                 data_in = (struct rte_pipeline_table_entry *)buffer;
1377
1378                 status = match_convert(match, &match_ll, 1);
1379                 if (status) {
1380                         free(buffer);
1381                         free(rule);
1382                         return -1;
1383                 }
1384
1385                 status = action_convert(table->a, action, data_in);
1386                 if (status) {
1387                         free(buffer);
1388                         free(rule);
1389                         return -1;
1390                 }
1391
1392                 /* Add rule (match, action) to table */
1393                 status = rte_pipeline_table_entry_add(p->p,
1394                                 table_id,
1395                                 &match_ll,
1396                                 data_in,
1397                                 &key_found,
1398                                 &data_out);
1399                 if (status) {
1400                         free(buffer);
1401                         free(rule);
1402                         return -1;
1403                 }
1404
1405                 /* Write Response */
1406                 rule->data = data_out;
1407                 table_rule_add(table, rule);
1408
1409                 free(buffer);
1410                 return 0;
1411         }
1412
1413         /* Allocate request */
1414         req = pipeline_msg_alloc();
1415         if (req == NULL) {
1416                 free(rule);
1417                 return -1;
1418         }
1419
1420         /* Write request */
1421         req->type = PIPELINE_REQ_TABLE_RULE_ADD;
1422         req->id = table_id;
1423         memcpy(&req->table_rule_add.match, match, sizeof(*match));
1424         memcpy(&req->table_rule_add.action, action, sizeof(*action));
1425
1426         /* Send request and wait for response */
1427         rsp = pipeline_msg_send_recv(p, req);
1428         if (rsp == NULL) {
1429                 free(rule);
1430                 return -1;
1431         }
1432
1433         /* Read response */
1434         status = rsp->status;
1435         if (status == 0) {
1436                 rule->data = rsp->table_rule_add.data;
1437                 table_rule_add(table, rule);
1438         } else
1439                 free(rule);
1440
1441         /* Free response */
1442         pipeline_msg_free(rsp);
1443
1444         return status;
1445 }
1446
1447 int
1448 pipeline_table_rule_add_default(const char *pipeline_name,
1449         uint32_t table_id,
1450         struct table_rule_action *action)
1451 {
1452         struct pipeline *p;
1453         struct table *table;
1454         struct pipeline_msg_req *req;
1455         struct pipeline_msg_rsp *rsp;
1456         struct table_rule *rule;
1457         int status;
1458
1459         /* Check input params */
1460         if ((pipeline_name == NULL) ||
1461                 (action == NULL))
1462                 return -1;
1463
1464         p = pipeline_find(pipeline_name);
1465         if ((p == NULL) ||
1466                 (table_id >= p->n_tables) ||
1467                 action_default_check(action, p, table_id))
1468                 return -1;
1469
1470         table = &p->table[table_id];
1471
1472         rule = calloc(1, sizeof(struct table_rule));
1473         if (rule == NULL)
1474                 return -1;
1475
1476         memcpy(&rule->action, action, sizeof(*action));
1477
1478         if (!pipeline_is_running(p)) {
1479                 struct rte_pipeline_table_entry *data_in, *data_out;
1480                 uint8_t *buffer;
1481
1482                 buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
1483                 if (buffer == NULL) {
1484                         free(rule);
1485                         return -1;
1486                 }
1487
1488                 /* Apply actions */
1489                 data_in = (struct rte_pipeline_table_entry *)buffer;
1490
1491                 data_in->action = action->fwd.action;
1492                 if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
1493                         data_in->port_id = action->fwd.id;
1494                 if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
1495                         data_in->table_id = action->fwd.id;
1496
1497                 /* Add default rule to table */
1498                 status = rte_pipeline_table_default_entry_add(p->p,
1499                                 table_id,
1500                                 data_in,
1501                                 &data_out);
1502                 if (status) {
1503                         free(buffer);
1504                         free(rule);
1505                         return -1;
1506                 }
1507
1508                 /* Write Response */
1509                 rule->data = data_out;
1510                 table_rule_default_add(table, rule);
1511
1512                 free(buffer);
1513                 return 0;
1514         }
1515
1516         /* Allocate request */
1517         req = pipeline_msg_alloc();
1518         if (req == NULL) {
1519                 free(rule);
1520                 return -1;
1521         }
1522
1523         /* Write request */
1524         req->type = PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT;
1525         req->id = table_id;
1526         memcpy(&req->table_rule_add_default.action, action, sizeof(*action));
1527
1528         /* Send request and wait for response */
1529         rsp = pipeline_msg_send_recv(p, req);
1530         if (rsp == NULL) {
1531                 free(rule);
1532                 return -1;
1533         }
1534
1535         /* Read response */
1536         status = rsp->status;
1537         if (status == 0) {
1538                 rule->data = rsp->table_rule_add_default.data;
1539                 table_rule_default_add(table, rule);
1540         } else
1541                 free(rule);
1542
1543         /* Free response */
1544         pipeline_msg_free(rsp);
1545
1546         return status;
1547 }
1548
1549 static uint32_t
1550 table_rule_list_free(struct table_rule_list *list)
1551 {
1552         uint32_t n = 0;
1553
1554         if (!list)
1555                 return 0;
1556
1557         for ( ; ; ) {
1558                 struct table_rule *rule;
1559
1560                 rule = TAILQ_FIRST(list);
1561                 if (rule == NULL)
1562                         break;
1563
1564                 TAILQ_REMOVE(list, rule, node);
1565                 free(rule);
1566                 n++;
1567         }
1568
1569         free(list);
1570         return n;
1571 }
1572
1573 int
1574 pipeline_table_rule_add_bulk(const char *pipeline_name,
1575         uint32_t table_id,
1576         struct table_rule_list *list,
1577         uint32_t *n_rules_added,
1578         uint32_t *n_rules_not_added)
1579 {
1580         struct pipeline *p;
1581         struct table *table;
1582         struct pipeline_msg_req *req;
1583         struct pipeline_msg_rsp *rsp;
1584         struct table_rule *rule;
1585         int status = 0;
1586
1587         /* Check input params */
1588         if ((pipeline_name == NULL) ||
1589                 (list == NULL) ||
1590                 TAILQ_EMPTY(list) ||
1591                 (n_rules_added == NULL) ||
1592                 (n_rules_not_added == NULL)) {
1593                 table_rule_list_free(list);
1594                 return -EINVAL;
1595         }
1596
1597         p = pipeline_find(pipeline_name);
1598         if ((p == NULL) ||
1599                 (table_id >= p->n_tables)) {
1600                 table_rule_list_free(list);
1601                 return -EINVAL;
1602         }
1603
1604         table = &p->table[table_id];
1605
1606         TAILQ_FOREACH(rule, list, node)
1607                 if (match_check(&rule->match, p, table_id) ||
1608                         action_check(&rule->action, p, table_id)) {
1609                         table_rule_list_free(list);
1610                         return -EINVAL;
1611                 }
1612
1613         if (!pipeline_is_running(p)) {
1614                 struct table_ll table_ll = {
1615                         .p = p->p,
1616                         .table_id = table_id,
1617                         .a = table->a,
1618                         .bulk_supported = table->params.match_type == TABLE_ACL,
1619                 };
1620
1621                 status = table_rule_add_bulk_ll(&table_ll, list, n_rules_added);
1622                 if (status) {
1623                         table_rule_list_free(list);
1624                         return status;
1625                 }
1626
1627                 table_rule_add_bulk(table, list, *n_rules_added);
1628                 *n_rules_not_added = table_rule_list_free(list);
1629                 return 0;
1630         }
1631
1632         /* Allocate request */
1633         req = pipeline_msg_alloc();
1634         if (req == NULL) {
1635                 table_rule_list_free(list);
1636                 return -ENOMEM;
1637         }
1638
1639         /* Write request */
1640         req->type = PIPELINE_REQ_TABLE_RULE_ADD_BULK;
1641         req->id = table_id;
1642         req->table_rule_add_bulk.list = list;
1643         req->table_rule_add_bulk.bulk = table->params.match_type == TABLE_ACL;
1644
1645         /* Send request and wait for response */
1646         rsp = pipeline_msg_send_recv(p, req);
1647         if (rsp == NULL) {
1648                 table_rule_list_free(list);
1649                 return -ENOMEM;
1650         }
1651
1652         /* Read response */
1653         status = rsp->status;
1654         if (status == 0) {
1655                 *n_rules_added = rsp->table_rule_add_bulk.n_rules;
1656
1657                 table_rule_add_bulk(table, list, *n_rules_added);
1658                 *n_rules_not_added = table_rule_list_free(list);
1659         } else
1660                 table_rule_list_free(list);
1661
1662
1663         /* Free response */
1664         pipeline_msg_free(rsp);
1665
1666         return status;
1667 }
1668
1669 int
1670 pipeline_table_rule_delete(const char *pipeline_name,
1671         uint32_t table_id,
1672         struct table_rule_match *match)
1673 {
1674         struct pipeline *p;
1675         struct table *table;
1676         struct pipeline_msg_req *req;
1677         struct pipeline_msg_rsp *rsp;
1678         int status;
1679
1680         /* Check input params */
1681         if ((pipeline_name == NULL) ||
1682                 (match == NULL))
1683                 return -1;
1684
1685         p = pipeline_find(pipeline_name);
1686         if ((p == NULL) ||
1687                 (table_id >= p->n_tables) ||
1688                 match_check(match, p, table_id))
1689                 return -1;
1690
1691         table = &p->table[table_id];
1692
1693         if (!pipeline_is_running(p)) {
1694                 union table_rule_match_low_level match_ll;
1695                 int key_found;
1696
1697                 status = match_convert(match, &match_ll, 0);
1698                 if (status)
1699                         return -1;
1700
1701                 status = rte_pipeline_table_entry_delete(p->p,
1702                                 table_id,
1703                                 &match_ll,
1704                                 &key_found,
1705                                 NULL);
1706
1707                 if (status == 0)
1708                         table_rule_delete(table, match);
1709
1710                 return status;
1711         }
1712
1713         /* Allocate request */
1714         req = pipeline_msg_alloc();
1715         if (req == NULL)
1716                 return -1;
1717
1718         /* Write request */
1719         req->type = PIPELINE_REQ_TABLE_RULE_DELETE;
1720         req->id = table_id;
1721         memcpy(&req->table_rule_delete.match, match, sizeof(*match));
1722
1723         /* Send request and wait for response */
1724         rsp = pipeline_msg_send_recv(p, req);
1725         if (rsp == NULL)
1726                 return -1;
1727
1728         /* Read response */
1729         status = rsp->status;
1730         if (status == 0)
1731                 table_rule_delete(table, match);
1732
1733         /* Free response */
1734         pipeline_msg_free(rsp);
1735
1736         return status;
1737 }
1738
1739 int
1740 pipeline_table_rule_delete_default(const char *pipeline_name,
1741         uint32_t table_id)
1742 {
1743         struct pipeline *p;
1744         struct table *table;
1745         struct pipeline_msg_req *req;
1746         struct pipeline_msg_rsp *rsp;
1747         int status;
1748
1749         /* Check input params */
1750         if (pipeline_name == NULL)
1751                 return -1;
1752
1753         p = pipeline_find(pipeline_name);
1754         if ((p == NULL) ||
1755                 (table_id >= p->n_tables))
1756                 return -1;
1757
1758         table = &p->table[table_id];
1759
1760         if (!pipeline_is_running(p)) {
1761                 status = rte_pipeline_table_default_entry_delete(p->p,
1762                         table_id,
1763                         NULL);
1764
1765                 if (status == 0)
1766                         table_rule_default_delete(table);
1767
1768                 return status;
1769         }
1770
1771         /* Allocate request */
1772         req = pipeline_msg_alloc();
1773         if (req == NULL)
1774                 return -1;
1775
1776         /* Write request */
1777         req->type = PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT;
1778         req->id = table_id;
1779
1780         /* Send request and wait for response */
1781         rsp = pipeline_msg_send_recv(p, req);
1782         if (rsp == NULL)
1783                 return -1;
1784
1785         /* Read response */
1786         status = rsp->status;
1787         if (status == 0)
1788                 table_rule_default_delete(table);
1789
1790         /* Free response */
1791         pipeline_msg_free(rsp);
1792
1793         return status;
1794 }
1795
1796 int
1797 pipeline_table_rule_stats_read(const char *pipeline_name,
1798         uint32_t table_id,
1799         struct table_rule_match *match,
1800         struct rte_table_action_stats_counters *stats,
1801         int clear)
1802 {
1803         struct pipeline *p;
1804         struct table *table;
1805         struct pipeline_msg_req *req;
1806         struct pipeline_msg_rsp *rsp;
1807         struct table_rule *rule;
1808         int status;
1809
1810         /* Check input params */
1811         if ((pipeline_name == NULL) ||
1812                 (match == NULL) ||
1813                 (stats == NULL))
1814                 return -1;
1815
1816         p = pipeline_find(pipeline_name);
1817         if ((p == NULL) ||
1818                 (table_id >= p->n_tables) ||
1819                 match_check(match, p, table_id))
1820                 return -1;
1821
1822         table = &p->table[table_id];
1823         rule = table_rule_find(table, match);
1824         if (rule == NULL)
1825                 return -1;
1826
1827         if (!pipeline_is_running(p)) {
1828                 status = rte_table_action_stats_read(table->a,
1829                         rule->data,
1830                         stats,
1831                         clear);
1832
1833                 return status;
1834         }
1835
1836         /* Allocate request */
1837         req = pipeline_msg_alloc();
1838         if (req == NULL)
1839                 return -1;
1840
1841         /* Write request */
1842         req->type = PIPELINE_REQ_TABLE_RULE_STATS_READ;
1843         req->id = table_id;
1844         req->table_rule_stats_read.data = rule->data;
1845         req->table_rule_stats_read.clear = clear;
1846
1847         /* Send request and wait for response */
1848         rsp = pipeline_msg_send_recv(p, req);
1849         if (rsp == NULL)
1850                 return -1;
1851
1852         /* Read response */
1853         status = rsp->status;
1854         if (status == 0)
1855                 memcpy(stats, &rsp->table_rule_stats_read.stats, sizeof(*stats));
1856
1857         /* Free response */
1858         pipeline_msg_free(rsp);
1859
1860         return status;
1861 }
1862
1863 int
1864 pipeline_table_mtr_profile_add(const char *pipeline_name,
1865         uint32_t table_id,
1866         uint32_t meter_profile_id,
1867         struct rte_table_action_meter_profile *profile)
1868 {
1869         struct pipeline *p;
1870         struct pipeline_msg_req *req;
1871         struct pipeline_msg_rsp *rsp;
1872         int status;
1873
1874         /* Check input params */
1875         if ((pipeline_name == NULL) ||
1876                 (profile == NULL))
1877                 return -1;
1878
1879         p = pipeline_find(pipeline_name);
1880         if ((p == NULL) ||
1881                 (table_id >= p->n_tables))
1882                 return -1;
1883
1884         if (!pipeline_is_running(p)) {
1885                 struct rte_table_action *a = p->table[table_id].a;
1886
1887                 status = rte_table_action_meter_profile_add(a,
1888                         meter_profile_id,
1889                         profile);
1890
1891                 return status;
1892         }
1893
1894         /* Allocate request */
1895         req = pipeline_msg_alloc();
1896         if (req == NULL)
1897                 return -1;
1898
1899         /* Write request */
1900         req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_ADD;
1901         req->id = table_id;
1902         req->table_mtr_profile_add.meter_profile_id = meter_profile_id;
1903         memcpy(&req->table_mtr_profile_add.profile, profile, sizeof(*profile));
1904
1905         /* Send request and wait for response */
1906         rsp = pipeline_msg_send_recv(p, req);
1907         if (rsp == NULL)
1908                 return -1;
1909
1910         /* Read response */
1911         status = rsp->status;
1912
1913         /* Free response */
1914         pipeline_msg_free(rsp);
1915
1916         return status;
1917 }
1918
1919 int
1920 pipeline_table_mtr_profile_delete(const char *pipeline_name,
1921         uint32_t table_id,
1922         uint32_t meter_profile_id)
1923 {
1924         struct pipeline *p;
1925         struct pipeline_msg_req *req;
1926         struct pipeline_msg_rsp *rsp;
1927         int status;
1928
1929         /* Check input params */
1930         if (pipeline_name == NULL)
1931                 return -1;
1932
1933         p = pipeline_find(pipeline_name);
1934         if ((p == NULL) ||
1935                 (table_id >= p->n_tables))
1936                 return -1;
1937
1938         if (!pipeline_is_running(p)) {
1939                 struct rte_table_action *a = p->table[table_id].a;
1940
1941                 status = rte_table_action_meter_profile_delete(a,
1942                                 meter_profile_id);
1943
1944                 return status;
1945         }
1946
1947         /* Allocate request */
1948         req = pipeline_msg_alloc();
1949         if (req == NULL)
1950                 return -1;
1951
1952         /* Write request */
1953         req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE;
1954         req->id = table_id;
1955         req->table_mtr_profile_delete.meter_profile_id = meter_profile_id;
1956
1957         /* Send request and wait for response */
1958         rsp = pipeline_msg_send_recv(p, req);
1959         if (rsp == NULL)
1960                 return -1;
1961
1962         /* Read response */
1963         status = rsp->status;
1964
1965         /* Free response */
1966         pipeline_msg_free(rsp);
1967
1968         return status;
1969 }
1970
1971 int
1972 pipeline_table_rule_mtr_read(const char *pipeline_name,
1973         uint32_t table_id,
1974         struct table_rule_match *match,
1975         struct rte_table_action_mtr_counters *stats,
1976         int clear)
1977 {
1978         struct pipeline *p;
1979         struct table *table;
1980         struct pipeline_msg_req *req;
1981         struct pipeline_msg_rsp *rsp;
1982         struct table_rule *rule;
1983         uint32_t tc_mask;
1984         int status;
1985
1986         /* Check input params */
1987         if ((pipeline_name == NULL) ||
1988                 (match == NULL) ||
1989                 (stats == NULL))
1990                 return -1;
1991
1992         p = pipeline_find(pipeline_name);
1993         if ((p == NULL) ||
1994                 (table_id >= p->n_tables) ||
1995                 match_check(match, p, table_id))
1996                 return -1;
1997
1998         table = &p->table[table_id];
1999         tc_mask = (1 << table->ap->params.mtr.n_tc) - 1;
2000
2001         rule = table_rule_find(table, match);
2002         if (rule == NULL)
2003                 return -1;
2004
2005         if (!pipeline_is_running(p)) {
2006                 status = rte_table_action_meter_read(table->a,
2007                                 rule->data,
2008                                 tc_mask,
2009                                 stats,
2010                                 clear);
2011
2012                 return status;
2013         }
2014
2015         /* Allocate request */
2016         req = pipeline_msg_alloc();
2017         if (req == NULL)
2018                 return -1;
2019
2020         /* Write request */
2021         req->type = PIPELINE_REQ_TABLE_RULE_MTR_READ;
2022         req->id = table_id;
2023         req->table_rule_mtr_read.data = rule->data;
2024         req->table_rule_mtr_read.tc_mask = tc_mask;
2025         req->table_rule_mtr_read.clear = clear;
2026
2027         /* Send request and wait for response */
2028         rsp = pipeline_msg_send_recv(p, req);
2029         if (rsp == NULL)
2030                 return -1;
2031
2032         /* Read response */
2033         status = rsp->status;
2034         if (status == 0)
2035                 memcpy(stats, &rsp->table_rule_mtr_read.stats, sizeof(*stats));
2036
2037         /* Free response */
2038         pipeline_msg_free(rsp);
2039
2040         return status;
2041 }
2042
2043 int
2044 pipeline_table_dscp_table_update(const char *pipeline_name,
2045         uint32_t table_id,
2046         uint64_t dscp_mask,
2047         struct rte_table_action_dscp_table *dscp_table)
2048 {
2049         struct pipeline *p;
2050         struct pipeline_msg_req *req;
2051         struct pipeline_msg_rsp *rsp;
2052         int status;
2053
2054         /* Check input params */
2055         if ((pipeline_name == NULL) ||
2056                 (dscp_table == NULL))
2057                 return -1;
2058
2059         p = pipeline_find(pipeline_name);
2060         if ((p == NULL) ||
2061                 (table_id >= p->n_tables))
2062                 return -1;
2063
2064         if (!pipeline_is_running(p)) {
2065                 struct rte_table_action *a = p->table[table_id].a;
2066
2067                 status = rte_table_action_dscp_table_update(a,
2068                                 dscp_mask,
2069                                 dscp_table);
2070
2071                 return status;
2072         }
2073
2074         /* Allocate request */
2075         req = pipeline_msg_alloc();
2076         if (req == NULL)
2077                 return -1;
2078
2079         /* Write request */
2080         req->type = PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE;
2081         req->id = table_id;
2082         req->table_dscp_table_update.dscp_mask = dscp_mask;
2083         memcpy(&req->table_dscp_table_update.dscp_table,
2084                 dscp_table, sizeof(*dscp_table));
2085
2086         /* Send request and wait for response */
2087         rsp = pipeline_msg_send_recv(p, req);
2088         if (rsp == NULL)
2089                 return -1;
2090
2091         /* Read response */
2092         status = rsp->status;
2093
2094         /* Free response */
2095         pipeline_msg_free(rsp);
2096
2097         return status;
2098 }
2099
2100 int
2101 pipeline_table_rule_ttl_read(const char *pipeline_name,
2102         uint32_t table_id,
2103         struct table_rule_match *match,
2104         struct rte_table_action_ttl_counters *stats,
2105         int clear)
2106 {
2107         struct pipeline *p;
2108         struct table *table;
2109         struct pipeline_msg_req *req;
2110         struct pipeline_msg_rsp *rsp;
2111         struct table_rule *rule;
2112         int status;
2113
2114         /* Check input params */
2115         if ((pipeline_name == NULL) ||
2116                 (match == NULL) ||
2117                 (stats == NULL))
2118                 return -1;
2119
2120         p = pipeline_find(pipeline_name);
2121         if ((p == NULL) ||
2122                 (table_id >= p->n_tables) ||
2123                 match_check(match, p, table_id))
2124                 return -1;
2125
2126         table = &p->table[table_id];
2127         if (!table->ap->params.ttl.n_packets_enabled)
2128                 return -1;
2129
2130         rule = table_rule_find(table, match);
2131         if (rule == NULL)
2132                 return -1;
2133
2134         if (!pipeline_is_running(p)) {
2135                 status = rte_table_action_ttl_read(table->a,
2136                                 rule->data,
2137                                 stats,
2138                                 clear);
2139
2140                 return status;
2141         }
2142
2143         /* Allocate request */
2144         req = pipeline_msg_alloc();
2145         if (req == NULL)
2146                 return -1;
2147
2148         /* Write request */
2149         req->type = PIPELINE_REQ_TABLE_RULE_TTL_READ;
2150         req->id = table_id;
2151         req->table_rule_ttl_read.data = rule->data;
2152         req->table_rule_ttl_read.clear = clear;
2153
2154         /* Send request and wait for response */
2155         rsp = pipeline_msg_send_recv(p, req);
2156         if (rsp == NULL)
2157                 return -1;
2158
2159         /* Read response */
2160         status = rsp->status;
2161         if (status == 0)
2162                 memcpy(stats, &rsp->table_rule_ttl_read.stats, sizeof(*stats));
2163
2164         /* Free response */
2165         pipeline_msg_free(rsp);
2166
2167         return status;
2168 }
2169
2170 /**
2171  * Data plane threads: message handling
2172  */
2173 static inline struct pipeline_msg_req *
2174 pipeline_msg_recv(struct rte_ring *msgq_req)
2175 {
2176         struct pipeline_msg_req *req;
2177
2178         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
2179
2180         if (status != 0)
2181                 return NULL;
2182
2183         return req;
2184 }
2185
2186 static inline void
2187 pipeline_msg_send(struct rte_ring *msgq_rsp,
2188         struct pipeline_msg_rsp *rsp)
2189 {
2190         int status;
2191
2192         do {
2193                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
2194         } while (status == -ENOBUFS);
2195 }
2196
2197 static struct pipeline_msg_rsp *
2198 pipeline_msg_handle_port_in_stats_read(struct pipeline_data *p,
2199         struct pipeline_msg_req *req)
2200 {
2201         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2202         uint32_t port_id = req->id;
2203         int clear = req->port_in_stats_read.clear;
2204
2205         rsp->status = rte_pipeline_port_in_stats_read(p->p,
2206                 port_id,
2207                 &rsp->port_in_stats_read.stats,
2208                 clear);
2209
2210         return rsp;
2211 }
2212
2213 static struct pipeline_msg_rsp *
2214 pipeline_msg_handle_port_in_enable(struct pipeline_data *p,
2215         struct pipeline_msg_req *req)
2216 {
2217         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2218         uint32_t port_id = req->id;
2219
2220         rsp->status = rte_pipeline_port_in_enable(p->p,
2221                 port_id);
2222
2223         return rsp;
2224 }
2225
2226 static struct pipeline_msg_rsp *
2227 pipeline_msg_handle_port_in_disable(struct pipeline_data *p,
2228         struct pipeline_msg_req *req)
2229 {
2230         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2231         uint32_t port_id = req->id;
2232
2233         rsp->status = rte_pipeline_port_in_disable(p->p,
2234                 port_id);
2235
2236         return rsp;
2237 }
2238
2239 static struct pipeline_msg_rsp *
2240 pipeline_msg_handle_port_out_stats_read(struct pipeline_data *p,
2241         struct pipeline_msg_req *req)
2242 {
2243         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2244         uint32_t port_id = req->id;
2245         int clear = req->port_out_stats_read.clear;
2246
2247         rsp->status = rte_pipeline_port_out_stats_read(p->p,
2248                 port_id,
2249                 &rsp->port_out_stats_read.stats,
2250                 clear);
2251
2252         return rsp;
2253 }
2254
2255 static struct pipeline_msg_rsp *
2256 pipeline_msg_handle_table_stats_read(struct pipeline_data *p,
2257         struct pipeline_msg_req *req)
2258 {
2259         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2260         uint32_t port_id = req->id;
2261         int clear = req->table_stats_read.clear;
2262
2263         rsp->status = rte_pipeline_table_stats_read(p->p,
2264                 port_id,
2265                 &rsp->table_stats_read.stats,
2266                 clear);
2267
2268         return rsp;
2269 }
2270
2271 static int
2272 match_convert_ipv6_depth(uint32_t depth, uint32_t *depth32)
2273 {
2274         if (depth > 128)
2275                 return -1;
2276
2277         switch (depth / 32) {
2278         case 0:
2279                 depth32[0] = depth;
2280                 depth32[1] = 0;
2281                 depth32[2] = 0;
2282                 depth32[3] = 0;
2283                 return 0;
2284
2285         case 1:
2286                 depth32[0] = 32;
2287                 depth32[1] = depth - 32;
2288                 depth32[2] = 0;
2289                 depth32[3] = 0;
2290                 return 0;
2291
2292         case 2:
2293                 depth32[0] = 32;
2294                 depth32[1] = 32;
2295                 depth32[2] = depth - 64;
2296                 depth32[3] = 0;
2297                 return 0;
2298
2299         case 3:
2300                 depth32[0] = 32;
2301                 depth32[1] = 32;
2302                 depth32[2] = 32;
2303                 depth32[3] = depth - 96;
2304                 return 0;
2305
2306         case 4:
2307                 depth32[0] = 32;
2308                 depth32[1] = 32;
2309                 depth32[2] = 32;
2310                 depth32[3] = 32;
2311                 return 0;
2312
2313         default:
2314                 return -1;
2315         }
2316 }
2317
2318 static int
2319 match_convert(struct table_rule_match *mh,
2320         union table_rule_match_low_level *ml,
2321         int add)
2322 {
2323         memset(ml, 0, sizeof(*ml));
2324
2325         switch (mh->match_type) {
2326         case TABLE_ACL:
2327                 if (mh->match.acl.ip_version)
2328                         if (add) {
2329                                 ml->acl_add.field_value[0].value.u8 =
2330                                         mh->match.acl.proto;
2331                                 ml->acl_add.field_value[0].mask_range.u8 =
2332                                         mh->match.acl.proto_mask;
2333
2334                                 ml->acl_add.field_value[1].value.u32 =
2335                                         mh->match.acl.ipv4.sa;
2336                                 ml->acl_add.field_value[1].mask_range.u32 =
2337                                         mh->match.acl.sa_depth;
2338
2339                                 ml->acl_add.field_value[2].value.u32 =
2340                                         mh->match.acl.ipv4.da;
2341                                 ml->acl_add.field_value[2].mask_range.u32 =
2342                                         mh->match.acl.da_depth;
2343
2344                                 ml->acl_add.field_value[3].value.u16 =
2345                                         mh->match.acl.sp0;
2346                                 ml->acl_add.field_value[3].mask_range.u16 =
2347                                         mh->match.acl.sp1;
2348
2349                                 ml->acl_add.field_value[4].value.u16 =
2350                                         mh->match.acl.dp0;
2351                                 ml->acl_add.field_value[4].mask_range.u16 =
2352                                         mh->match.acl.dp1;
2353
2354                                 ml->acl_add.priority =
2355                                         (int32_t) mh->match.acl.priority;
2356                         } else {
2357                                 ml->acl_delete.field_value[0].value.u8 =
2358                                         mh->match.acl.proto;
2359                                 ml->acl_delete.field_value[0].mask_range.u8 =
2360                                         mh->match.acl.proto_mask;
2361
2362                                 ml->acl_delete.field_value[1].value.u32 =
2363                                         mh->match.acl.ipv4.sa;
2364                                 ml->acl_delete.field_value[1].mask_range.u32 =
2365                                         mh->match.acl.sa_depth;
2366
2367                                 ml->acl_delete.field_value[2].value.u32 =
2368                                         mh->match.acl.ipv4.da;
2369                                 ml->acl_delete.field_value[2].mask_range.u32 =
2370                                         mh->match.acl.da_depth;
2371
2372                                 ml->acl_delete.field_value[3].value.u16 =
2373                                         mh->match.acl.sp0;
2374                                 ml->acl_delete.field_value[3].mask_range.u16 =
2375                                         mh->match.acl.sp1;
2376
2377                                 ml->acl_delete.field_value[4].value.u16 =
2378                                         mh->match.acl.dp0;
2379                                 ml->acl_delete.field_value[4].mask_range.u16 =
2380                                         mh->match.acl.dp1;
2381                         }
2382                 else
2383                         if (add) {
2384                                 uint32_t *sa32 =
2385                                         (uint32_t *) mh->match.acl.ipv6.sa;
2386                                 uint32_t *da32 =
2387                                         (uint32_t *) mh->match.acl.ipv6.da;
2388                                 uint32_t sa32_depth[4], da32_depth[4];
2389                                 int status;
2390
2391                                 status = match_convert_ipv6_depth(
2392                                         mh->match.acl.sa_depth,
2393                                         sa32_depth);
2394                                 if (status)
2395                                         return status;
2396
2397                                 status = match_convert_ipv6_depth(
2398                                         mh->match.acl.da_depth,
2399                                         da32_depth);
2400                                 if (status)
2401                                         return status;
2402
2403                                 ml->acl_add.field_value[0].value.u8 =
2404                                         mh->match.acl.proto;
2405                                 ml->acl_add.field_value[0].mask_range.u8 =
2406                                         mh->match.acl.proto_mask;
2407
2408                                 ml->acl_add.field_value[1].value.u32 =
2409                                         rte_be_to_cpu_32(sa32[0]);
2410                                 ml->acl_add.field_value[1].mask_range.u32 =
2411                                         sa32_depth[0];
2412                                 ml->acl_add.field_value[2].value.u32 =
2413                                         rte_be_to_cpu_32(sa32[1]);
2414                                 ml->acl_add.field_value[2].mask_range.u32 =
2415                                         sa32_depth[1];
2416                                 ml->acl_add.field_value[3].value.u32 =
2417                                         rte_be_to_cpu_32(sa32[2]);
2418                                 ml->acl_add.field_value[3].mask_range.u32 =
2419                                         sa32_depth[2];
2420                                 ml->acl_add.field_value[4].value.u32 =
2421                                         rte_be_to_cpu_32(sa32[3]);
2422                                 ml->acl_add.field_value[4].mask_range.u32 =
2423                                         sa32_depth[3];
2424
2425                                 ml->acl_add.field_value[5].value.u32 =
2426                                         rte_be_to_cpu_32(da32[0]);
2427                                 ml->acl_add.field_value[5].mask_range.u32 =
2428                                         da32_depth[0];
2429                                 ml->acl_add.field_value[6].value.u32 =
2430                                         rte_be_to_cpu_32(da32[1]);
2431                                 ml->acl_add.field_value[6].mask_range.u32 =
2432                                         da32_depth[1];
2433                                 ml->acl_add.field_value[7].value.u32 =
2434                                         rte_be_to_cpu_32(da32[2]);
2435                                 ml->acl_add.field_value[7].mask_range.u32 =
2436                                         da32_depth[2];
2437                                 ml->acl_add.field_value[8].value.u32 =
2438                                         rte_be_to_cpu_32(da32[3]);
2439                                 ml->acl_add.field_value[8].mask_range.u32 =
2440                                         da32_depth[3];
2441
2442                                 ml->acl_add.field_value[9].value.u16 =
2443                                         mh->match.acl.sp0;
2444                                 ml->acl_add.field_value[9].mask_range.u16 =
2445                                         mh->match.acl.sp1;
2446
2447                                 ml->acl_add.field_value[10].value.u16 =
2448                                         mh->match.acl.dp0;
2449                                 ml->acl_add.field_value[10].mask_range.u16 =
2450                                         mh->match.acl.dp1;
2451
2452                                 ml->acl_add.priority =
2453                                         (int32_t) mh->match.acl.priority;
2454                         } else {
2455                                 uint32_t *sa32 =
2456                                         (uint32_t *) mh->match.acl.ipv6.sa;
2457                                 uint32_t *da32 =
2458                                         (uint32_t *) mh->match.acl.ipv6.da;
2459                                 uint32_t sa32_depth[4], da32_depth[4];
2460                                 int status;
2461
2462                                 status = match_convert_ipv6_depth(
2463                                         mh->match.acl.sa_depth,
2464                                         sa32_depth);
2465                                 if (status)
2466                                         return status;
2467
2468                                 status = match_convert_ipv6_depth(
2469                                         mh->match.acl.da_depth,
2470                                         da32_depth);
2471                                 if (status)
2472                                         return status;
2473
2474                                 ml->acl_delete.field_value[0].value.u8 =
2475                                         mh->match.acl.proto;
2476                                 ml->acl_delete.field_value[0].mask_range.u8 =
2477                                         mh->match.acl.proto_mask;
2478
2479                                 ml->acl_delete.field_value[1].value.u32 =
2480                                         rte_be_to_cpu_32(sa32[0]);
2481                                 ml->acl_delete.field_value[1].mask_range.u32 =
2482                                         sa32_depth[0];
2483                                 ml->acl_delete.field_value[2].value.u32 =
2484                                         rte_be_to_cpu_32(sa32[1]);
2485                                 ml->acl_delete.field_value[2].mask_range.u32 =
2486                                         sa32_depth[1];
2487                                 ml->acl_delete.field_value[3].value.u32 =
2488                                         rte_be_to_cpu_32(sa32[2]);
2489                                 ml->acl_delete.field_value[3].mask_range.u32 =
2490                                         sa32_depth[2];
2491                                 ml->acl_delete.field_value[4].value.u32 =
2492                                         rte_be_to_cpu_32(sa32[3]);
2493                                 ml->acl_delete.field_value[4].mask_range.u32 =
2494                                         sa32_depth[3];
2495
2496                                 ml->acl_delete.field_value[5].value.u32 =
2497                                         rte_be_to_cpu_32(da32[0]);
2498                                 ml->acl_delete.field_value[5].mask_range.u32 =
2499                                         da32_depth[0];
2500                                 ml->acl_delete.field_value[6].value.u32 =
2501                                         rte_be_to_cpu_32(da32[1]);
2502                                 ml->acl_delete.field_value[6].mask_range.u32 =
2503                                         da32_depth[1];
2504                                 ml->acl_delete.field_value[7].value.u32 =
2505                                         rte_be_to_cpu_32(da32[2]);
2506                                 ml->acl_delete.field_value[7].mask_range.u32 =
2507                                         da32_depth[2];
2508                                 ml->acl_delete.field_value[8].value.u32 =
2509                                         rte_be_to_cpu_32(da32[3]);
2510                                 ml->acl_delete.field_value[8].mask_range.u32 =
2511                                         da32_depth[3];
2512
2513                                 ml->acl_delete.field_value[9].value.u16 =
2514                                         mh->match.acl.sp0;
2515                                 ml->acl_delete.field_value[9].mask_range.u16 =
2516                                         mh->match.acl.sp1;
2517
2518                                 ml->acl_delete.field_value[10].value.u16 =
2519                                         mh->match.acl.dp0;
2520                                 ml->acl_delete.field_value[10].mask_range.u16 =
2521                                         mh->match.acl.dp1;
2522                         }
2523                 return 0;
2524
2525         case TABLE_ARRAY:
2526                 ml->array.pos = mh->match.array.pos;
2527                 return 0;
2528
2529         case TABLE_HASH:
2530                 memcpy(ml->hash, mh->match.hash.key, sizeof(ml->hash));
2531                 return 0;
2532
2533         case TABLE_LPM:
2534                 if (mh->match.lpm.ip_version) {
2535                         ml->lpm_ipv4.ip = mh->match.lpm.ipv4;
2536                         ml->lpm_ipv4.depth = mh->match.lpm.depth;
2537                 } else {
2538                         memcpy(ml->lpm_ipv6.ip,
2539                                 mh->match.lpm.ipv6, sizeof(ml->lpm_ipv6.ip));
2540                         ml->lpm_ipv6.depth = mh->match.lpm.depth;
2541                 }
2542
2543                 return 0;
2544
2545         default:
2546                 return -1;
2547         }
2548 }
2549
2550 static int
2551 action_convert(struct rte_table_action *a,
2552         struct table_rule_action *action,
2553         struct rte_pipeline_table_entry *data)
2554 {
2555         int status;
2556
2557         /* Apply actions */
2558         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
2559                 status = rte_table_action_apply(a,
2560                         data,
2561                         RTE_TABLE_ACTION_FWD,
2562                         &action->fwd);
2563
2564                 if (status)
2565                         return status;
2566         }
2567
2568         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) {
2569                 status = rte_table_action_apply(a,
2570                         data,
2571                         RTE_TABLE_ACTION_LB,
2572                         &action->lb);
2573
2574                 if (status)
2575                         return status;
2576         }
2577
2578         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
2579                 status = rte_table_action_apply(a,
2580                         data,
2581                         RTE_TABLE_ACTION_MTR,
2582                         &action->mtr);
2583
2584                 if (status)
2585                         return status;
2586         }
2587
2588         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
2589                 status = rte_table_action_apply(a,
2590                         data,
2591                         RTE_TABLE_ACTION_TM,
2592                         &action->tm);
2593
2594                 if (status)
2595                         return status;
2596         }
2597
2598         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
2599                 status = rte_table_action_apply(a,
2600                         data,
2601                         RTE_TABLE_ACTION_ENCAP,
2602                         &action->encap);
2603
2604                 if (status)
2605                         return status;
2606         }
2607
2608         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
2609                 status = rte_table_action_apply(a,
2610                         data,
2611                         RTE_TABLE_ACTION_NAT,
2612                         &action->nat);
2613
2614                 if (status)
2615                         return status;
2616         }
2617
2618         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
2619                 status = rte_table_action_apply(a,
2620                         data,
2621                         RTE_TABLE_ACTION_TTL,
2622                         &action->ttl);
2623
2624                 if (status)
2625                         return status;
2626         }
2627
2628         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
2629                 status = rte_table_action_apply(a,
2630                         data,
2631                         RTE_TABLE_ACTION_STATS,
2632                         &action->stats);
2633
2634                 if (status)
2635                         return status;
2636         }
2637
2638         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
2639                 status = rte_table_action_apply(a,
2640                         data,
2641                         RTE_TABLE_ACTION_TIME,
2642                         &action->time);
2643
2644                 if (status)
2645                         return status;
2646         }
2647
2648         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) {
2649                 status = rte_table_action_apply(a,
2650                         data,
2651                         RTE_TABLE_ACTION_SYM_CRYPTO,
2652                         &action->sym_crypto);
2653
2654                 if (status)
2655                         return status;
2656         }
2657
2658         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) {
2659                 status = rte_table_action_apply(a,
2660                         data,
2661                         RTE_TABLE_ACTION_TAG,
2662                         &action->tag);
2663
2664                 if (status)
2665                         return status;
2666         }
2667
2668         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) {
2669                 status = rte_table_action_apply(a,
2670                         data,
2671                         RTE_TABLE_ACTION_DECAP,
2672                         &action->decap);
2673
2674                 if (status)
2675                         return status;
2676         }
2677
2678         return 0;
2679 }
2680
2681 static struct pipeline_msg_rsp *
2682 pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
2683         struct pipeline_msg_req *req)
2684 {
2685         union table_rule_match_low_level match_ll;
2686         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2687         struct table_rule_match *match = &req->table_rule_add.match;
2688         struct table_rule_action *action = &req->table_rule_add.action;
2689         struct rte_pipeline_table_entry *data_in, *data_out;
2690         uint32_t table_id = req->id;
2691         int key_found, status;
2692         struct rte_table_action *a = p->table_data[table_id].a;
2693
2694         /* Apply actions */
2695         memset(p->buffer, 0, sizeof(p->buffer));
2696         data_in = (struct rte_pipeline_table_entry *) p->buffer;
2697
2698         status = match_convert(match, &match_ll, 1);
2699         if (status) {
2700                 rsp->status = -1;
2701                 return rsp;
2702         }
2703
2704         status = action_convert(a, action, data_in);
2705         if (status) {
2706                 rsp->status = -1;
2707                 return rsp;
2708         }
2709
2710         status = rte_pipeline_table_entry_add(p->p,
2711                 table_id,
2712                 &match_ll,
2713                 data_in,
2714                 &key_found,
2715                 &data_out);
2716         if (status) {
2717                 rsp->status = -1;
2718                 return rsp;
2719         }
2720
2721         /* Write response */
2722         rsp->status = 0;
2723         rsp->table_rule_add.data = data_out;
2724
2725         return rsp;
2726 }
2727
2728 static struct pipeline_msg_rsp *
2729 pipeline_msg_handle_table_rule_add_default(struct pipeline_data *p,
2730         struct pipeline_msg_req *req)
2731 {
2732         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2733         struct table_rule_action *action = &req->table_rule_add_default.action;
2734         struct rte_pipeline_table_entry *data_in, *data_out;
2735         uint32_t table_id = req->id;
2736         int status;
2737
2738         /* Apply actions */
2739         memset(p->buffer, 0, sizeof(p->buffer));
2740         data_in = (struct rte_pipeline_table_entry *) p->buffer;
2741
2742         data_in->action = action->fwd.action;
2743         if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
2744                 data_in->port_id = action->fwd.id;
2745         if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
2746                 data_in->table_id = action->fwd.id;
2747
2748         /* Add default rule to table */
2749         status = rte_pipeline_table_default_entry_add(p->p,
2750                 table_id,
2751                 data_in,
2752                 &data_out);
2753         if (status) {
2754                 rsp->status = -1;
2755                 return rsp;
2756         }
2757
2758         /* Write response */
2759         rsp->status = 0;
2760         rsp->table_rule_add_default.data = data_out;
2761
2762         return rsp;
2763 }
2764
2765 static struct pipeline_msg_rsp *
2766 pipeline_msg_handle_table_rule_add_bulk(struct pipeline_data *p,
2767         struct pipeline_msg_req *req)
2768 {
2769         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2770
2771         uint32_t table_id = req->id;
2772         struct table_rule_list *list = req->table_rule_add_bulk.list;
2773         uint32_t bulk = req->table_rule_add_bulk.bulk;
2774
2775         uint32_t n_rules_added;
2776         int status;
2777
2778         struct table_ll table_ll = {
2779                 .p = p->p,
2780                 .table_id = table_id,
2781                 .a = p->table_data[table_id].a,
2782                 .bulk_supported = bulk,
2783         };
2784
2785         status = table_rule_add_bulk_ll(&table_ll, list, &n_rules_added);
2786         if (status) {
2787                 rsp->status = -1;
2788                 rsp->table_rule_add_bulk.n_rules = 0;
2789                 return rsp;
2790         }
2791
2792         /* Write response */
2793         rsp->status = 0;
2794         rsp->table_rule_add_bulk.n_rules = n_rules_added;
2795         return rsp;
2796 }
2797
2798 static struct pipeline_msg_rsp *
2799 pipeline_msg_handle_table_rule_delete(struct pipeline_data *p,
2800         struct pipeline_msg_req *req)
2801 {
2802         union table_rule_match_low_level match_ll;
2803         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2804         struct table_rule_match *match = &req->table_rule_delete.match;
2805         uint32_t table_id = req->id;
2806         int key_found, status;
2807
2808         status = match_convert(match, &match_ll, 0);
2809         if (status) {
2810                 rsp->status = -1;
2811                 return rsp;
2812         }
2813
2814         rsp->status = rte_pipeline_table_entry_delete(p->p,
2815                 table_id,
2816                 &match_ll,
2817                 &key_found,
2818                 NULL);
2819
2820         return rsp;
2821 }
2822
2823 static struct pipeline_msg_rsp *
2824 pipeline_msg_handle_table_rule_delete_default(struct pipeline_data *p,
2825         struct pipeline_msg_req *req)
2826 {
2827         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2828         uint32_t table_id = req->id;
2829
2830         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
2831                 table_id,
2832                 NULL);
2833
2834         return rsp;
2835 }
2836
2837 static struct pipeline_msg_rsp *
2838 pipeline_msg_handle_table_rule_stats_read(struct pipeline_data *p,
2839         struct pipeline_msg_req *req)
2840 {
2841         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2842         uint32_t table_id = req->id;
2843         void *data = req->table_rule_stats_read.data;
2844         int clear = req->table_rule_stats_read.clear;
2845         struct rte_table_action *a = p->table_data[table_id].a;
2846
2847         rsp->status = rte_table_action_stats_read(a,
2848                 data,
2849                 &rsp->table_rule_stats_read.stats,
2850                 clear);
2851
2852         return rsp;
2853 }
2854
2855 static struct pipeline_msg_rsp *
2856 pipeline_msg_handle_table_mtr_profile_add(struct pipeline_data *p,
2857         struct pipeline_msg_req *req)
2858 {
2859         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2860         uint32_t table_id = req->id;
2861         uint32_t meter_profile_id = req->table_mtr_profile_add.meter_profile_id;
2862         struct rte_table_action_meter_profile *profile =
2863                 &req->table_mtr_profile_add.profile;
2864         struct rte_table_action *a = p->table_data[table_id].a;
2865
2866         rsp->status = rte_table_action_meter_profile_add(a,
2867                 meter_profile_id,
2868                 profile);
2869
2870         return rsp;
2871 }
2872
2873 static struct pipeline_msg_rsp *
2874 pipeline_msg_handle_table_mtr_profile_delete(struct pipeline_data *p,
2875         struct pipeline_msg_req *req)
2876 {
2877         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2878         uint32_t table_id = req->id;
2879         uint32_t meter_profile_id =
2880                 req->table_mtr_profile_delete.meter_profile_id;
2881         struct rte_table_action *a = p->table_data[table_id].a;
2882
2883         rsp->status = rte_table_action_meter_profile_delete(a,
2884                 meter_profile_id);
2885
2886         return rsp;
2887 }
2888
2889 static struct pipeline_msg_rsp *
2890 pipeline_msg_handle_table_rule_mtr_read(struct pipeline_data *p,
2891         struct pipeline_msg_req *req)
2892 {
2893         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2894         uint32_t table_id = req->id;
2895         void *data = req->table_rule_mtr_read.data;
2896         uint32_t tc_mask = req->table_rule_mtr_read.tc_mask;
2897         int clear = req->table_rule_mtr_read.clear;
2898         struct rte_table_action *a = p->table_data[table_id].a;
2899
2900         rsp->status = rte_table_action_meter_read(a,
2901                 data,
2902                 tc_mask,
2903                 &rsp->table_rule_mtr_read.stats,
2904                 clear);
2905
2906         return rsp;
2907 }
2908
2909 static struct pipeline_msg_rsp *
2910 pipeline_msg_handle_table_dscp_table_update(struct pipeline_data *p,
2911         struct pipeline_msg_req *req)
2912 {
2913         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2914         uint32_t table_id = req->id;
2915         uint64_t dscp_mask = req->table_dscp_table_update.dscp_mask;
2916         struct rte_table_action_dscp_table *dscp_table =
2917                 &req->table_dscp_table_update.dscp_table;
2918         struct rte_table_action *a = p->table_data[table_id].a;
2919
2920         rsp->status = rte_table_action_dscp_table_update(a,
2921                 dscp_mask,
2922                 dscp_table);
2923
2924         return rsp;
2925 }
2926
2927 static struct pipeline_msg_rsp *
2928 pipeline_msg_handle_table_rule_ttl_read(struct pipeline_data *p,
2929         struct pipeline_msg_req *req)
2930 {
2931         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2932         uint32_t table_id = req->id;
2933         void *data = req->table_rule_ttl_read.data;
2934         int clear = req->table_rule_ttl_read.clear;
2935         struct rte_table_action *a = p->table_data[table_id].a;
2936
2937         rsp->status = rte_table_action_ttl_read(a,
2938                 data,
2939                 &rsp->table_rule_ttl_read.stats,
2940                 clear);
2941
2942         return rsp;
2943 }
2944
2945 static void
2946 pipeline_msg_handle(struct pipeline_data *p)
2947 {
2948         for ( ; ; ) {
2949                 struct pipeline_msg_req *req;
2950                 struct pipeline_msg_rsp *rsp;
2951
2952                 req = pipeline_msg_recv(p->msgq_req);
2953                 if (req == NULL)
2954                         break;
2955
2956                 switch (req->type) {
2957                 case PIPELINE_REQ_PORT_IN_STATS_READ:
2958                         rsp = pipeline_msg_handle_port_in_stats_read(p, req);
2959                         break;
2960
2961                 case PIPELINE_REQ_PORT_IN_ENABLE:
2962                         rsp = pipeline_msg_handle_port_in_enable(p, req);
2963                         break;
2964
2965                 case PIPELINE_REQ_PORT_IN_DISABLE:
2966                         rsp = pipeline_msg_handle_port_in_disable(p, req);
2967                         break;
2968
2969                 case PIPELINE_REQ_PORT_OUT_STATS_READ:
2970                         rsp = pipeline_msg_handle_port_out_stats_read(p, req);
2971                         break;
2972
2973                 case PIPELINE_REQ_TABLE_STATS_READ:
2974                         rsp = pipeline_msg_handle_table_stats_read(p, req);
2975                         break;
2976
2977                 case PIPELINE_REQ_TABLE_RULE_ADD:
2978                         rsp = pipeline_msg_handle_table_rule_add(p, req);
2979                         break;
2980
2981                 case PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT:
2982                         rsp = pipeline_msg_handle_table_rule_add_default(p,     req);
2983                         break;
2984
2985                 case PIPELINE_REQ_TABLE_RULE_ADD_BULK:
2986                         rsp = pipeline_msg_handle_table_rule_add_bulk(p, req);
2987                         break;
2988
2989                 case PIPELINE_REQ_TABLE_RULE_DELETE:
2990                         rsp = pipeline_msg_handle_table_rule_delete(p, req);
2991                         break;
2992
2993                 case PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT:
2994                         rsp = pipeline_msg_handle_table_rule_delete_default(p, req);
2995                         break;
2996
2997                 case PIPELINE_REQ_TABLE_RULE_STATS_READ:
2998                         rsp = pipeline_msg_handle_table_rule_stats_read(p, req);
2999                         break;
3000
3001                 case PIPELINE_REQ_TABLE_MTR_PROFILE_ADD:
3002                         rsp = pipeline_msg_handle_table_mtr_profile_add(p, req);
3003                         break;
3004
3005                 case PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE:
3006                         rsp = pipeline_msg_handle_table_mtr_profile_delete(p, req);
3007                         break;
3008
3009                 case PIPELINE_REQ_TABLE_RULE_MTR_READ:
3010                         rsp = pipeline_msg_handle_table_rule_mtr_read(p, req);
3011                         break;
3012
3013                 case PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE:
3014                         rsp = pipeline_msg_handle_table_dscp_table_update(p, req);
3015                         break;
3016
3017                 case PIPELINE_REQ_TABLE_RULE_TTL_READ:
3018                         rsp = pipeline_msg_handle_table_rule_ttl_read(p, req);
3019                         break;
3020
3021                 default:
3022                         rsp = (struct pipeline_msg_rsp *) req;
3023                         rsp->status = -1;
3024                 }
3025
3026                 pipeline_msg_send(p->msgq_rsp, rsp);
3027         }
3028 }
3029
3030 /**
3031  * Data plane threads: main
3032  */
3033 int
3034 thread_main(void *arg __rte_unused)
3035 {
3036         struct thread_data *t;
3037         uint32_t thread_id, i;
3038
3039         thread_id = rte_lcore_id();
3040         t = &thread_data[thread_id];
3041
3042         /* Dispatch loop */
3043         for (i = 0; ; i++) {
3044                 uint32_t j;
3045
3046                 /* Data Plane */
3047                 for (j = 0; j < t->n_pipelines; j++)
3048                         rte_pipeline_run(t->p[j]);
3049
3050                 /* Control Plane */
3051                 if ((i & 0xF) == 0) {
3052                         uint64_t time = rte_get_tsc_cycles();
3053                         uint64_t time_next_min = UINT64_MAX;
3054
3055                         if (time < t->time_next_min)
3056                                 continue;
3057
3058                         /* Pipeline message queues */
3059                         for (j = 0; j < t->n_pipelines; j++) {
3060                                 struct pipeline_data *p =
3061                                         &t->pipeline_data[j];
3062                                 uint64_t time_next = p->time_next;
3063
3064                                 if (time_next <= time) {
3065                                         pipeline_msg_handle(p);
3066                                         rte_pipeline_flush(p->p);
3067                                         time_next = time + p->timer_period;
3068                                         p->time_next = time_next;
3069                                 }
3070
3071                                 if (time_next < time_next_min)
3072                                         time_next_min = time_next;
3073                         }
3074
3075                         /* Thread message queues */
3076                         {
3077                                 uint64_t time_next = t->time_next;
3078
3079                                 if (time_next <= time) {
3080                                         thread_msg_handle(t);
3081                                         time_next = time + t->timer_period;
3082                                         t->time_next = time_next;
3083                                 }
3084
3085                                 if (time_next < time_next_min)
3086                                         time_next_min = time_next;
3087                         }
3088
3089                         t->time_next_min = time_next_min;
3090                 }
3091         }
3092
3093         return 0;
3094 }