examples/ip_pipeline/thread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdlib.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_cycles.h>
   9 #include <rte_lcore.h>
  10 #include <rte_ring.h>
  11
  12 #include <rte_table_acl.h>
  13 #include <rte_table_array.h>
  14 #include <rte_table_hash.h>
  15 #include <rte_table_lpm.h>
  16 #include <rte_table_lpm_ipv6.h>
  17
  18 #include "common.h"
  19 #include "thread.h"
  20 #include "pipeline.h"
  21
  22 #ifndef THREAD_PIPELINES_MAX
  23 #define THREAD_PIPELINES_MAX                               256
  24 #endif
  25
  26 #ifndef THREAD_MSGQ_SIZE
  27 #define THREAD_MSGQ_SIZE                                   64
  28 #endif
  29
  30 #ifndef THREAD_TIMER_PERIOD_MS
  31 #define THREAD_TIMER_PERIOD_MS                             100
  32 #endif
  33
  34 /**
  35  * Master thead: data plane thread context
  36  */
  37 struct thread {
  38         struct rte_ring *msgq_req;
  39         struct rte_ring *msgq_rsp;
  40
  41         uint32_t enabled;
  42 };
  43
  44 static struct thread thread[RTE_MAX_LCORE];
  45
  46 /**
  47  * Data plane threads: context
  48  */
  49 struct table_data {
  50         struct rte_table_action *a;
  51 };
  52
  53 struct pipeline_data {
  54         struct rte_pipeline *p;
  55         struct table_data table_data[RTE_PIPELINE_TABLE_MAX];
  56         uint32_t n_tables;
  57
  58         struct rte_ring *msgq_req;
  59         struct rte_ring *msgq_rsp;
  60         uint64_t timer_period; /* Measured in CPU cycles. */
  61         uint64_t time_next;
  62
  63         uint8_t buffer[TABLE_RULE_ACTION_SIZE_MAX];
  64 };
  65
  66 struct thread_data {
  67         struct rte_pipeline *p[THREAD_PIPELINES_MAX];
  68         uint32_t n_pipelines;
  69
  70         struct pipeline_data pipeline_data[THREAD_PIPELINES_MAX];
  71         struct rte_ring *msgq_req;
  72         struct rte_ring *msgq_rsp;
  73         uint64_t timer_period; /* Measured in CPU cycles. */
  74         uint64_t time_next;
  75         uint64_t time_next_min;
  76 } __rte_cache_aligned;
  77
  78 static struct thread_data thread_data[RTE_MAX_LCORE];
  79
  80 /**
  81  * Master thread: data plane thread init
  82  */
  83 static void
  84 thread_free(void)
  85 {
  86         uint32_t i;
  87
  88         for (i = 0; i < RTE_MAX_LCORE; i++) {
  89                 struct thread *t = &thread[i];
  90
  91                 if (!rte_lcore_is_enabled(i))
  92                         continue;
  93
  94                 /* MSGQs */
  95                 if (t->msgq_req)
  96                         rte_ring_free(t->msgq_req);
  97
  98                 if (t->msgq_rsp)
  99                         rte_ring_free(t->msgq_rsp);
 100         }
 101 }
 102
 103 int
 104 thread_init(void)
 105 {
 106         uint32_t i;
 107
 108         RTE_LCORE_FOREACH_SLAVE(i) {
 109                 char name[NAME_MAX];
 110                 struct rte_ring *msgq_req, *msgq_rsp;
 111                 struct thread *t = &thread[i];
 112                 struct thread_data *t_data = &thread_data[i];
 113                 uint32_t cpu_id = rte_lcore_to_socket_id(i);
 114
 115                 /* MSGQs */
 116                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-REQ", i);
 117
 118                 msgq_req = rte_ring_create(name,
 119                         THREAD_MSGQ_SIZE,
 120                         cpu_id,
 121                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 122
 123                 if (msgq_req == NULL) {
 124                         thread_free();
 125                         return -1;
 126                 }
 127
 128                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-RSP", i);
 129
 130                 msgq_rsp = rte_ring_create(name,
 131                         THREAD_MSGQ_SIZE,
 132                         cpu_id,
 133                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 134
 135                 if (msgq_rsp == NULL) {
 136                         thread_free();
 137                         return -1;
 138                 }
 139
 140                 /* Master thread records */
 141                 t->msgq_req = msgq_req;
 142                 t->msgq_rsp = msgq_rsp;
 143                 t->enabled = 1;
 144
 145                 /* Data plane thread records */
 146                 t_data->n_pipelines = 0;
 147                 t_data->msgq_req = msgq_req;
 148                 t_data->msgq_rsp = msgq_rsp;
 149                 t_data->timer_period =
 150                         (rte_get_tsc_hz() * THREAD_TIMER_PERIOD_MS) / 1000;
 151                 t_data->time_next = rte_get_tsc_cycles() + t_data->timer_period;
 152                 t_data->time_next_min = t_data->time_next;
 153         }
 154
 155         return 0;
 156 }
 157
 158 /**
 159  * Master thread & data plane threads: message passing
 160  */
 161 enum thread_req_type {
 162         THREAD_REQ_PIPELINE_ENABLE = 0,
 163         THREAD_REQ_PIPELINE_DISABLE,
 164         THREAD_REQ_MAX
 165 };
 166
 167 struct thread_msg_req {
 168         enum thread_req_type type;
 169
 170         union {
 171                 struct {
 172                         struct rte_pipeline *p;
 173                         struct {
 174                                 struct rte_table_action *a;
 175                         } table[RTE_PIPELINE_TABLE_MAX];
 176                         struct rte_ring *msgq_req;
 177                         struct rte_ring *msgq_rsp;
 178                         uint32_t timer_period_ms;
 179                         uint32_t n_tables;
 180                 } pipeline_enable;
 181
 182                 struct {
 183                         struct rte_pipeline *p;
 184                 } pipeline_disable;
 185         };
 186 };
 187
 188 struct thread_msg_rsp {
 189         int status;
 190 };
 191
 192 /**
 193  * Master thread
 194  */
 195 static struct thread_msg_req *
 196 thread_msg_alloc(void)
 197 {
 198         size_t size = RTE_MAX(sizeof(struct thread_msg_req),
 199                 sizeof(struct thread_msg_rsp));
 200
 201         return calloc(1, size);
 202 }
 203
 204 static void
 205 thread_msg_free(struct thread_msg_rsp *rsp)
 206 {
 207         free(rsp);
 208 }
 209
 210 static struct thread_msg_rsp *
 211 thread_msg_send_recv(uint32_t thread_id,
 212         struct thread_msg_req *req)
 213 {
 214         struct thread *t = &thread[thread_id];
 215         struct rte_ring *msgq_req = t->msgq_req;
 216         struct rte_ring *msgq_rsp = t->msgq_rsp;
 217         struct thread_msg_rsp *rsp;
 218         int status;
 219
 220         /* send */
 221         do {
 222                 status = rte_ring_sp_enqueue(msgq_req, req);
 223         } while (status == -ENOBUFS);
 224
 225         /* recv */
 226         do {
 227                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 228         } while (status != 0);
 229
 230         return rsp;
 231 }
 232
 233 int
 234 thread_pipeline_enable(uint32_t thread_id,
 235         const char *pipeline_name)
 236 {
 237         struct pipeline *p = pipeline_find(pipeline_name);
 238         struct thread *t;
 239         struct thread_msg_req *req;
 240         struct thread_msg_rsp *rsp;
 241         uint32_t i;
 242         int status;
 243
 244         /* Check input params */
 245         if ((thread_id >= RTE_MAX_LCORE) ||
 246                 (p == NULL) ||
 247                 (p->n_ports_in == 0) ||
 248                 (p->n_ports_out == 0) ||
 249                 (p->n_tables == 0))
 250                 return -1;
 251
 252         t = &thread[thread_id];
 253         if ((t->enabled == 0) ||
 254                 p->enabled)
 255                 return -1;
 256
 257         /* Allocate request */
 258         req = thread_msg_alloc();
 259         if (req == NULL)
 260                 return -1;
 261
 262         /* Write request */
 263         req->type = THREAD_REQ_PIPELINE_ENABLE;
 264         req->pipeline_enable.p = p->p;
 265         for (i = 0; i < p->n_tables; i++)
 266                 req->pipeline_enable.table[i].a =
 267                         p->table[i].a;
 268         req->pipeline_enable.msgq_req = p->msgq_req;
 269         req->pipeline_enable.msgq_rsp = p->msgq_rsp;
 270         req->pipeline_enable.timer_period_ms = p->timer_period_ms;
 271         req->pipeline_enable.n_tables = p->n_tables;
 272
 273         /* Send request and wait for response */
 274         rsp = thread_msg_send_recv(thread_id, req);
 275         if (rsp == NULL)
 276                 return -1;
 277
 278         /* Read response */
 279         status = rsp->status;
 280
 281         /* Free response */
 282         thread_msg_free(rsp);
 283
 284         /* Request completion */
 285         if (status)
 286                 return status;
 287
 288         p->thread_id = thread_id;
 289         p->enabled = 1;
 290
 291         return 0;
 292 }
 293
 294 int
 295 thread_pipeline_disable(uint32_t thread_id,
 296         const char *pipeline_name)
 297 {
 298         struct pipeline *p = pipeline_find(pipeline_name);
 299         struct thread *t;
 300         struct thread_msg_req *req;
 301         struct thread_msg_rsp *rsp;
 302         int status;
 303
 304         /* Check input params */
 305         if ((thread_id >= RTE_MAX_LCORE) ||
 306                 (p == NULL))
 307                 return -1;
 308
 309         t = &thread[thread_id];
 310         if (t->enabled == 0)
 311                 return -1;
 312
 313         if (p->enabled == 0)
 314                 return 0;
 315
 316         if (p->thread_id != thread_id)
 317                 return -1;
 318
 319         /* Allocate request */
 320         req = thread_msg_alloc();
 321         if (req == NULL)
 322                 return -1;
 323
 324         /* Write request */
 325         req->type = THREAD_REQ_PIPELINE_DISABLE;
 326         req->pipeline_disable.p = p->p;
 327
 328         /* Send request and wait for response */
 329         rsp = thread_msg_send_recv(thread_id, req);
 330         if (rsp == NULL)
 331                 return -1;
 332
 333         /* Read response */
 334         status = rsp->status;
 335
 336         /* Free response */
 337         thread_msg_free(rsp);
 338
 339         /* Request completion */
 340         if (status)
 341                 return status;
 342
 343         p->enabled = 0;
 344
 345         return 0;
 346 }
 347
 348 /**
 349  * Data plane threads: message handling
 350  */
 351 static inline struct thread_msg_req *
 352 thread_msg_recv(struct rte_ring *msgq_req)
 353 {
 354         struct thread_msg_req *req;
 355
 356         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
 357
 358         if (status != 0)
 359                 return NULL;
 360
 361         return req;
 362 }
 363
 364 static inline void
 365 thread_msg_send(struct rte_ring *msgq_rsp,
 366         struct thread_msg_rsp *rsp)
 367 {
 368         int status;
 369
 370         do {
 371                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
 372         } while (status == -ENOBUFS);
 373 }
 374
 375 static struct thread_msg_rsp *
 376 thread_msg_handle_pipeline_enable(struct thread_data *t,
 377         struct thread_msg_req *req)
 378 {
 379         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 380         struct pipeline_data *p = &t->pipeline_data[t->n_pipelines];
 381         uint32_t i;
 382
 383         /* Request */
 384         if (t->n_pipelines >= THREAD_PIPELINES_MAX) {
 385                 rsp->status = -1;
 386                 return rsp;
 387         }
 388
 389         t->p[t->n_pipelines] = req->pipeline_enable.p;
 390
 391         p->p = req->pipeline_enable.p;
 392         for (i = 0; i < req->pipeline_enable.n_tables; i++)
 393                 p->table_data[i].a =
 394                         req->pipeline_enable.table[i].a;
 395
 396         p->n_tables = req->pipeline_enable.n_tables;
 397
 398         p->msgq_req = req->pipeline_enable.msgq_req;
 399         p->msgq_rsp = req->pipeline_enable.msgq_rsp;
 400         p->timer_period =
 401                 (rte_get_tsc_hz() * req->pipeline_enable.timer_period_ms) / 1000;
 402         p->time_next = rte_get_tsc_cycles() + p->timer_period;
 403
 404         t->n_pipelines++;
 405
 406         /* Response */
 407         rsp->status = 0;
 408         return rsp;
 409 }
 410
 411 static struct thread_msg_rsp *
 412 thread_msg_handle_pipeline_disable(struct thread_data *t,
 413         struct thread_msg_req *req)
 414 {
 415         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 416         uint32_t n_pipelines = t->n_pipelines;
 417         struct rte_pipeline *pipeline = req->pipeline_disable.p;
 418         uint32_t i;
 419
 420         /* find pipeline */
 421         for (i = 0; i < n_pipelines; i++) {
 422                 struct pipeline_data *p = &t->pipeline_data[i];
 423
 424                 if (p->p != pipeline)
 425                         continue;
 426
 427                 if (i < n_pipelines - 1) {
 428                         struct rte_pipeline *pipeline_last =
 429                                 t->p[n_pipelines - 1];
 430                         struct pipeline_data *p_last =
 431                                 &t->pipeline_data[n_pipelines - 1];
 432
 433                         t->p[i] = pipeline_last;
 434                         memcpy(p, p_last, sizeof(*p));
 435                 }
 436
 437                 t->n_pipelines--;
 438
 439                 rsp->status = 0;
 440                 return rsp;
 441         }
 442
 443         /* should not get here */
 444         rsp->status = 0;
 445         return rsp;
 446 }
 447
 448 static void
 449 thread_msg_handle(struct thread_data *t)
 450 {
 451         for ( ; ; ) {
 452                 struct thread_msg_req *req;
 453                 struct thread_msg_rsp *rsp;
 454
 455                 req = thread_msg_recv(t->msgq_req);
 456                 if (req == NULL)
 457                         break;
 458
 459                 switch (req->type) {
 460                 case THREAD_REQ_PIPELINE_ENABLE:
 461                         rsp = thread_msg_handle_pipeline_enable(t, req);
 462                         break;
 463
 464                 case THREAD_REQ_PIPELINE_DISABLE:
 465                         rsp = thread_msg_handle_pipeline_disable(t, req);
 466                         break;
 467
 468                 default:
 469                         rsp = (struct thread_msg_rsp *) req;
 470                         rsp->status = -1;
 471                 }
 472
 473                 thread_msg_send(t->msgq_rsp, rsp);
 474         }
 475 }
 476
 477 /**
 478  * Master thread & data plane threads: message passing
 479  */
 480 enum pipeline_req_type {
 481         /* Port IN */
 482         PIPELINE_REQ_PORT_IN_STATS_READ,
 483         PIPELINE_REQ_PORT_IN_ENABLE,
 484         PIPELINE_REQ_PORT_IN_DISABLE,
 485
 486         /* Port OUT */
 487         PIPELINE_REQ_PORT_OUT_STATS_READ,
 488
 489         /* Table */
 490         PIPELINE_REQ_TABLE_STATS_READ,
 491         PIPELINE_REQ_TABLE_RULE_ADD,
 492         PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT,
 493         PIPELINE_REQ_TABLE_RULE_DELETE,
 494         PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT,
 495
 496         PIPELINE_REQ_MAX
 497 };
 498
 499 struct pipeline_msg_req_port_in_stats_read {
 500         int clear;
 501 };
 502
 503 struct pipeline_msg_req_port_out_stats_read {
 504         int clear;
 505 };
 506
 507 struct pipeline_msg_req_table_stats_read {
 508         int clear;
 509 };
 510
 511 struct pipeline_msg_req_table_rule_add {
 512         struct table_rule_match match;
 513         struct table_rule_action action;
 514 };
 515
 516 struct pipeline_msg_req_table_rule_add_default {
 517         struct table_rule_action action;
 518 };
 519
 520 struct pipeline_msg_req_table_rule_delete {
 521         struct table_rule_match match;
 522 };
 523
 524 struct pipeline_msg_req {
 525         enum pipeline_req_type type;
 526         uint32_t id; /* Port IN, port OUT or table ID */
 527
 528         RTE_STD_C11
 529         union {
 530                 struct pipeline_msg_req_port_in_stats_read port_in_stats_read;
 531                 struct pipeline_msg_req_port_out_stats_read port_out_stats_read;
 532                 struct pipeline_msg_req_table_stats_read table_stats_read;
 533                 struct pipeline_msg_req_table_rule_add table_rule_add;
 534                 struct pipeline_msg_req_table_rule_add_default table_rule_add_default;
 535                 struct pipeline_msg_req_table_rule_delete table_rule_delete;
 536         };
 537 };
 538
 539 struct pipeline_msg_rsp_port_in_stats_read {
 540         struct rte_pipeline_port_in_stats stats;
 541 };
 542
 543 struct pipeline_msg_rsp_port_out_stats_read {
 544         struct rte_pipeline_port_out_stats stats;
 545 };
 546
 547 struct pipeline_msg_rsp_table_stats_read {
 548         struct rte_pipeline_table_stats stats;
 549 };
 550
 551 struct pipeline_msg_rsp_table_rule_add {
 552         void *data;
 553 };
 554
 555 struct pipeline_msg_rsp_table_rule_add_default {
 556         void *data;
 557 };
 558
 559 struct pipeline_msg_rsp {
 560         int status;
 561
 562         RTE_STD_C11
 563         union {
 564                 struct pipeline_msg_rsp_port_in_stats_read port_in_stats_read;
 565                 struct pipeline_msg_rsp_port_out_stats_read port_out_stats_read;
 566                 struct pipeline_msg_rsp_table_stats_read table_stats_read;
 567                 struct pipeline_msg_rsp_table_rule_add table_rule_add;
 568                 struct pipeline_msg_rsp_table_rule_add_default table_rule_add_default;
 569         };
 570 };
 571
 572 /**
 573  * Master thread
 574  */
 575 static struct pipeline_msg_req *
 576 pipeline_msg_alloc(void)
 577 {
 578         size_t size = RTE_MAX(sizeof(struct pipeline_msg_req),
 579                 sizeof(struct pipeline_msg_rsp));
 580
 581         return calloc(1, size);
 582 }
 583
 584 static void
 585 pipeline_msg_free(struct pipeline_msg_rsp *rsp)
 586 {
 587         free(rsp);
 588 }
 589
 590 static struct pipeline_msg_rsp *
 591 pipeline_msg_send_recv(struct pipeline *p,
 592         struct pipeline_msg_req *req)
 593 {
 594         struct rte_ring *msgq_req = p->msgq_req;
 595         struct rte_ring *msgq_rsp = p->msgq_rsp;
 596         struct pipeline_msg_rsp *rsp;
 597         int status;
 598
 599         /* send */
 600         do {
 601                 status = rte_ring_sp_enqueue(msgq_req, req);
 602         } while (status == -ENOBUFS);
 603
 604         /* recv */
 605         do {
 606                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 607         } while (status != 0);
 608
 609         return rsp;
 610 }
 611
 612 int
 613 pipeline_port_in_stats_read(const char *pipeline_name,
 614         uint32_t port_id,
 615         struct rte_pipeline_port_in_stats *stats,
 616         int clear)
 617 {
 618         struct pipeline *p;
 619         struct pipeline_msg_req *req;
 620         struct pipeline_msg_rsp *rsp;
 621         int status;
 622
 623         /* Check input params */
 624         if ((pipeline_name == NULL) ||
 625                 (stats == NULL))
 626                 return -1;
 627
 628         p = pipeline_find(pipeline_name);
 629         if ((p == NULL) ||
 630                 (p->enabled == 0) ||
 631                 (port_id >= p->n_ports_in))
 632                 return -1;
 633
 634         /* Allocate request */
 635         req = pipeline_msg_alloc();
 636         if (req == NULL)
 637                 return -1;
 638
 639         /* Write request */
 640         req->type = PIPELINE_REQ_PORT_IN_STATS_READ;
 641         req->id = port_id;
 642         req->port_in_stats_read.clear = clear;
 643
 644         /* Send request and wait for response */
 645         rsp = pipeline_msg_send_recv(p, req);
 646         if (rsp == NULL)
 647                 return -1;
 648
 649         /* Read response */
 650         status = rsp->status;
 651         if (status)
 652                 memcpy(stats, &rsp->port_in_stats_read.stats, sizeof(*stats));
 653
 654         /* Free response */
 655         pipeline_msg_free(rsp);
 656
 657         return status;
 658 }
 659
 660 int
 661 pipeline_port_in_enable(const char *pipeline_name,
 662         uint32_t port_id)
 663 {
 664         struct pipeline *p;
 665         struct pipeline_msg_req *req;
 666         struct pipeline_msg_rsp *rsp;
 667         int status;
 668
 669         /* Check input params */
 670         if (pipeline_name == NULL)
 671                 return -1;
 672
 673         p = pipeline_find(pipeline_name);
 674         if ((p == NULL) ||
 675                 (p->enabled == 0) ||
 676                 (port_id >= p->n_ports_in))
 677                 return -1;
 678
 679         /* Allocate request */
 680         req = pipeline_msg_alloc();
 681         if (req == NULL)
 682                 return -1;
 683
 684         /* Write request */
 685         req->type = PIPELINE_REQ_PORT_IN_ENABLE;
 686         req->id = port_id;
 687
 688         /* Send request and wait for response */
 689         rsp = pipeline_msg_send_recv(p, req);
 690         if (rsp == NULL)
 691                 return -1;
 692
 693         /* Read response */
 694         status = rsp->status;
 695
 696         /* Free response */
 697         pipeline_msg_free(rsp);
 698
 699         return status;
 700 }
 701
 702 int
 703 pipeline_port_in_disable(const char *pipeline_name,
 704         uint32_t port_id)
 705 {
 706         struct pipeline *p;
 707         struct pipeline_msg_req *req;
 708         struct pipeline_msg_rsp *rsp;
 709         int status;
 710
 711         /* Check input params */
 712         if (pipeline_name == NULL)
 713                 return -1;
 714
 715         p = pipeline_find(pipeline_name);
 716         if ((p == NULL) ||
 717                 (p->enabled == 0) ||
 718                 (port_id >= p->n_ports_in))
 719                 return -1;
 720
 721         /* Allocate request */
 722         req = pipeline_msg_alloc();
 723         if (req == NULL)
 724                 return -1;
 725
 726         /* Write request */
 727         req->type = PIPELINE_REQ_PORT_IN_DISABLE;
 728         req->id = port_id;
 729
 730         /* Send request and wait for response */
 731         rsp = pipeline_msg_send_recv(p, req);
 732         if (rsp == NULL)
 733                 return -1;
 734
 735         /* Read response */
 736         status = rsp->status;
 737
 738         /* Free response */
 739         pipeline_msg_free(rsp);
 740
 741         return status;
 742 }
 743
 744 int
 745 pipeline_port_out_stats_read(const char *pipeline_name,
 746         uint32_t port_id,
 747         struct rte_pipeline_port_out_stats *stats,
 748         int clear)
 749 {
 750         struct pipeline *p;
 751         struct pipeline_msg_req *req;
 752         struct pipeline_msg_rsp *rsp;
 753         int status;
 754
 755         /* Check input params */
 756         if ((pipeline_name == NULL) ||
 757                 (stats == NULL))
 758                 return -1;
 759
 760         p = pipeline_find(pipeline_name);
 761         if ((p == NULL) ||
 762                 (p->enabled == 0) ||
 763                 (port_id >= p->n_ports_out))
 764                 return -1;
 765
 766         /* Allocate request */
 767         req = pipeline_msg_alloc();
 768         if (req == NULL)
 769                 return -1;
 770
 771         /* Write request */
 772         req->type = PIPELINE_REQ_PORT_OUT_STATS_READ;
 773         req->id = port_id;
 774         req->port_out_stats_read.clear = clear;
 775
 776         /* Send request and wait for response */
 777         rsp = pipeline_msg_send_recv(p, req);
 778         if (rsp == NULL)
 779                 return -1;
 780
 781         /* Read response */
 782         status = rsp->status;
 783         if (status)
 784                 memcpy(stats, &rsp->port_out_stats_read.stats, sizeof(*stats));
 785
 786         /* Free response */
 787         pipeline_msg_free(rsp);
 788
 789         return status;
 790 }
 791
 792 int
 793 pipeline_table_stats_read(const char *pipeline_name,
 794         uint32_t table_id,
 795         struct rte_pipeline_table_stats *stats,
 796         int clear)
 797 {
 798         struct pipeline *p;
 799         struct pipeline_msg_req *req;
 800         struct pipeline_msg_rsp *rsp;
 801         int status;
 802
 803         /* Check input params */
 804         if ((pipeline_name == NULL) ||
 805                 (stats == NULL))
 806                 return -1;
 807
 808         p = pipeline_find(pipeline_name);
 809         if ((p == NULL) ||
 810                 (p->enabled == 0) ||
 811                 (table_id >= p->n_tables))
 812                 return -1;
 813
 814         /* Allocate request */
 815         req = pipeline_msg_alloc();
 816         if (req == NULL)
 817                 return -1;
 818
 819         /* Write request */
 820         req->type = PIPELINE_REQ_TABLE_STATS_READ;
 821         req->id = table_id;
 822         req->table_stats_read.clear = clear;
 823
 824         /* Send request and wait for response */
 825         rsp = pipeline_msg_send_recv(p, req);
 826         if (rsp == NULL)
 827                 return -1;
 828
 829         /* Read response */
 830         status = rsp->status;
 831         if (status)
 832                 memcpy(stats, &rsp->table_stats_read.stats, sizeof(*stats));
 833
 834         /* Free response */
 835         pipeline_msg_free(rsp);
 836
 837         return status;
 838 }
 839
 840 static int
 841 match_check(struct table_rule_match *match,
 842         struct pipeline *p,
 843         uint32_t table_id)
 844 {
 845         struct table *table;
 846
 847         if ((match == NULL) ||
 848                 (p == NULL) ||
 849                 (table_id >= p->n_tables))
 850                 return -1;
 851
 852         table = &p->table[table_id];
 853         if (match->match_type != table->params.match_type)
 854                 return -1;
 855
 856         switch (match->match_type) {
 857         case TABLE_ACL:
 858         {
 859                 struct table_acl_params *t = &table->params.match.acl;
 860                 struct table_rule_match_acl *r = &match->match.acl;
 861
 862                 if ((r->ip_version && (t->ip_version == 0)) ||
 863                         ((r->ip_version == 0) && t->ip_version))
 864                         return -1;
 865
 866                 if (r->ip_version) {
 867                         if ((r->sa_depth > 32) ||
 868                                 (r->da_depth > 32))
 869                                 return -1;
 870                 } else {
 871                         if ((r->sa_depth > 128) ||
 872                                 (r->da_depth > 128))
 873                                 return -1;
 874                 }
 875                 return 0;
 876         }
 877
 878         case TABLE_ARRAY:
 879                 return 0;
 880
 881         case TABLE_HASH:
 882                 return 0;
 883
 884         case TABLE_LPM:
 885         {
 886                 struct table_lpm_params *t = &table->params.match.lpm;
 887                 struct table_rule_match_lpm *r = &match->match.lpm;
 888
 889                 if ((r->ip_version && (t->key_size != 4)) ||
 890                         ((r->ip_version == 0) && (t->key_size != 16)))
 891                         return -1;
 892
 893                 if (r->ip_version) {
 894                         if (r->depth > 32)
 895                                 return -1;
 896                 } else {
 897                         if (r->depth > 128)
 898                                 return -1;
 899                 }
 900                 return 0;
 901         }
 902
 903         case TABLE_STUB:
 904                 return -1;
 905
 906         default:
 907                 return -1;
 908         }
 909 }
 910
 911 static int
 912 action_check(struct table_rule_action *action,
 913         struct pipeline *p,
 914         uint32_t table_id)
 915 {
 916         struct table_action_profile *ap;
 917
 918         if ((action == NULL) ||
 919                 (p == NULL) ||
 920                 (table_id >= p->n_tables))
 921                 return -1;
 922
 923         ap = p->table[table_id].ap;
 924         if (action->action_mask != ap->params.action_mask)
 925                 return -1;
 926
 927         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
 928                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
 929                         (action->fwd.id >= p->n_ports_out))
 930                         return -1;
 931
 932                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
 933                         (action->fwd.id >= p->n_tables))
 934                         return -1;
 935         }
 936
 937         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
 938                 uint32_t tc_mask0 = (1 << ap->params.mtr.n_tc) - 1;
 939                 uint32_t tc_mask1 = action->mtr.tc_mask;
 940
 941                 if (tc_mask1 != tc_mask0)
 942                         return -1;
 943         }
 944
 945         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
 946                 uint32_t n_subports_per_port =
 947                         ap->params.tm.n_subports_per_port;
 948                 uint32_t n_pipes_per_subport =
 949                         ap->params.tm.n_pipes_per_subport;
 950                 uint32_t subport_id = action->tm.subport_id;
 951                 uint32_t pipe_id = action->tm.pipe_id;
 952
 953                 if ((subport_id >= n_subports_per_port) ||
 954                         (pipe_id >= n_pipes_per_subport))
 955                         return -1;
 956         }
 957
 958         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
 959                 uint64_t encap_mask = ap->params.encap.encap_mask;
 960                 enum rte_table_action_encap_type type = action->encap.type;
 961
 962                 if ((encap_mask & (1LLU << type)) == 0)
 963                         return -1;
 964         }
 965
 966         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
 967                 int ip_version0 = ap->params.common.ip_version;
 968                 int ip_version1 = action->nat.ip_version;
 969
 970                 if ((ip_version1 && (ip_version0 == 0)) ||
 971                         ((ip_version1 == 0) && ip_version0))
 972                         return -1;
 973         }
 974
 975         return 0;
 976 }
 977
 978 static int
 979 action_default_check(struct table_rule_action *action,
 980         struct pipeline *p,
 981         uint32_t table_id)
 982 {
 983         if ((action == NULL) ||
 984                 (action->action_mask != (1LLU << RTE_TABLE_ACTION_FWD)) ||
 985                 (p == NULL) ||
 986                 (table_id >= p->n_tables))
 987                 return -1;
 988
 989         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
 990                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
 991                         (action->fwd.id >= p->n_ports_out))
 992                         return -1;
 993
 994                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
 995                         (action->fwd.id >= p->n_tables))
 996                         return -1;
 997         }
 998
 999         return 0;
1000 }
1001
1002 int
1003 pipeline_table_rule_add(const char *pipeline_name,
1004         uint32_t table_id,
1005         struct table_rule_match *match,
1006         struct table_rule_action *action,
1007         void **data)
1008 {
1009         struct pipeline *p;
1010         struct pipeline_msg_req *req;
1011         struct pipeline_msg_rsp *rsp;
1012         int status;
1013
1014         /* Check input params */
1015         if ((pipeline_name == NULL) ||
1016                 (match == NULL) ||
1017                 (action == NULL) ||
1018                 (data == NULL))
1019                 return -1;
1020
1021         p = pipeline_find(pipeline_name);
1022         if ((p == NULL) ||
1023                 (p->enabled == 0) ||
1024                 (table_id >= p->n_tables) ||
1025                 match_check(match, p, table_id) ||
1026                 action_check(action, p, table_id))
1027                 return -1;
1028
1029         /* Allocate request */
1030         req = pipeline_msg_alloc();
1031         if (req == NULL)
1032                 return -1;
1033
1034         /* Write request */
1035         req->type = PIPELINE_REQ_TABLE_RULE_ADD;
1036         req->id = table_id;
1037         memcpy(&req->table_rule_add.match, match, sizeof(*match));
1038         memcpy(&req->table_rule_add.action, action, sizeof(*action));
1039
1040         /* Send request and wait for response */
1041         rsp = pipeline_msg_send_recv(p, req);
1042         if (rsp == NULL)
1043                 return -1;
1044
1045         /* Read response */
1046         status = rsp->status;
1047         if (status == 0)
1048                 *data = rsp->table_rule_add.data;
1049
1050         /* Free response */
1051         pipeline_msg_free(rsp);
1052
1053         return status;
1054 }
1055
1056 int
1057 pipeline_table_rule_add_default(const char *pipeline_name,
1058         uint32_t table_id,
1059         struct table_rule_action *action,
1060         void **data)
1061 {
1062         struct pipeline *p;
1063         struct pipeline_msg_req *req;
1064         struct pipeline_msg_rsp *rsp;
1065         int status;
1066
1067         /* Check input params */
1068         if ((pipeline_name == NULL) ||
1069                 (action == NULL) ||
1070                 (data == NULL))
1071                 return -1;
1072
1073         p = pipeline_find(pipeline_name);
1074         if ((p == NULL) ||
1075                 (p->enabled == 0) ||
1076                 (table_id >= p->n_tables) ||
1077                 action_default_check(action, p, table_id))
1078                 return -1;
1079
1080         /* Allocate request */
1081         req = pipeline_msg_alloc();
1082         if (req == NULL)
1083                 return -1;
1084
1085         /* Write request */
1086         req->type = PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT;
1087         req->id = table_id;
1088         memcpy(&req->table_rule_add_default.action, action, sizeof(*action));
1089
1090         /* Send request and wait for response */
1091         rsp = pipeline_msg_send_recv(p, req);
1092         if (rsp == NULL)
1093                 return -1;
1094
1095         /* Read response */
1096         status = rsp->status;
1097         if (status == 0)
1098                 *data = rsp->table_rule_add_default.data;
1099
1100         /* Free response */
1101         pipeline_msg_free(rsp);
1102
1103         return status;
1104 }
1105
1106 int
1107 pipeline_table_rule_delete(const char *pipeline_name,
1108         uint32_t table_id,
1109         struct table_rule_match *match)
1110 {
1111         struct pipeline *p;
1112         struct pipeline_msg_req *req;
1113         struct pipeline_msg_rsp *rsp;
1114         int status;
1115
1116         /* Check input params */
1117         if ((pipeline_name == NULL) ||
1118                 (match == NULL))
1119                 return -1;
1120
1121         p = pipeline_find(pipeline_name);
1122         if ((p == NULL) ||
1123                 (p->enabled == 0) ||
1124                 (table_id >= p->n_tables) ||
1125                 match_check(match, p, table_id))
1126                 return -1;
1127
1128         /* Allocate request */
1129         req = pipeline_msg_alloc();
1130         if (req == NULL)
1131                 return -1;
1132
1133         /* Write request */
1134         req->type = PIPELINE_REQ_TABLE_RULE_DELETE;
1135         req->id = table_id;
1136         memcpy(&req->table_rule_delete.match, match, sizeof(*match));
1137
1138         /* Send request and wait for response */
1139         rsp = pipeline_msg_send_recv(p, req);
1140         if (rsp == NULL)
1141                 return -1;
1142
1143         /* Read response */
1144         status = rsp->status;
1145
1146         /* Free response */
1147         pipeline_msg_free(rsp);
1148
1149         return status;
1150 }
1151
1152 int
1153 pipeline_table_rule_delete_default(const char *pipeline_name,
1154         uint32_t table_id)
1155 {
1156         struct pipeline *p;
1157         struct pipeline_msg_req *req;
1158         struct pipeline_msg_rsp *rsp;
1159         int status;
1160
1161         /* Check input params */
1162         if (pipeline_name == NULL)
1163                 return -1;
1164
1165         p = pipeline_find(pipeline_name);
1166         if ((p == NULL) ||
1167                 (p->enabled == 0) ||
1168                 (table_id >= p->n_tables))
1169                 return -1;
1170
1171         /* Allocate request */
1172         req = pipeline_msg_alloc();
1173         if (req == NULL)
1174                 return -1;
1175
1176         /* Write request */
1177         req->type = PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT;
1178         req->id = table_id;
1179
1180         /* Send request and wait for response */
1181         rsp = pipeline_msg_send_recv(p, req);
1182         if (rsp == NULL)
1183                 return -1;
1184
1185         /* Read response */
1186         status = rsp->status;
1187
1188         /* Free response */
1189         pipeline_msg_free(rsp);
1190
1191         return status;
1192 }
1193
1194 /**
1195  * Data plane threads: message handling
1196  */
1197 static inline struct pipeline_msg_req *
1198 pipeline_msg_recv(struct rte_ring *msgq_req)
1199 {
1200         struct pipeline_msg_req *req;
1201
1202         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
1203
1204         if (status != 0)
1205                 return NULL;
1206
1207         return req;
1208 }
1209
1210 static inline void
1211 pipeline_msg_send(struct rte_ring *msgq_rsp,
1212         struct pipeline_msg_rsp *rsp)
1213 {
1214         int status;
1215
1216         do {
1217                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
1218         } while (status == -ENOBUFS);
1219 }
1220
1221 static struct pipeline_msg_rsp *
1222 pipeline_msg_handle_port_in_stats_read(struct pipeline_data *p,
1223         struct pipeline_msg_req *req)
1224 {
1225         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1226         uint32_t port_id = req->id;
1227         int clear = req->port_in_stats_read.clear;
1228
1229         rsp->status = rte_pipeline_port_in_stats_read(p->p,
1230                 port_id,
1231                 &rsp->port_in_stats_read.stats,
1232                 clear);
1233
1234         return rsp;
1235 }
1236
1237 static struct pipeline_msg_rsp *
1238 pipeline_msg_handle_port_in_enable(struct pipeline_data *p,
1239         struct pipeline_msg_req *req)
1240 {
1241         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1242         uint32_t port_id = req->id;
1243
1244         rsp->status = rte_pipeline_port_in_enable(p->p,
1245                 port_id);
1246
1247         return rsp;
1248 }
1249
1250 static struct pipeline_msg_rsp *
1251 pipeline_msg_handle_port_in_disable(struct pipeline_data *p,
1252         struct pipeline_msg_req *req)
1253 {
1254         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1255         uint32_t port_id = req->id;
1256
1257         rsp->status = rte_pipeline_port_in_disable(p->p,
1258                 port_id);
1259
1260         return rsp;
1261 }
1262
1263 static struct pipeline_msg_rsp *
1264 pipeline_msg_handle_port_out_stats_read(struct pipeline_data *p,
1265         struct pipeline_msg_req *req)
1266 {
1267         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1268         uint32_t port_id = req->id;
1269         int clear = req->port_out_stats_read.clear;
1270
1271         rsp->status = rte_pipeline_port_out_stats_read(p->p,
1272                 port_id,
1273                 &rsp->port_out_stats_read.stats,
1274                 clear);
1275
1276         return rsp;
1277 }
1278
1279 static struct pipeline_msg_rsp *
1280 pipeline_msg_handle_table_stats_read(struct pipeline_data *p,
1281         struct pipeline_msg_req *req)
1282 {
1283         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1284         uint32_t port_id = req->id;
1285         int clear = req->table_stats_read.clear;
1286
1287         rsp->status = rte_pipeline_table_stats_read(p->p,
1288                 port_id,
1289                 &rsp->table_stats_read.stats,
1290                 clear);
1291
1292         return rsp;
1293 }
1294
1295 union table_rule_match_low_level {
1296         struct rte_table_acl_rule_add_params acl_add;
1297         struct rte_table_acl_rule_delete_params acl_delete;
1298         struct rte_table_array_key array;
1299         uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
1300         struct rte_table_lpm_key lpm_ipv4;
1301         struct rte_table_lpm_ipv6_key lpm_ipv6;
1302 };
1303
1304 static int
1305 match_convert_ipv6_depth(uint32_t depth, uint32_t *depth32)
1306 {
1307         if (depth > 128)
1308                 return -1;
1309
1310         switch (depth / 32) {
1311         case 0:
1312                 depth32[0] = depth;
1313                 depth32[1] = 0;
1314                 depth32[2] = 0;
1315                 depth32[3] = 0;
1316                 return 0;
1317
1318         case 1:
1319                 depth32[0] = 32;
1320                 depth32[1] = depth - 32;
1321                 depth32[2] = 0;
1322                 depth32[3] = 0;
1323                 return 0;
1324
1325         case 2:
1326                 depth32[0] = 32;
1327                 depth32[1] = 32;
1328                 depth32[2] = depth - 64;
1329                 depth32[3] = 0;
1330                 return 0;
1331
1332         case 3:
1333                 depth32[0] = 32;
1334                 depth32[1] = 32;
1335                 depth32[2] = 32;
1336                 depth32[3] = depth - 96;
1337                 return 0;
1338
1339         case 4:
1340                 depth32[0] = 32;
1341                 depth32[1] = 32;
1342                 depth32[2] = 32;
1343                 depth32[3] = 32;
1344                 return 0;
1345
1346         default:
1347                 return -1;
1348         }
1349 }
1350
1351 static int
1352 match_convert(struct table_rule_match *mh,
1353         union table_rule_match_low_level *ml,
1354         int add)
1355 {
1356         memset(ml, 0, sizeof(*ml));
1357
1358         switch (mh->match_type) {
1359         case TABLE_ACL:
1360                 if (mh->match.acl.ip_version)
1361                         if (add) {
1362                                 ml->acl_add.field_value[0].value.u8 =
1363                                         mh->match.acl.proto;
1364                                 ml->acl_add.field_value[0].mask_range.u8 =
1365                                         mh->match.acl.proto_mask;
1366
1367                                 ml->acl_add.field_value[1].value.u32 =
1368                                         mh->match.acl.ipv4.sa;
1369                                 ml->acl_add.field_value[1].mask_range.u32 =
1370                                         mh->match.acl.sa_depth;
1371
1372                                 ml->acl_add.field_value[2].value.u32 =
1373                                         mh->match.acl.ipv4.da;
1374                                 ml->acl_add.field_value[2].mask_range.u32 =
1375                                         mh->match.acl.da_depth;
1376
1377                                 ml->acl_add.field_value[3].value.u16 =
1378                                         mh->match.acl.sp0;
1379                                 ml->acl_add.field_value[3].mask_range.u16 =
1380                                         mh->match.acl.sp1;
1381
1382                                 ml->acl_add.field_value[4].value.u16 =
1383                                         mh->match.acl.dp0;
1384                                 ml->acl_add.field_value[4].mask_range.u16 =
1385                                         mh->match.acl.dp1;
1386
1387                                 ml->acl_add.priority =
1388                                         (int32_t) mh->match.acl.priority;
1389                         } else {
1390                                 ml->acl_delete.field_value[0].value.u8 =
1391                                         mh->match.acl.proto;
1392                                 ml->acl_delete.field_value[0].mask_range.u8 =
1393                                         mh->match.acl.proto_mask;
1394
1395                                 ml->acl_delete.field_value[1].value.u32 =
1396                                         mh->match.acl.ipv4.sa;
1397                                 ml->acl_delete.field_value[1].mask_range.u32 =
1398                                         mh->match.acl.sa_depth;
1399
1400                                 ml->acl_delete.field_value[2].value.u32 =
1401                                         mh->match.acl.ipv4.da;
1402                                 ml->acl_delete.field_value[2].mask_range.u32 =
1403                                         mh->match.acl.da_depth;
1404
1405                                 ml->acl_delete.field_value[3].value.u16 =
1406                                         mh->match.acl.sp0;
1407                                 ml->acl_delete.field_value[3].mask_range.u16 =
1408                                         mh->match.acl.sp1;
1409
1410                                 ml->acl_delete.field_value[4].value.u16 =
1411                                         mh->match.acl.dp0;
1412                                 ml->acl_delete.field_value[4].mask_range.u16 =
1413                                         mh->match.acl.dp1;
1414                         }
1415                 else
1416                         if (add) {
1417                                 uint32_t *sa32 =
1418                                         (uint32_t *) mh->match.acl.ipv6.sa;
1419                                 uint32_t *da32 =
1420                                         (uint32_t *) mh->match.acl.ipv6.da;
1421                                 uint32_t sa32_depth[4], da32_depth[4];
1422                                 int status;
1423
1424                                 status = match_convert_ipv6_depth(
1425                                         mh->match.acl.sa_depth,
1426                                         sa32_depth);
1427                                 if (status)
1428                                         return status;
1429
1430                                 status = match_convert_ipv6_depth(
1431                                         mh->match.acl.da_depth,
1432                                         da32_depth);
1433                                 if (status)
1434                                         return status;
1435
1436                                 ml->acl_add.field_value[0].value.u8 =
1437                                         mh->match.acl.proto;
1438                                 ml->acl_add.field_value[0].mask_range.u8 =
1439                                         mh->match.acl.proto_mask;
1440
1441                                 ml->acl_add.field_value[1].value.u32 = sa32[0];
1442                                 ml->acl_add.field_value[1].mask_range.u32 =
1443                                         sa32_depth[0];
1444                                 ml->acl_add.field_value[2].value.u32 = sa32[1];
1445                                 ml->acl_add.field_value[2].mask_range.u32 =
1446                                         sa32_depth[1];
1447                                 ml->acl_add.field_value[3].value.u32 = sa32[2];
1448                                 ml->acl_add.field_value[3].mask_range.u32 =
1449                                         sa32_depth[2];
1450                                 ml->acl_add.field_value[4].value.u32 = sa32[3];
1451                                 ml->acl_add.field_value[4].mask_range.u32 =
1452                                         sa32_depth[3];
1453
1454                                 ml->acl_add.field_value[5].value.u32 = da32[0];
1455                                 ml->acl_add.field_value[5].mask_range.u32 =
1456                                         da32_depth[0];
1457                                 ml->acl_add.field_value[6].value.u32 = da32[1];
1458                                 ml->acl_add.field_value[6].mask_range.u32 =
1459                                         da32_depth[1];
1460                                 ml->acl_add.field_value[7].value.u32 = da32[2];
1461                                 ml->acl_add.field_value[7].mask_range.u32 =
1462                                         da32_depth[2];
1463                                 ml->acl_add.field_value[8].value.u32 = da32[3];
1464                                 ml->acl_add.field_value[8].mask_range.u32 =
1465                                         da32_depth[3];
1466
1467                                 ml->acl_add.field_value[9].value.u16 =
1468                                         mh->match.acl.sp0;
1469                                 ml->acl_add.field_value[9].mask_range.u16 =
1470                                         mh->match.acl.sp1;
1471
1472                                 ml->acl_add.field_value[10].value.u16 =
1473                                         mh->match.acl.dp0;
1474                                 ml->acl_add.field_value[10].mask_range.u16 =
1475                                         mh->match.acl.dp1;
1476
1477                                 ml->acl_add.priority =
1478                                         (int32_t) mh->match.acl.priority;
1479                         } else {
1480                                 uint32_t *sa32 =
1481                                         (uint32_t *) mh->match.acl.ipv6.sa;
1482                                 uint32_t *da32 =
1483                                         (uint32_t *) mh->match.acl.ipv6.da;
1484                                 uint32_t sa32_depth[4], da32_depth[4];
1485                                 int status;
1486
1487                                 status = match_convert_ipv6_depth(
1488                                         mh->match.acl.sa_depth,
1489                                         sa32_depth);
1490                                 if (status)
1491                                         return status;
1492
1493                                 status = match_convert_ipv6_depth(
1494                                         mh->match.acl.da_depth,
1495                                         da32_depth);
1496                                 if (status)
1497                                         return status;
1498
1499                                 ml->acl_delete.field_value[0].value.u8 =
1500                                         mh->match.acl.proto;
1501                                 ml->acl_delete.field_value[0].mask_range.u8 =
1502                                         mh->match.acl.proto_mask;
1503
1504                                 ml->acl_delete.field_value[1].value.u32 =
1505                                         sa32[0];
1506                                 ml->acl_delete.field_value[1].mask_range.u32 =
1507                                         sa32_depth[0];
1508                                 ml->acl_delete.field_value[2].value.u32 =
1509                                         sa32[1];
1510                                 ml->acl_delete.field_value[2].mask_range.u32 =
1511                                         sa32_depth[1];
1512                                 ml->acl_delete.field_value[3].value.u32 =
1513                                         sa32[2];
1514                                 ml->acl_delete.field_value[3].mask_range.u32 =
1515                                         sa32_depth[2];
1516                                 ml->acl_delete.field_value[4].value.u32 =
1517                                         sa32[3];
1518                                 ml->acl_delete.field_value[4].mask_range.u32 =
1519                                         sa32_depth[3];
1520
1521                                 ml->acl_delete.field_value[5].value.u32 =
1522                                         da32[0];
1523                                 ml->acl_delete.field_value[5].mask_range.u32 =
1524                                         da32_depth[0];
1525                                 ml->acl_delete.field_value[6].value.u32 =
1526                                         da32[1];
1527                                 ml->acl_delete.field_value[6].mask_range.u32 =
1528                                         da32_depth[1];
1529                                 ml->acl_delete.field_value[7].value.u32 =
1530                                         da32[2];
1531                                 ml->acl_delete.field_value[7].mask_range.u32 =
1532                                         da32_depth[2];
1533                                 ml->acl_delete.field_value[8].value.u32 =
1534                                         da32[3];
1535                                 ml->acl_delete.field_value[8].mask_range.u32 =
1536                                         da32_depth[3];
1537
1538                                 ml->acl_delete.field_value[9].value.u16 =
1539                                         mh->match.acl.sp0;
1540                                 ml->acl_delete.field_value[9].mask_range.u16 =
1541                                         mh->match.acl.sp1;
1542
1543                                 ml->acl_delete.field_value[10].value.u16 =
1544                                         mh->match.acl.dp0;
1545                                 ml->acl_delete.field_value[10].mask_range.u16 =
1546                                         mh->match.acl.dp1;
1547                         }
1548                 return 0;
1549
1550         case TABLE_ARRAY:
1551                 ml->array.pos = mh->match.array.pos;
1552                 return 0;
1553
1554         case TABLE_HASH:
1555                 memcpy(ml->hash, mh->match.hash.key, sizeof(ml->hash));
1556                 return 0;
1557
1558         case TABLE_LPM:
1559                 if (mh->match.lpm.ip_version) {
1560                         ml->lpm_ipv4.ip = mh->match.lpm.ipv4;
1561                         ml->lpm_ipv4.depth = mh->match.lpm.depth;
1562                 } else {
1563                         memcpy(ml->lpm_ipv6.ip,
1564                                 mh->match.lpm.ipv6, sizeof(ml->lpm_ipv6.ip));
1565                         ml->lpm_ipv6.depth = mh->match.lpm.depth;
1566                 }
1567
1568                 return 0;
1569
1570         default:
1571                 return -1;
1572         }
1573 }
1574
1575 static struct pipeline_msg_rsp *
1576 pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
1577         struct pipeline_msg_req *req)
1578 {
1579         union table_rule_match_low_level match_ll;
1580         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1581         struct table_rule_match *match = &req->table_rule_add.match;
1582         struct table_rule_action *action = &req->table_rule_add.action;
1583         struct rte_pipeline_table_entry *data_in, *data_out;
1584         uint32_t table_id = req->id;
1585         int key_found, status;
1586         struct rte_table_action *a = p->table_data[table_id].a;
1587
1588         /* Apply actions */
1589         memset(p->buffer, 0, sizeof(p->buffer));
1590         data_in = (struct rte_pipeline_table_entry *) p->buffer;
1591
1592         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1593                 status = rte_table_action_apply(a,
1594                         data_in,
1595                         RTE_TABLE_ACTION_FWD,
1596                         &action->fwd);
1597
1598                 if (status) {
1599                         rsp->status = -1;
1600                         return rsp;
1601                 }
1602         }
1603
1604         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1605                 status = rte_table_action_apply(a,
1606                         data_in,
1607                         RTE_TABLE_ACTION_MTR,
1608                         &action->mtr);
1609
1610                 if (status) {
1611                         rsp->status = -1;
1612                         return rsp;
1613                 }
1614         }
1615
1616         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1617                 status = rte_table_action_apply(a,
1618                         data_in,
1619                         RTE_TABLE_ACTION_TM,
1620                         &action->tm);
1621
1622                 if (status) {
1623                         rsp->status = -1;
1624                         return rsp;
1625                 }
1626         }
1627
1628         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1629                 status = rte_table_action_apply(a,
1630                         data_in,
1631                         RTE_TABLE_ACTION_ENCAP,
1632                         &action->encap);
1633
1634                 if (status) {
1635                         rsp->status = -1;
1636                         return rsp;
1637                 }
1638         }
1639
1640         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1641                 status = rte_table_action_apply(a,
1642                         data_in,
1643                         RTE_TABLE_ACTION_NAT,
1644                         &action->nat);
1645
1646                 if (status) {
1647                         rsp->status = -1;
1648                         return rsp;
1649                 }
1650         }
1651
1652         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
1653                 status = rte_table_action_apply(a,
1654                         data_in,
1655                         RTE_TABLE_ACTION_TTL,
1656                         &action->ttl);
1657
1658                 if (status) {
1659                         rsp->status = -1;
1660                         return rsp;
1661                 }
1662         }
1663
1664         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
1665                 status = rte_table_action_apply(a,
1666                         data_in,
1667                         RTE_TABLE_ACTION_STATS,
1668                         &action->stats);
1669
1670                 if (status) {
1671                         rsp->status = -1;
1672                         return rsp;
1673                 }
1674         }
1675
1676         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
1677                 status = rte_table_action_apply(a,
1678                         data_in,
1679                         RTE_TABLE_ACTION_TIME,
1680                         &action->time);
1681
1682                 if (status) {
1683                         rsp->status = -1;
1684                         return rsp;
1685                 }
1686         }
1687
1688         /* Add rule (match, action) to table */
1689         status = match_convert(match, &match_ll, 1);
1690         if (status) {
1691                 rsp->status = -1;
1692                 return rsp;
1693         }
1694
1695         status = rte_pipeline_table_entry_add(p->p,
1696                 table_id,
1697                 &match_ll,
1698                 data_in,
1699                 &key_found,
1700                 &data_out);
1701         if (status) {
1702                 rsp->status = -1;
1703                 return rsp;
1704         }
1705
1706         /* Write response */
1707         rsp->status = 0;
1708         rsp->table_rule_add.data = data_out;
1709
1710         return rsp;
1711 }
1712
1713 static struct pipeline_msg_rsp *
1714 pipeline_msg_handle_table_rule_add_default(struct pipeline_data *p,
1715         struct pipeline_msg_req *req)
1716 {
1717         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1718         struct table_rule_action *action = &req->table_rule_add_default.action;
1719         struct rte_pipeline_table_entry *data_in, *data_out;
1720         uint32_t table_id = req->id;
1721         int status;
1722
1723         /* Apply actions */
1724         memset(p->buffer, 0, sizeof(p->buffer));
1725         data_in = (struct rte_pipeline_table_entry *) p->buffer;
1726
1727         data_in->action = action->fwd.action;
1728         if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
1729                 data_in->port_id = action->fwd.id;
1730         if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
1731                 data_in->table_id = action->fwd.id;
1732
1733         /* Add default rule to table */
1734         status = rte_pipeline_table_default_entry_add(p->p,
1735                 table_id,
1736                 data_in,
1737                 &data_out);
1738         if (status) {
1739                 rsp->status = -1;
1740                 return rsp;
1741         }
1742
1743         /* Write response */
1744         rsp->status = 0;
1745         rsp->table_rule_add_default.data = data_out;
1746
1747         return rsp;
1748 }
1749
1750 static struct pipeline_msg_rsp *
1751 pipeline_msg_handle_table_rule_delete(struct pipeline_data *p,
1752         struct pipeline_msg_req *req)
1753 {
1754         union table_rule_match_low_level match_ll;
1755         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1756         struct table_rule_match *match = &req->table_rule_delete.match;
1757         uint32_t table_id = req->id;
1758         int key_found, status;
1759
1760         status = match_convert(match, &match_ll, 0);
1761         if (status) {
1762                 rsp->status = -1;
1763                 return rsp;
1764         }
1765
1766         rsp->status = rte_pipeline_table_entry_delete(p->p,
1767                 table_id,
1768                 &match_ll,
1769                 &key_found,
1770                 NULL);
1771
1772         return rsp;
1773 }
1774
1775 static struct pipeline_msg_rsp *
1776 pipeline_msg_handle_table_rule_delete_default(struct pipeline_data *p,
1777         struct pipeline_msg_req *req)
1778 {
1779         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1780         uint32_t table_id = req->id;
1781
1782         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1783                 table_id,
1784                 NULL);
1785
1786         return rsp;
1787 }
1788
1789 static void
1790 pipeline_msg_handle(struct pipeline_data *p)
1791 {
1792         for ( ; ; ) {
1793                 struct pipeline_msg_req *req;
1794                 struct pipeline_msg_rsp *rsp;
1795
1796                 req = pipeline_msg_recv(p->msgq_req);
1797                 if (req == NULL)
1798                         break;
1799
1800                 switch (req->type) {
1801                 case PIPELINE_REQ_PORT_IN_STATS_READ:
1802                         rsp = pipeline_msg_handle_port_in_stats_read(p, req);
1803                         break;
1804
1805                 case PIPELINE_REQ_PORT_IN_ENABLE:
1806                         rsp = pipeline_msg_handle_port_in_enable(p, req);
1807                         break;
1808
1809                 case PIPELINE_REQ_PORT_IN_DISABLE:
1810                         rsp = pipeline_msg_handle_port_in_disable(p, req);
1811                         break;
1812
1813                 case PIPELINE_REQ_PORT_OUT_STATS_READ:
1814                         rsp = pipeline_msg_handle_port_out_stats_read(p, req);
1815                         break;
1816
1817                 case PIPELINE_REQ_TABLE_STATS_READ:
1818                         rsp = pipeline_msg_handle_table_stats_read(p, req);
1819                         break;
1820
1821                 case PIPELINE_REQ_TABLE_RULE_ADD:
1822                         rsp = pipeline_msg_handle_table_rule_add(p, req);
1823                         break;
1824
1825                 case PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT:
1826                         rsp = pipeline_msg_handle_table_rule_add_default(p,     req);
1827                         break;
1828
1829                 case PIPELINE_REQ_TABLE_RULE_DELETE:
1830                         rsp = pipeline_msg_handle_table_rule_delete(p, req);
1831                         break;
1832
1833                 case PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT:
1834                         rsp = pipeline_msg_handle_table_rule_delete_default(p, req);
1835                         break;
1836
1837                 default:
1838                         rsp = (struct pipeline_msg_rsp *) req;
1839                         rsp->status = -1;
1840                 }
1841
1842                 pipeline_msg_send(p->msgq_rsp, rsp);
1843         }
1844 }
1845
1846 /**
1847  * Data plane threads: main
1848  */
1849 int
1850 thread_main(void *arg __rte_unused)
1851 {
1852         struct thread_data *t;
1853         uint32_t thread_id, i;
1854
1855         thread_id = rte_lcore_id();
1856         t = &thread_data[thread_id];
1857
1858         /* Dispatch loop */
1859         for (i = 0; ; i++) {
1860                 uint32_t j;
1861
1862                 /* Data Plane */
1863                 for (j = 0; j < t->n_pipelines; j++)
1864                         rte_pipeline_run(t->p[j]);
1865
1866                 /* Control Plane */
1867                 if ((i & 0xF) == 0) {
1868                         uint64_t time = rte_get_tsc_cycles();
1869                         uint64_t time_next_min = UINT64_MAX;
1870
1871                         if (time < t->time_next_min)
1872                                 continue;
1873
1874                         /* Pipeline message queues */
1875                         for (j = 0; j < t->n_pipelines; j++) {
1876                                 struct pipeline_data *p =
1877                                         &t->pipeline_data[j];
1878                                 uint64_t time_next = p->time_next;
1879
1880                                 if (time_next <= time) {
1881                                         pipeline_msg_handle(p);
1882                                         rte_pipeline_flush(p->p);
1883                                         time_next = time + p->timer_period;
1884                                         p->time_next = time_next;
1885                                 }
1886
1887                                 if (time_next < time_next_min)
1888                                         time_next_min = time_next;
1889                         }
1890
1891                         /* Thread message queues */
1892                         {
1893                                 uint64_t time_next = t->time_next;
1894
1895                                 if (time_next <= time) {
1896                                         thread_msg_handle(t);
1897                                         time_next = time + t->timer_period;
1898                                         t->time_next = time_next;
1899                                 }
1900
1901                                 if (time_next < time_next_min)
1902                                         time_next_min = time_next;
1903                         }
1904
1905                         t->time_next_min = time_next_min;
1906                 }
1907         }
1908
1909         return 0;
1910 }