examples/ip_pipeline/thread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdlib.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_cycles.h>
   9 #include <rte_lcore.h>
  10 #include <rte_ring.h>
  11
  12 #include <rte_table_acl.h>
  13 #include <rte_table_array.h>
  14 #include <rte_table_hash.h>
  15 #include <rte_table_lpm.h>
  16 #include <rte_table_lpm_ipv6.h>
  17
  18 #include "common.h"
  19 #include "thread.h"
  20 #include "pipeline.h"
  21
  22 #ifndef THREAD_PIPELINES_MAX
  23 #define THREAD_PIPELINES_MAX                               256
  24 #endif
  25
  26 #ifndef THREAD_MSGQ_SIZE
  27 #define THREAD_MSGQ_SIZE                                   64
  28 #endif
  29
  30 #ifndef THREAD_TIMER_PERIOD_MS
  31 #define THREAD_TIMER_PERIOD_MS                             100
  32 #endif
  33
  34 /**
  35  * Master thead: data plane thread context
  36  */
  37 struct thread {
  38         struct rte_ring *msgq_req;
  39         struct rte_ring *msgq_rsp;
  40
  41         uint32_t enabled;
  42 };
  43
  44 static struct thread thread[RTE_MAX_LCORE];
  45
  46 /**
  47  * Data plane threads: context
  48  */
  49 struct table_data {
  50         struct rte_table_action *a;
  51 };
  52
  53 struct pipeline_data {
  54         struct rte_pipeline *p;
  55         struct table_data table_data[RTE_PIPELINE_TABLE_MAX];
  56         uint32_t n_tables;
  57
  58         struct rte_ring *msgq_req;
  59         struct rte_ring *msgq_rsp;
  60         uint64_t timer_period; /* Measured in CPU cycles. */
  61         uint64_t time_next;
  62
  63         uint8_t buffer[TABLE_RULE_ACTION_SIZE_MAX];
  64 };
  65
  66 struct thread_data {
  67         struct rte_pipeline *p[THREAD_PIPELINES_MAX];
  68         uint32_t n_pipelines;
  69
  70         struct pipeline_data pipeline_data[THREAD_PIPELINES_MAX];
  71         struct rte_ring *msgq_req;
  72         struct rte_ring *msgq_rsp;
  73         uint64_t timer_period; /* Measured in CPU cycles. */
  74         uint64_t time_next;
  75         uint64_t time_next_min;
  76 } __rte_cache_aligned;
  77
  78 static struct thread_data thread_data[RTE_MAX_LCORE];
  79
  80 /**
  81  * Master thread: data plane thread init
  82  */
  83 static void
  84 thread_free(void)
  85 {
  86         uint32_t i;
  87
  88         for (i = 0; i < RTE_MAX_LCORE; i++) {
  89                 struct thread *t = &thread[i];
  90
  91                 if (!rte_lcore_is_enabled(i))
  92                         continue;
  93
  94                 /* MSGQs */
  95                 if (t->msgq_req)
  96                         rte_ring_free(t->msgq_req);
  97
  98                 if (t->msgq_rsp)
  99                         rte_ring_free(t->msgq_rsp);
 100         }
 101 }
 102
 103 int
 104 thread_init(void)
 105 {
 106         uint32_t i;
 107
 108         RTE_LCORE_FOREACH_SLAVE(i) {
 109                 char name[NAME_MAX];
 110                 struct rte_ring *msgq_req, *msgq_rsp;
 111                 struct thread *t = &thread[i];
 112                 struct thread_data *t_data = &thread_data[i];
 113                 uint32_t cpu_id = rte_lcore_to_socket_id(i);
 114
 115                 /* MSGQs */
 116                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-REQ", i);
 117
 118                 msgq_req = rte_ring_create(name,
 119                         THREAD_MSGQ_SIZE,
 120                         cpu_id,
 121                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 122
 123                 if (msgq_req == NULL) {
 124                         thread_free();
 125                         return -1;
 126                 }
 127
 128                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-RSP", i);
 129
 130                 msgq_rsp = rte_ring_create(name,
 131                         THREAD_MSGQ_SIZE,
 132                         cpu_id,
 133                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 134
 135                 if (msgq_rsp == NULL) {
 136                         thread_free();
 137                         return -1;
 138                 }
 139
 140                 /* Master thread records */
 141                 t->msgq_req = msgq_req;
 142                 t->msgq_rsp = msgq_rsp;
 143                 t->enabled = 1;
 144
 145                 /* Data plane thread records */
 146                 t_data->n_pipelines = 0;
 147                 t_data->msgq_req = msgq_req;
 148                 t_data->msgq_rsp = msgq_rsp;
 149                 t_data->timer_period =
 150                         (rte_get_tsc_hz() * THREAD_TIMER_PERIOD_MS) / 1000;
 151                 t_data->time_next = rte_get_tsc_cycles() + t_data->timer_period;
 152                 t_data->time_next_min = t_data->time_next;
 153         }
 154
 155         return 0;
 156 }
 157
 158 /**
 159  * Master thread & data plane threads: message passing
 160  */
 161 enum thread_req_type {
 162         THREAD_REQ_PIPELINE_ENABLE = 0,
 163         THREAD_REQ_PIPELINE_DISABLE,
 164         THREAD_REQ_MAX
 165 };
 166
 167 struct thread_msg_req {
 168         enum thread_req_type type;
 169
 170         union {
 171                 struct {
 172                         struct rte_pipeline *p;
 173                         struct {
 174                                 struct rte_table_action *a;
 175                         } table[RTE_PIPELINE_TABLE_MAX];
 176                         struct rte_ring *msgq_req;
 177                         struct rte_ring *msgq_rsp;
 178                         uint32_t timer_period_ms;
 179                         uint32_t n_tables;
 180                 } pipeline_enable;
 181
 182                 struct {
 183                         struct rte_pipeline *p;
 184                 } pipeline_disable;
 185         };
 186 };
 187
 188 struct thread_msg_rsp {
 189         int status;
 190 };
 191
 192 /**
 193  * Master thread
 194  */
 195 static struct thread_msg_req *
 196 thread_msg_alloc(void)
 197 {
 198         size_t size = RTE_MAX(sizeof(struct thread_msg_req),
 199                 sizeof(struct thread_msg_rsp));
 200
 201         return calloc(1, size);
 202 }
 203
 204 static void
 205 thread_msg_free(struct thread_msg_rsp *rsp)
 206 {
 207         free(rsp);
 208 }
 209
 210 static struct thread_msg_rsp *
 211 thread_msg_send_recv(uint32_t thread_id,
 212         struct thread_msg_req *req)
 213 {
 214         struct thread *t = &thread[thread_id];
 215         struct rte_ring *msgq_req = t->msgq_req;
 216         struct rte_ring *msgq_rsp = t->msgq_rsp;
 217         struct thread_msg_rsp *rsp;
 218         int status;
 219
 220         /* send */
 221         do {
 222                 status = rte_ring_sp_enqueue(msgq_req, req);
 223         } while (status == -ENOBUFS);
 224
 225         /* recv */
 226         do {
 227                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 228         } while (status != 0);
 229
 230         return rsp;
 231 }
 232
 233 int
 234 thread_pipeline_enable(uint32_t thread_id,
 235         const char *pipeline_name)
 236 {
 237         struct pipeline *p = pipeline_find(pipeline_name);
 238         struct thread *t;
 239         struct thread_msg_req *req;
 240         struct thread_msg_rsp *rsp;
 241         uint32_t i;
 242         int status;
 243
 244         /* Check input params */
 245         if ((thread_id >= RTE_MAX_LCORE) ||
 246                 (p == NULL) ||
 247                 (p->n_ports_in == 0) ||
 248                 (p->n_ports_out == 0) ||
 249                 (p->n_tables == 0))
 250                 return -1;
 251
 252         t = &thread[thread_id];
 253         if ((t->enabled == 0) ||
 254                 p->enabled)
 255                 return -1;
 256
 257         /* Allocate request */
 258         req = thread_msg_alloc();
 259         if (req == NULL)
 260                 return -1;
 261
 262         /* Write request */
 263         req->type = THREAD_REQ_PIPELINE_ENABLE;
 264         req->pipeline_enable.p = p->p;
 265         for (i = 0; i < p->n_tables; i++)
 266                 req->pipeline_enable.table[i].a =
 267                         p->table[i].a;
 268         req->pipeline_enable.msgq_req = p->msgq_req;
 269         req->pipeline_enable.msgq_rsp = p->msgq_rsp;
 270         req->pipeline_enable.timer_period_ms = p->timer_period_ms;
 271         req->pipeline_enable.n_tables = p->n_tables;
 272
 273         /* Send request and wait for response */
 274         rsp = thread_msg_send_recv(thread_id, req);
 275         if (rsp == NULL)
 276                 return -1;
 277
 278         /* Read response */
 279         status = rsp->status;
 280
 281         /* Free response */
 282         thread_msg_free(rsp);
 283
 284         /* Request completion */
 285         if (status)
 286                 return status;
 287
 288         p->thread_id = thread_id;
 289         p->enabled = 1;
 290
 291         return 0;
 292 }
 293
 294 int
 295 thread_pipeline_disable(uint32_t thread_id,
 296         const char *pipeline_name)
 297 {
 298         struct pipeline *p = pipeline_find(pipeline_name);
 299         struct thread *t;
 300         struct thread_msg_req *req;
 301         struct thread_msg_rsp *rsp;
 302         int status;
 303
 304         /* Check input params */
 305         if ((thread_id >= RTE_MAX_LCORE) ||
 306                 (p == NULL))
 307                 return -1;
 308
 309         t = &thread[thread_id];
 310         if (t->enabled == 0)
 311                 return -1;
 312
 313         if (p->enabled == 0)
 314                 return 0;
 315
 316         if (p->thread_id != thread_id)
 317                 return -1;
 318
 319         /* Allocate request */
 320         req = thread_msg_alloc();
 321         if (req == NULL)
 322                 return -1;
 323
 324         /* Write request */
 325         req->type = THREAD_REQ_PIPELINE_DISABLE;
 326         req->pipeline_disable.p = p->p;
 327
 328         /* Send request and wait for response */
 329         rsp = thread_msg_send_recv(thread_id, req);
 330         if (rsp == NULL)
 331                 return -1;
 332
 333         /* Read response */
 334         status = rsp->status;
 335
 336         /* Free response */
 337         thread_msg_free(rsp);
 338
 339         /* Request completion */
 340         if (status)
 341                 return status;
 342
 343         p->enabled = 0;
 344
 345         return 0;
 346 }
 347
 348 /**
 349  * Data plane threads: message handling
 350  */
 351 static inline struct thread_msg_req *
 352 thread_msg_recv(struct rte_ring *msgq_req)
 353 {
 354         struct thread_msg_req *req;
 355
 356         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
 357
 358         if (status != 0)
 359                 return NULL;
 360
 361         return req;
 362 }
 363
 364 static inline void
 365 thread_msg_send(struct rte_ring *msgq_rsp,
 366         struct thread_msg_rsp *rsp)
 367 {
 368         int status;
 369
 370         do {
 371                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
 372         } while (status == -ENOBUFS);
 373 }
 374
 375 static struct thread_msg_rsp *
 376 thread_msg_handle_pipeline_enable(struct thread_data *t,
 377         struct thread_msg_req *req)
 378 {
 379         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 380         struct pipeline_data *p = &t->pipeline_data[t->n_pipelines];
 381         uint32_t i;
 382
 383         /* Request */
 384         if (t->n_pipelines >= THREAD_PIPELINES_MAX) {
 385                 rsp->status = -1;
 386                 return rsp;
 387         }
 388
 389         t->p[t->n_pipelines] = req->pipeline_enable.p;
 390
 391         p->p = req->pipeline_enable.p;
 392         for (i = 0; i < req->pipeline_enable.n_tables; i++)
 393                 p->table_data[i].a =
 394                         req->pipeline_enable.table[i].a;
 395
 396         p->n_tables = req->pipeline_enable.n_tables;
 397
 398         p->msgq_req = req->pipeline_enable.msgq_req;
 399         p->msgq_rsp = req->pipeline_enable.msgq_rsp;
 400         p->timer_period =
 401                 (rte_get_tsc_hz() * req->pipeline_enable.timer_period_ms) / 1000;
 402         p->time_next = rte_get_tsc_cycles() + p->timer_period;
 403
 404         t->n_pipelines++;
 405
 406         /* Response */
 407         rsp->status = 0;
 408         return rsp;
 409 }
 410
 411 static struct thread_msg_rsp *
 412 thread_msg_handle_pipeline_disable(struct thread_data *t,
 413         struct thread_msg_req *req)
 414 {
 415         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 416         uint32_t n_pipelines = t->n_pipelines;
 417         struct rte_pipeline *pipeline = req->pipeline_disable.p;
 418         uint32_t i;
 419
 420         /* find pipeline */
 421         for (i = 0; i < n_pipelines; i++) {
 422                 struct pipeline_data *p = &t->pipeline_data[i];
 423
 424                 if (p->p != pipeline)
 425                         continue;
 426
 427                 if (i < n_pipelines - 1) {
 428                         struct rte_pipeline *pipeline_last =
 429                                 t->p[n_pipelines - 1];
 430                         struct pipeline_data *p_last =
 431                                 &t->pipeline_data[n_pipelines - 1];
 432
 433                         t->p[i] = pipeline_last;
 434                         memcpy(p, p_last, sizeof(*p));
 435                 }
 436
 437                 t->n_pipelines--;
 438
 439                 rsp->status = 0;
 440                 return rsp;
 441         }
 442
 443         /* should not get here */
 444         rsp->status = 0;
 445         return rsp;
 446 }
 447
 448 static void
 449 thread_msg_handle(struct thread_data *t)
 450 {
 451         for ( ; ; ) {
 452                 struct thread_msg_req *req;
 453                 struct thread_msg_rsp *rsp;
 454
 455                 req = thread_msg_recv(t->msgq_req);
 456                 if (req == NULL)
 457                         break;
 458
 459                 switch (req->type) {
 460                 case THREAD_REQ_PIPELINE_ENABLE:
 461                         rsp = thread_msg_handle_pipeline_enable(t, req);
 462                         break;
 463
 464                 case THREAD_REQ_PIPELINE_DISABLE:
 465                         rsp = thread_msg_handle_pipeline_disable(t, req);
 466                         break;
 467
 468                 default:
 469                         rsp = (struct thread_msg_rsp *) req;
 470                         rsp->status = -1;
 471                 }
 472
 473                 thread_msg_send(t->msgq_rsp, rsp);
 474         }
 475 }
 476
 477 /**
 478  * Master thread & data plane threads: message passing
 479  */
 480 enum pipeline_req_type {
 481         /* Port IN */
 482         PIPELINE_REQ_PORT_IN_STATS_READ,
 483         PIPELINE_REQ_PORT_IN_ENABLE,
 484         PIPELINE_REQ_PORT_IN_DISABLE,
 485
 486         /* Port OUT */
 487         PIPELINE_REQ_PORT_OUT_STATS_READ,
 488
 489         /* Table */
 490         PIPELINE_REQ_TABLE_STATS_READ,
 491         PIPELINE_REQ_TABLE_RULE_ADD,
 492         PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT,
 493
 494         PIPELINE_REQ_MAX
 495 };
 496
 497 struct pipeline_msg_req_port_in_stats_read {
 498         int clear;
 499 };
 500
 501 struct pipeline_msg_req_port_out_stats_read {
 502         int clear;
 503 };
 504
 505 struct pipeline_msg_req_table_stats_read {
 506         int clear;
 507 };
 508
 509 struct pipeline_msg_req_table_rule_add {
 510         struct table_rule_match match;
 511         struct table_rule_action action;
 512 };
 513
 514 struct pipeline_msg_req_table_rule_add_default {
 515         struct table_rule_action action;
 516 };
 517
 518 struct pipeline_msg_req {
 519         enum pipeline_req_type type;
 520         uint32_t id; /* Port IN, port OUT or table ID */
 521
 522         RTE_STD_C11
 523         union {
 524                 struct pipeline_msg_req_port_in_stats_read port_in_stats_read;
 525                 struct pipeline_msg_req_port_out_stats_read port_out_stats_read;
 526                 struct pipeline_msg_req_table_stats_read table_stats_read;
 527                 struct pipeline_msg_req_table_rule_add table_rule_add;
 528                 struct pipeline_msg_req_table_rule_add_default table_rule_add_default;
 529         };
 530 };
 531
 532 struct pipeline_msg_rsp_port_in_stats_read {
 533         struct rte_pipeline_port_in_stats stats;
 534 };
 535
 536 struct pipeline_msg_rsp_port_out_stats_read {
 537         struct rte_pipeline_port_out_stats stats;
 538 };
 539
 540 struct pipeline_msg_rsp_table_stats_read {
 541         struct rte_pipeline_table_stats stats;
 542 };
 543
 544 struct pipeline_msg_rsp_table_rule_add {
 545         void *data;
 546 };
 547
 548 struct pipeline_msg_rsp_table_rule_add_default {
 549         void *data;
 550 };
 551
 552 struct pipeline_msg_rsp {
 553         int status;
 554
 555         RTE_STD_C11
 556         union {
 557                 struct pipeline_msg_rsp_port_in_stats_read port_in_stats_read;
 558                 struct pipeline_msg_rsp_port_out_stats_read port_out_stats_read;
 559                 struct pipeline_msg_rsp_table_stats_read table_stats_read;
 560                 struct pipeline_msg_rsp_table_rule_add table_rule_add;
 561                 struct pipeline_msg_rsp_table_rule_add_default table_rule_add_default;
 562         };
 563 };
 564
 565 /**
 566  * Master thread
 567  */
 568 static struct pipeline_msg_req *
 569 pipeline_msg_alloc(void)
 570 {
 571         size_t size = RTE_MAX(sizeof(struct pipeline_msg_req),
 572                 sizeof(struct pipeline_msg_rsp));
 573
 574         return calloc(1, size);
 575 }
 576
 577 static void
 578 pipeline_msg_free(struct pipeline_msg_rsp *rsp)
 579 {
 580         free(rsp);
 581 }
 582
 583 static struct pipeline_msg_rsp *
 584 pipeline_msg_send_recv(struct pipeline *p,
 585         struct pipeline_msg_req *req)
 586 {
 587         struct rte_ring *msgq_req = p->msgq_req;
 588         struct rte_ring *msgq_rsp = p->msgq_rsp;
 589         struct pipeline_msg_rsp *rsp;
 590         int status;
 591
 592         /* send */
 593         do {
 594                 status = rte_ring_sp_enqueue(msgq_req, req);
 595         } while (status == -ENOBUFS);
 596
 597         /* recv */
 598         do {
 599                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 600         } while (status != 0);
 601
 602         return rsp;
 603 }
 604
 605 int
 606 pipeline_port_in_stats_read(const char *pipeline_name,
 607         uint32_t port_id,
 608         struct rte_pipeline_port_in_stats *stats,
 609         int clear)
 610 {
 611         struct pipeline *p;
 612         struct pipeline_msg_req *req;
 613         struct pipeline_msg_rsp *rsp;
 614         int status;
 615
 616         /* Check input params */
 617         if ((pipeline_name == NULL) ||
 618                 (stats == NULL))
 619                 return -1;
 620
 621         p = pipeline_find(pipeline_name);
 622         if ((p == NULL) ||
 623                 (p->enabled == 0) ||
 624                 (port_id >= p->n_ports_in))
 625                 return -1;
 626
 627         /* Allocate request */
 628         req = pipeline_msg_alloc();
 629         if (req == NULL)
 630                 return -1;
 631
 632         /* Write request */
 633         req->type = PIPELINE_REQ_PORT_IN_STATS_READ;
 634         req->id = port_id;
 635         req->port_in_stats_read.clear = clear;
 636
 637         /* Send request and wait for response */
 638         rsp = pipeline_msg_send_recv(p, req);
 639         if (rsp == NULL)
 640                 return -1;
 641
 642         /* Read response */
 643         status = rsp->status;
 644         if (status)
 645                 memcpy(stats, &rsp->port_in_stats_read.stats, sizeof(*stats));
 646
 647         /* Free response */
 648         pipeline_msg_free(rsp);
 649
 650         return status;
 651 }
 652
 653 int
 654 pipeline_port_in_enable(const char *pipeline_name,
 655         uint32_t port_id)
 656 {
 657         struct pipeline *p;
 658         struct pipeline_msg_req *req;
 659         struct pipeline_msg_rsp *rsp;
 660         int status;
 661
 662         /* Check input params */
 663         if (pipeline_name == NULL)
 664                 return -1;
 665
 666         p = pipeline_find(pipeline_name);
 667         if ((p == NULL) ||
 668                 (p->enabled == 0) ||
 669                 (port_id >= p->n_ports_in))
 670                 return -1;
 671
 672         /* Allocate request */
 673         req = pipeline_msg_alloc();
 674         if (req == NULL)
 675                 return -1;
 676
 677         /* Write request */
 678         req->type = PIPELINE_REQ_PORT_IN_ENABLE;
 679         req->id = port_id;
 680
 681         /* Send request and wait for response */
 682         rsp = pipeline_msg_send_recv(p, req);
 683         if (rsp == NULL)
 684                 return -1;
 685
 686         /* Read response */
 687         status = rsp->status;
 688
 689         /* Free response */
 690         pipeline_msg_free(rsp);
 691
 692         return status;
 693 }
 694
 695 int
 696 pipeline_port_in_disable(const char *pipeline_name,
 697         uint32_t port_id)
 698 {
 699         struct pipeline *p;
 700         struct pipeline_msg_req *req;
 701         struct pipeline_msg_rsp *rsp;
 702         int status;
 703
 704         /* Check input params */
 705         if (pipeline_name == NULL)
 706                 return -1;
 707
 708         p = pipeline_find(pipeline_name);
 709         if ((p == NULL) ||
 710                 (p->enabled == 0) ||
 711                 (port_id >= p->n_ports_in))
 712                 return -1;
 713
 714         /* Allocate request */
 715         req = pipeline_msg_alloc();
 716         if (req == NULL)
 717                 return -1;
 718
 719         /* Write request */
 720         req->type = PIPELINE_REQ_PORT_IN_DISABLE;
 721         req->id = port_id;
 722
 723         /* Send request and wait for response */
 724         rsp = pipeline_msg_send_recv(p, req);
 725         if (rsp == NULL)
 726                 return -1;
 727
 728         /* Read response */
 729         status = rsp->status;
 730
 731         /* Free response */
 732         pipeline_msg_free(rsp);
 733
 734         return status;
 735 }
 736
 737 int
 738 pipeline_port_out_stats_read(const char *pipeline_name,
 739         uint32_t port_id,
 740         struct rte_pipeline_port_out_stats *stats,
 741         int clear)
 742 {
 743         struct pipeline *p;
 744         struct pipeline_msg_req *req;
 745         struct pipeline_msg_rsp *rsp;
 746         int status;
 747
 748         /* Check input params */
 749         if ((pipeline_name == NULL) ||
 750                 (stats == NULL))
 751                 return -1;
 752
 753         p = pipeline_find(pipeline_name);
 754         if ((p == NULL) ||
 755                 (p->enabled == 0) ||
 756                 (port_id >= p->n_ports_out))
 757                 return -1;
 758
 759         /* Allocate request */
 760         req = pipeline_msg_alloc();
 761         if (req == NULL)
 762                 return -1;
 763
 764         /* Write request */
 765         req->type = PIPELINE_REQ_PORT_OUT_STATS_READ;
 766         req->id = port_id;
 767         req->port_out_stats_read.clear = clear;
 768
 769         /* Send request and wait for response */
 770         rsp = pipeline_msg_send_recv(p, req);
 771         if (rsp == NULL)
 772                 return -1;
 773
 774         /* Read response */
 775         status = rsp->status;
 776         if (status)
 777                 memcpy(stats, &rsp->port_out_stats_read.stats, sizeof(*stats));
 778
 779         /* Free response */
 780         pipeline_msg_free(rsp);
 781
 782         return status;
 783 }
 784
 785 int
 786 pipeline_table_stats_read(const char *pipeline_name,
 787         uint32_t table_id,
 788         struct rte_pipeline_table_stats *stats,
 789         int clear)
 790 {
 791         struct pipeline *p;
 792         struct pipeline_msg_req *req;
 793         struct pipeline_msg_rsp *rsp;
 794         int status;
 795
 796         /* Check input params */
 797         if ((pipeline_name == NULL) ||
 798                 (stats == NULL))
 799                 return -1;
 800
 801         p = pipeline_find(pipeline_name);
 802         if ((p == NULL) ||
 803                 (p->enabled == 0) ||
 804                 (table_id >= p->n_tables))
 805                 return -1;
 806
 807         /* Allocate request */
 808         req = pipeline_msg_alloc();
 809         if (req == NULL)
 810                 return -1;
 811
 812         /* Write request */
 813         req->type = PIPELINE_REQ_TABLE_STATS_READ;
 814         req->id = table_id;
 815         req->table_stats_read.clear = clear;
 816
 817         /* Send request and wait for response */
 818         rsp = pipeline_msg_send_recv(p, req);
 819         if (rsp == NULL)
 820                 return -1;
 821
 822         /* Read response */
 823         status = rsp->status;
 824         if (status)
 825                 memcpy(stats, &rsp->table_stats_read.stats, sizeof(*stats));
 826
 827         /* Free response */
 828         pipeline_msg_free(rsp);
 829
 830         return status;
 831 }
 832
 833 static int
 834 match_check(struct table_rule_match *match,
 835         struct pipeline *p,
 836         uint32_t table_id)
 837 {
 838         struct table *table;
 839
 840         if ((match == NULL) ||
 841                 (p == NULL) ||
 842                 (table_id >= p->n_tables))
 843                 return -1;
 844
 845         table = &p->table[table_id];
 846         if (match->match_type != table->params.match_type)
 847                 return -1;
 848
 849         switch (match->match_type) {
 850         case TABLE_ACL:
 851         {
 852                 struct table_acl_params *t = &table->params.match.acl;
 853                 struct table_rule_match_acl *r = &match->match.acl;
 854
 855                 if ((r->ip_version && (t->ip_version == 0)) ||
 856                         ((r->ip_version == 0) && t->ip_version))
 857                         return -1;
 858
 859                 if (r->ip_version) {
 860                         if ((r->sa_depth > 32) ||
 861                                 (r->da_depth > 32))
 862                                 return -1;
 863                 } else {
 864                         if ((r->sa_depth > 128) ||
 865                                 (r->da_depth > 128))
 866                                 return -1;
 867                 }
 868                 return 0;
 869         }
 870
 871         case TABLE_ARRAY:
 872                 return 0;
 873
 874         case TABLE_HASH:
 875                 return 0;
 876
 877         case TABLE_LPM:
 878         {
 879                 struct table_lpm_params *t = &table->params.match.lpm;
 880                 struct table_rule_match_lpm *r = &match->match.lpm;
 881
 882                 if ((r->ip_version && (t->key_size != 4)) ||
 883                         ((r->ip_version == 0) && (t->key_size != 16)))
 884                         return -1;
 885
 886                 if (r->ip_version) {
 887                         if (r->depth > 32)
 888                                 return -1;
 889                 } else {
 890                         if (r->depth > 128)
 891                                 return -1;
 892                 }
 893                 return 0;
 894         }
 895
 896         case TABLE_STUB:
 897                 return -1;
 898
 899         default:
 900                 return -1;
 901         }
 902 }
 903
 904 static int
 905 action_check(struct table_rule_action *action,
 906         struct pipeline *p,
 907         uint32_t table_id)
 908 {
 909         struct table_action_profile *ap;
 910
 911         if ((action == NULL) ||
 912                 (p == NULL) ||
 913                 (table_id >= p->n_tables))
 914                 return -1;
 915
 916         ap = p->table[table_id].ap;
 917         if (action->action_mask != ap->params.action_mask)
 918                 return -1;
 919
 920         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
 921                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
 922                         (action->fwd.id >= p->n_ports_out))
 923                         return -1;
 924
 925                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
 926                         (action->fwd.id >= p->n_tables))
 927                         return -1;
 928         }
 929
 930         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
 931                 uint32_t tc_mask0 = (1 << ap->params.mtr.n_tc) - 1;
 932                 uint32_t tc_mask1 = action->mtr.tc_mask;
 933
 934                 if (tc_mask1 != tc_mask0)
 935                         return -1;
 936         }
 937
 938         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
 939                 uint32_t n_subports_per_port =
 940                         ap->params.tm.n_subports_per_port;
 941                 uint32_t n_pipes_per_subport =
 942                         ap->params.tm.n_pipes_per_subport;
 943                 uint32_t subport_id = action->tm.subport_id;
 944                 uint32_t pipe_id = action->tm.pipe_id;
 945
 946                 if ((subport_id >= n_subports_per_port) ||
 947                         (pipe_id >= n_pipes_per_subport))
 948                         return -1;
 949         }
 950
 951         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
 952                 uint64_t encap_mask = ap->params.encap.encap_mask;
 953                 enum rte_table_action_encap_type type = action->encap.type;
 954
 955                 if ((encap_mask & (1LLU << type)) == 0)
 956                         return -1;
 957         }
 958
 959         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
 960                 int ip_version0 = ap->params.common.ip_version;
 961                 int ip_version1 = action->nat.ip_version;
 962
 963                 if ((ip_version1 && (ip_version0 == 0)) ||
 964                         ((ip_version1 == 0) && ip_version0))
 965                         return -1;
 966         }
 967
 968         return 0;
 969 }
 970
 971 static int
 972 action_default_check(struct table_rule_action *action,
 973         struct pipeline *p,
 974         uint32_t table_id)
 975 {
 976         if ((action == NULL) ||
 977                 (action->action_mask != (1LLU << RTE_TABLE_ACTION_FWD)) ||
 978                 (p == NULL) ||
 979                 (table_id >= p->n_tables))
 980                 return -1;
 981
 982         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
 983                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
 984                         (action->fwd.id >= p->n_ports_out))
 985                         return -1;
 986
 987                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
 988                         (action->fwd.id >= p->n_tables))
 989                         return -1;
 990         }
 991
 992         return 0;
 993 }
 994
 995 int
 996 pipeline_table_rule_add(const char *pipeline_name,
 997         uint32_t table_id,
 998         struct table_rule_match *match,
 999         struct table_rule_action *action,
1000         void **data)
1001 {
1002         struct pipeline *p;
1003         struct pipeline_msg_req *req;
1004         struct pipeline_msg_rsp *rsp;
1005         int status;
1006
1007         /* Check input params */
1008         if ((pipeline_name == NULL) ||
1009                 (match == NULL) ||
1010                 (action == NULL) ||
1011                 (data == NULL))
1012                 return -1;
1013
1014         p = pipeline_find(pipeline_name);
1015         if ((p == NULL) ||
1016                 (p->enabled == 0) ||
1017                 (table_id >= p->n_tables) ||
1018                 match_check(match, p, table_id) ||
1019                 action_check(action, p, table_id))
1020                 return -1;
1021
1022         /* Allocate request */
1023         req = pipeline_msg_alloc();
1024         if (req == NULL)
1025                 return -1;
1026
1027         /* Write request */
1028         req->type = PIPELINE_REQ_TABLE_RULE_ADD;
1029         req->id = table_id;
1030         memcpy(&req->table_rule_add.match, match, sizeof(*match));
1031         memcpy(&req->table_rule_add.action, action, sizeof(*action));
1032
1033         /* Send request and wait for response */
1034         rsp = pipeline_msg_send_recv(p, req);
1035         if (rsp == NULL)
1036                 return -1;
1037
1038         /* Read response */
1039         status = rsp->status;
1040         if (status == 0)
1041                 *data = rsp->table_rule_add.data;
1042
1043         /* Free response */
1044         pipeline_msg_free(rsp);
1045
1046         return status;
1047 }
1048
1049 int
1050 pipeline_table_rule_add_default(const char *pipeline_name,
1051         uint32_t table_id,
1052         struct table_rule_action *action,
1053         void **data)
1054 {
1055         struct pipeline *p;
1056         struct pipeline_msg_req *req;
1057         struct pipeline_msg_rsp *rsp;
1058         int status;
1059
1060         /* Check input params */
1061         if ((pipeline_name == NULL) ||
1062                 (action == NULL) ||
1063                 (data == NULL))
1064                 return -1;
1065
1066         p = pipeline_find(pipeline_name);
1067         if ((p == NULL) ||
1068                 (p->enabled == 0) ||
1069                 (table_id >= p->n_tables) ||
1070                 action_default_check(action, p, table_id))
1071                 return -1;
1072
1073         /* Allocate request */
1074         req = pipeline_msg_alloc();
1075         if (req == NULL)
1076                 return -1;
1077
1078         /* Write request */
1079         req->type = PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT;
1080         req->id = table_id;
1081         memcpy(&req->table_rule_add_default.action, action, sizeof(*action));
1082
1083         /* Send request and wait for response */
1084         rsp = pipeline_msg_send_recv(p, req);
1085         if (rsp == NULL)
1086                 return -1;
1087
1088         /* Read response */
1089         status = rsp->status;
1090         if (status == 0)
1091                 *data = rsp->table_rule_add_default.data;
1092
1093         /* Free response */
1094         pipeline_msg_free(rsp);
1095
1096         return status;
1097 }
1098
1099 /**
1100  * Data plane threads: message handling
1101  */
1102 static inline struct pipeline_msg_req *
1103 pipeline_msg_recv(struct rte_ring *msgq_req)
1104 {
1105         struct pipeline_msg_req *req;
1106
1107         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
1108
1109         if (status != 0)
1110                 return NULL;
1111
1112         return req;
1113 }
1114
1115 static inline void
1116 pipeline_msg_send(struct rte_ring *msgq_rsp,
1117         struct pipeline_msg_rsp *rsp)
1118 {
1119         int status;
1120
1121         do {
1122                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
1123         } while (status == -ENOBUFS);
1124 }
1125
1126 static struct pipeline_msg_rsp *
1127 pipeline_msg_handle_port_in_stats_read(struct pipeline_data *p,
1128         struct pipeline_msg_req *req)
1129 {
1130         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1131         uint32_t port_id = req->id;
1132         int clear = req->port_in_stats_read.clear;
1133
1134         rsp->status = rte_pipeline_port_in_stats_read(p->p,
1135                 port_id,
1136                 &rsp->port_in_stats_read.stats,
1137                 clear);
1138
1139         return rsp;
1140 }
1141
1142 static struct pipeline_msg_rsp *
1143 pipeline_msg_handle_port_in_enable(struct pipeline_data *p,
1144         struct pipeline_msg_req *req)
1145 {
1146         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1147         uint32_t port_id = req->id;
1148
1149         rsp->status = rte_pipeline_port_in_enable(p->p,
1150                 port_id);
1151
1152         return rsp;
1153 }
1154
1155 static struct pipeline_msg_rsp *
1156 pipeline_msg_handle_port_in_disable(struct pipeline_data *p,
1157         struct pipeline_msg_req *req)
1158 {
1159         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1160         uint32_t port_id = req->id;
1161
1162         rsp->status = rte_pipeline_port_in_disable(p->p,
1163                 port_id);
1164
1165         return rsp;
1166 }
1167
1168 static struct pipeline_msg_rsp *
1169 pipeline_msg_handle_port_out_stats_read(struct pipeline_data *p,
1170         struct pipeline_msg_req *req)
1171 {
1172         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1173         uint32_t port_id = req->id;
1174         int clear = req->port_out_stats_read.clear;
1175
1176         rsp->status = rte_pipeline_port_out_stats_read(p->p,
1177                 port_id,
1178                 &rsp->port_out_stats_read.stats,
1179                 clear);
1180
1181         return rsp;
1182 }
1183
1184 static struct pipeline_msg_rsp *
1185 pipeline_msg_handle_table_stats_read(struct pipeline_data *p,
1186         struct pipeline_msg_req *req)
1187 {
1188         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1189         uint32_t port_id = req->id;
1190         int clear = req->table_stats_read.clear;
1191
1192         rsp->status = rte_pipeline_table_stats_read(p->p,
1193                 port_id,
1194                 &rsp->table_stats_read.stats,
1195                 clear);
1196
1197         return rsp;
1198 }
1199
1200 union table_rule_match_low_level {
1201         struct rte_table_acl_rule_add_params acl_add;
1202         struct rte_table_acl_rule_delete_params acl_delete;
1203         struct rte_table_array_key array;
1204         uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
1205         struct rte_table_lpm_key lpm_ipv4;
1206         struct rte_table_lpm_ipv6_key lpm_ipv6;
1207 };
1208
1209 static int
1210 match_convert_ipv6_depth(uint32_t depth, uint32_t *depth32)
1211 {
1212         if (depth > 128)
1213                 return -1;
1214
1215         switch (depth / 32) {
1216         case 0:
1217                 depth32[0] = depth;
1218                 depth32[1] = 0;
1219                 depth32[2] = 0;
1220                 depth32[3] = 0;
1221                 return 0;
1222
1223         case 1:
1224                 depth32[0] = 32;
1225                 depth32[1] = depth - 32;
1226                 depth32[2] = 0;
1227                 depth32[3] = 0;
1228                 return 0;
1229
1230         case 2:
1231                 depth32[0] = 32;
1232                 depth32[1] = 32;
1233                 depth32[2] = depth - 64;
1234                 depth32[3] = 0;
1235                 return 0;
1236
1237         case 3:
1238                 depth32[0] = 32;
1239                 depth32[1] = 32;
1240                 depth32[2] = 32;
1241                 depth32[3] = depth - 96;
1242                 return 0;
1243
1244         case 4:
1245                 depth32[0] = 32;
1246                 depth32[1] = 32;
1247                 depth32[2] = 32;
1248                 depth32[3] = 32;
1249                 return 0;
1250
1251         default:
1252                 return -1;
1253         }
1254 }
1255
1256 static int
1257 match_convert(struct table_rule_match *mh,
1258         union table_rule_match_low_level *ml,
1259         int add)
1260 {
1261         memset(ml, 0, sizeof(*ml));
1262
1263         switch (mh->match_type) {
1264         case TABLE_ACL:
1265                 if (mh->match.acl.ip_version)
1266                         if (add) {
1267                                 ml->acl_add.field_value[0].value.u8 =
1268                                         mh->match.acl.proto;
1269                                 ml->acl_add.field_value[0].mask_range.u8 =
1270                                         mh->match.acl.proto_mask;
1271
1272                                 ml->acl_add.field_value[1].value.u32 =
1273                                         mh->match.acl.ipv4.sa;
1274                                 ml->acl_add.field_value[1].mask_range.u32 =
1275                                         mh->match.acl.sa_depth;
1276
1277                                 ml->acl_add.field_value[2].value.u32 =
1278                                         mh->match.acl.ipv4.da;
1279                                 ml->acl_add.field_value[2].mask_range.u32 =
1280                                         mh->match.acl.da_depth;
1281
1282                                 ml->acl_add.field_value[3].value.u16 =
1283                                         mh->match.acl.sp0;
1284                                 ml->acl_add.field_value[3].mask_range.u16 =
1285                                         mh->match.acl.sp1;
1286
1287                                 ml->acl_add.field_value[4].value.u16 =
1288                                         mh->match.acl.dp0;
1289                                 ml->acl_add.field_value[4].mask_range.u16 =
1290                                         mh->match.acl.dp1;
1291
1292                                 ml->acl_add.priority =
1293                                         (int32_t) mh->match.acl.priority;
1294                         } else {
1295                                 ml->acl_delete.field_value[0].value.u8 =
1296                                         mh->match.acl.proto;
1297                                 ml->acl_delete.field_value[0].mask_range.u8 =
1298                                         mh->match.acl.proto_mask;
1299
1300                                 ml->acl_delete.field_value[1].value.u32 =
1301                                         mh->match.acl.ipv4.sa;
1302                                 ml->acl_delete.field_value[1].mask_range.u32 =
1303                                         mh->match.acl.sa_depth;
1304
1305                                 ml->acl_delete.field_value[2].value.u32 =
1306                                         mh->match.acl.ipv4.da;
1307                                 ml->acl_delete.field_value[2].mask_range.u32 =
1308                                         mh->match.acl.da_depth;
1309
1310                                 ml->acl_delete.field_value[3].value.u16 =
1311                                         mh->match.acl.sp0;
1312                                 ml->acl_delete.field_value[3].mask_range.u16 =
1313                                         mh->match.acl.sp1;
1314
1315                                 ml->acl_delete.field_value[4].value.u16 =
1316                                         mh->match.acl.dp0;
1317                                 ml->acl_delete.field_value[4].mask_range.u16 =
1318                                         mh->match.acl.dp1;
1319                         }
1320                 else
1321                         if (add) {
1322                                 uint32_t *sa32 =
1323                                         (uint32_t *) mh->match.acl.ipv6.sa;
1324                                 uint32_t *da32 =
1325                                         (uint32_t *) mh->match.acl.ipv6.da;
1326                                 uint32_t sa32_depth[4], da32_depth[4];
1327                                 int status;
1328
1329                                 status = match_convert_ipv6_depth(
1330                                         mh->match.acl.sa_depth,
1331                                         sa32_depth);
1332                                 if (status)
1333                                         return status;
1334
1335                                 status = match_convert_ipv6_depth(
1336                                         mh->match.acl.da_depth,
1337                                         da32_depth);
1338                                 if (status)
1339                                         return status;
1340
1341                                 ml->acl_add.field_value[0].value.u8 =
1342                                         mh->match.acl.proto;
1343                                 ml->acl_add.field_value[0].mask_range.u8 =
1344                                         mh->match.acl.proto_mask;
1345
1346                                 ml->acl_add.field_value[1].value.u32 = sa32[0];
1347                                 ml->acl_add.field_value[1].mask_range.u32 =
1348                                         sa32_depth[0];
1349                                 ml->acl_add.field_value[2].value.u32 = sa32[1];
1350                                 ml->acl_add.field_value[2].mask_range.u32 =
1351                                         sa32_depth[1];
1352                                 ml->acl_add.field_value[3].value.u32 = sa32[2];
1353                                 ml->acl_add.field_value[3].mask_range.u32 =
1354                                         sa32_depth[2];
1355                                 ml->acl_add.field_value[4].value.u32 = sa32[3];
1356                                 ml->acl_add.field_value[4].mask_range.u32 =
1357                                         sa32_depth[3];
1358
1359                                 ml->acl_add.field_value[5].value.u32 = da32[0];
1360                                 ml->acl_add.field_value[5].mask_range.u32 =
1361                                         da32_depth[0];
1362                                 ml->acl_add.field_value[6].value.u32 = da32[1];
1363                                 ml->acl_add.field_value[6].mask_range.u32 =
1364                                         da32_depth[1];
1365                                 ml->acl_add.field_value[7].value.u32 = da32[2];
1366                                 ml->acl_add.field_value[7].mask_range.u32 =
1367                                         da32_depth[2];
1368                                 ml->acl_add.field_value[8].value.u32 = da32[3];
1369                                 ml->acl_add.field_value[8].mask_range.u32 =
1370                                         da32_depth[3];
1371
1372                                 ml->acl_add.field_value[9].value.u16 =
1373                                         mh->match.acl.sp0;
1374                                 ml->acl_add.field_value[9].mask_range.u16 =
1375                                         mh->match.acl.sp1;
1376
1377                                 ml->acl_add.field_value[10].value.u16 =
1378                                         mh->match.acl.dp0;
1379                                 ml->acl_add.field_value[10].mask_range.u16 =
1380                                         mh->match.acl.dp1;
1381
1382                                 ml->acl_add.priority =
1383                                         (int32_t) mh->match.acl.priority;
1384                         } else {
1385                                 uint32_t *sa32 =
1386                                         (uint32_t *) mh->match.acl.ipv6.sa;
1387                                 uint32_t *da32 =
1388                                         (uint32_t *) mh->match.acl.ipv6.da;
1389                                 uint32_t sa32_depth[4], da32_depth[4];
1390                                 int status;
1391
1392                                 status = match_convert_ipv6_depth(
1393                                         mh->match.acl.sa_depth,
1394                                         sa32_depth);
1395                                 if (status)
1396                                         return status;
1397
1398                                 status = match_convert_ipv6_depth(
1399                                         mh->match.acl.da_depth,
1400                                         da32_depth);
1401                                 if (status)
1402                                         return status;
1403
1404                                 ml->acl_delete.field_value[0].value.u8 =
1405                                         mh->match.acl.proto;
1406                                 ml->acl_delete.field_value[0].mask_range.u8 =
1407                                         mh->match.acl.proto_mask;
1408
1409                                 ml->acl_delete.field_value[1].value.u32 =
1410                                         sa32[0];
1411                                 ml->acl_delete.field_value[1].mask_range.u32 =
1412                                         sa32_depth[0];
1413                                 ml->acl_delete.field_value[2].value.u32 =
1414                                         sa32[1];
1415                                 ml->acl_delete.field_value[2].mask_range.u32 =
1416                                         sa32_depth[1];
1417                                 ml->acl_delete.field_value[3].value.u32 =
1418                                         sa32[2];
1419                                 ml->acl_delete.field_value[3].mask_range.u32 =
1420                                         sa32_depth[2];
1421                                 ml->acl_delete.field_value[4].value.u32 =
1422                                         sa32[3];
1423                                 ml->acl_delete.field_value[4].mask_range.u32 =
1424                                         sa32_depth[3];
1425
1426                                 ml->acl_delete.field_value[5].value.u32 =
1427                                         da32[0];
1428                                 ml->acl_delete.field_value[5].mask_range.u32 =
1429                                         da32_depth[0];
1430                                 ml->acl_delete.field_value[6].value.u32 =
1431                                         da32[1];
1432                                 ml->acl_delete.field_value[6].mask_range.u32 =
1433                                         da32_depth[1];
1434                                 ml->acl_delete.field_value[7].value.u32 =
1435                                         da32[2];
1436                                 ml->acl_delete.field_value[7].mask_range.u32 =
1437                                         da32_depth[2];
1438                                 ml->acl_delete.field_value[8].value.u32 =
1439                                         da32[3];
1440                                 ml->acl_delete.field_value[8].mask_range.u32 =
1441                                         da32_depth[3];
1442
1443                                 ml->acl_delete.field_value[9].value.u16 =
1444                                         mh->match.acl.sp0;
1445                                 ml->acl_delete.field_value[9].mask_range.u16 =
1446                                         mh->match.acl.sp1;
1447
1448                                 ml->acl_delete.field_value[10].value.u16 =
1449                                         mh->match.acl.dp0;
1450                                 ml->acl_delete.field_value[10].mask_range.u16 =
1451                                         mh->match.acl.dp1;
1452                         }
1453                 return 0;
1454
1455         case TABLE_ARRAY:
1456                 ml->array.pos = mh->match.array.pos;
1457                 return 0;
1458
1459         case TABLE_HASH:
1460                 memcpy(ml->hash, mh->match.hash.key, sizeof(ml->hash));
1461                 return 0;
1462
1463         case TABLE_LPM:
1464                 if (mh->match.lpm.ip_version) {
1465                         ml->lpm_ipv4.ip = mh->match.lpm.ipv4;
1466                         ml->lpm_ipv4.depth = mh->match.lpm.depth;
1467                 } else {
1468                         memcpy(ml->lpm_ipv6.ip,
1469                                 mh->match.lpm.ipv6, sizeof(ml->lpm_ipv6.ip));
1470                         ml->lpm_ipv6.depth = mh->match.lpm.depth;
1471                 }
1472
1473                 return 0;
1474
1475         default:
1476                 return -1;
1477         }
1478 }
1479
1480 static struct pipeline_msg_rsp *
1481 pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
1482         struct pipeline_msg_req *req)
1483 {
1484         union table_rule_match_low_level match_ll;
1485         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1486         struct table_rule_match *match = &req->table_rule_add.match;
1487         struct table_rule_action *action = &req->table_rule_add.action;
1488         struct rte_pipeline_table_entry *data_in, *data_out;
1489         uint32_t table_id = req->id;
1490         int key_found, status;
1491         struct rte_table_action *a = p->table_data[table_id].a;
1492
1493         /* Apply actions */
1494         memset(p->buffer, 0, sizeof(p->buffer));
1495         data_in = (struct rte_pipeline_table_entry *) p->buffer;
1496
1497         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1498                 status = rte_table_action_apply(a,
1499                         data_in,
1500                         RTE_TABLE_ACTION_FWD,
1501                         &action->fwd);
1502
1503                 if (status) {
1504                         rsp->status = -1;
1505                         return rsp;
1506                 }
1507         }
1508
1509         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1510                 status = rte_table_action_apply(a,
1511                         data_in,
1512                         RTE_TABLE_ACTION_MTR,
1513                         &action->mtr);
1514
1515                 if (status) {
1516                         rsp->status = -1;
1517                         return rsp;
1518                 }
1519         }
1520
1521         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1522                 status = rte_table_action_apply(a,
1523                         data_in,
1524                         RTE_TABLE_ACTION_TM,
1525                         &action->tm);
1526
1527                 if (status) {
1528                         rsp->status = -1;
1529                         return rsp;
1530                 }
1531         }
1532
1533         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1534                 status = rte_table_action_apply(a,
1535                         data_in,
1536                         RTE_TABLE_ACTION_ENCAP,
1537                         &action->encap);
1538
1539                 if (status) {
1540                         rsp->status = -1;
1541                         return rsp;
1542                 }
1543         }
1544
1545         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1546                 status = rte_table_action_apply(a,
1547                         data_in,
1548                         RTE_TABLE_ACTION_NAT,
1549                         &action->nat);
1550
1551                 if (status) {
1552                         rsp->status = -1;
1553                         return rsp;
1554                 }
1555         }
1556
1557         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
1558                 status = rte_table_action_apply(a,
1559                         data_in,
1560                         RTE_TABLE_ACTION_TTL,
1561                         &action->ttl);
1562
1563                 if (status) {
1564                         rsp->status = -1;
1565                         return rsp;
1566                 }
1567         }
1568
1569         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
1570                 status = rte_table_action_apply(a,
1571                         data_in,
1572                         RTE_TABLE_ACTION_STATS,
1573                         &action->stats);
1574
1575                 if (status) {
1576                         rsp->status = -1;
1577                         return rsp;
1578                 }
1579         }
1580
1581         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
1582                 status = rte_table_action_apply(a,
1583                         data_in,
1584                         RTE_TABLE_ACTION_TIME,
1585                         &action->time);
1586
1587                 if (status) {
1588                         rsp->status = -1;
1589                         return rsp;
1590                 }
1591         }
1592
1593         /* Add rule (match, action) to table */
1594         status = match_convert(match, &match_ll, 1);
1595         if (status) {
1596                 rsp->status = -1;
1597                 return rsp;
1598         }
1599
1600         status = rte_pipeline_table_entry_add(p->p,
1601                 table_id,
1602                 &match_ll,
1603                 data_in,
1604                 &key_found,
1605                 &data_out);
1606         if (status) {
1607                 rsp->status = -1;
1608                 return rsp;
1609         }
1610
1611         /* Write response */
1612         rsp->status = 0;
1613         rsp->table_rule_add.data = data_out;
1614
1615         return rsp;
1616 }
1617
1618 static struct pipeline_msg_rsp *
1619 pipeline_msg_handle_table_rule_add_default(struct pipeline_data *p,
1620         struct pipeline_msg_req *req)
1621 {
1622         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1623         struct table_rule_action *action = &req->table_rule_add_default.action;
1624         struct rte_pipeline_table_entry *data_in, *data_out;
1625         uint32_t table_id = req->id;
1626         int status;
1627
1628         /* Apply actions */
1629         memset(p->buffer, 0, sizeof(p->buffer));
1630         data_in = (struct rte_pipeline_table_entry *) p->buffer;
1631
1632         data_in->action = action->fwd.action;
1633         if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
1634                 data_in->port_id = action->fwd.id;
1635         if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
1636                 data_in->table_id = action->fwd.id;
1637
1638         /* Add default rule to table */
1639         status = rte_pipeline_table_default_entry_add(p->p,
1640                 table_id,
1641                 data_in,
1642                 &data_out);
1643         if (status) {
1644                 rsp->status = -1;
1645                 return rsp;
1646         }
1647
1648         /* Write response */
1649         rsp->status = 0;
1650         rsp->table_rule_add_default.data = data_out;
1651
1652         return rsp;
1653 }
1654
1655 static void
1656 pipeline_msg_handle(struct pipeline_data *p)
1657 {
1658         for ( ; ; ) {
1659                 struct pipeline_msg_req *req;
1660                 struct pipeline_msg_rsp *rsp;
1661
1662                 req = pipeline_msg_recv(p->msgq_req);
1663                 if (req == NULL)
1664                         break;
1665
1666                 switch (req->type) {
1667                 case PIPELINE_REQ_PORT_IN_STATS_READ:
1668                         rsp = pipeline_msg_handle_port_in_stats_read(p, req);
1669                         break;
1670
1671                 case PIPELINE_REQ_PORT_IN_ENABLE:
1672                         rsp = pipeline_msg_handle_port_in_enable(p, req);
1673                         break;
1674
1675                 case PIPELINE_REQ_PORT_IN_DISABLE:
1676                         rsp = pipeline_msg_handle_port_in_disable(p, req);
1677                         break;
1678
1679                 case PIPELINE_REQ_PORT_OUT_STATS_READ:
1680                         rsp = pipeline_msg_handle_port_out_stats_read(p, req);
1681                         break;
1682
1683                 case PIPELINE_REQ_TABLE_STATS_READ:
1684                         rsp = pipeline_msg_handle_table_stats_read(p, req);
1685                         break;
1686
1687                 case PIPELINE_REQ_TABLE_RULE_ADD:
1688                         rsp = pipeline_msg_handle_table_rule_add(p, req);
1689                         break;
1690
1691                 case PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT:
1692                         rsp = pipeline_msg_handle_table_rule_add_default(p,     req);
1693                         break;
1694
1695                 default:
1696                         rsp = (struct pipeline_msg_rsp *) req;
1697                         rsp->status = -1;
1698                 }
1699
1700                 pipeline_msg_send(p->msgq_rsp, rsp);
1701         }
1702 }
1703
1704 /**
1705  * Data plane threads: main
1706  */
1707 int
1708 thread_main(void *arg __rte_unused)
1709 {
1710         struct thread_data *t;
1711         uint32_t thread_id, i;
1712
1713         thread_id = rte_lcore_id();
1714         t = &thread_data[thread_id];
1715
1716         /* Dispatch loop */
1717         for (i = 0; ; i++) {
1718                 uint32_t j;
1719
1720                 /* Data Plane */
1721                 for (j = 0; j < t->n_pipelines; j++)
1722                         rte_pipeline_run(t->p[j]);
1723
1724                 /* Control Plane */
1725                 if ((i & 0xF) == 0) {
1726                         uint64_t time = rte_get_tsc_cycles();
1727                         uint64_t time_next_min = UINT64_MAX;
1728
1729                         if (time < t->time_next_min)
1730                                 continue;
1731
1732                         /* Pipeline message queues */
1733                         for (j = 0; j < t->n_pipelines; j++) {
1734                                 struct pipeline_data *p =
1735                                         &t->pipeline_data[j];
1736                                 uint64_t time_next = p->time_next;
1737
1738                                 if (time_next <= time) {
1739                                         pipeline_msg_handle(p);
1740                                         rte_pipeline_flush(p->p);
1741                                         time_next = time + p->timer_period;
1742                                         p->time_next = time_next;
1743                                 }
1744
1745                                 if (time_next < time_next_min)
1746                                         time_next_min = time_next;
1747                         }
1748
1749                         /* Thread message queues */
1750                         {
1751                                 uint64_t time_next = t->time_next;
1752
1753                                 if (time_next <= time) {
1754                                         thread_msg_handle(t);
1755                                         time_next = time + t->timer_period;
1756                                         t->time_next = time_next;
1757                                 }
1758
1759                                 if (time_next < time_next_min)
1760                                         time_next_min = time_next;
1761                         }
1762
1763                         t->time_next_min = time_next_min;
1764                 }
1765         }
1766
1767         return 0;
1768 }