examples/ip_pipeline/thread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdlib.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_cycles.h>
   9 #include <rte_lcore.h>
  10 #include <rte_ring.h>
  11
  12 #include <rte_table_acl.h>
  13 #include <rte_table_array.h>
  14 #include <rte_table_hash.h>
  15 #include <rte_table_lpm.h>
  16 #include <rte_table_lpm_ipv6.h>
  17
  18 #include "common.h"
  19 #include "thread.h"
  20 #include "pipeline.h"
  21
  22 #ifndef THREAD_PIPELINES_MAX
  23 #define THREAD_PIPELINES_MAX                               256
  24 #endif
  25
  26 #ifndef THREAD_MSGQ_SIZE
  27 #define THREAD_MSGQ_SIZE                                   64
  28 #endif
  29
  30 #ifndef THREAD_TIMER_PERIOD_MS
  31 #define THREAD_TIMER_PERIOD_MS                             100
  32 #endif
  33
  34 /**
  35  * Master thead: data plane thread context
  36  */
  37 struct thread {
  38         struct rte_ring *msgq_req;
  39         struct rte_ring *msgq_rsp;
  40
  41         uint32_t enabled;
  42 };
  43
  44 static struct thread thread[RTE_MAX_LCORE];
  45
  46 /**
  47  * Data plane threads: context
  48  */
  49 struct table_data {
  50         struct rte_table_action *a;
  51 };
  52
  53 struct pipeline_data {
  54         struct rte_pipeline *p;
  55         struct table_data table_data[RTE_PIPELINE_TABLE_MAX];
  56         uint32_t n_tables;
  57
  58         struct rte_ring *msgq_req;
  59         struct rte_ring *msgq_rsp;
  60         uint64_t timer_period; /* Measured in CPU cycles. */
  61         uint64_t time_next;
  62
  63         uint8_t buffer[TABLE_RULE_ACTION_SIZE_MAX];
  64 };
  65
  66 struct thread_data {
  67         struct rte_pipeline *p[THREAD_PIPELINES_MAX];
  68         uint32_t n_pipelines;
  69
  70         struct pipeline_data pipeline_data[THREAD_PIPELINES_MAX];
  71         struct rte_ring *msgq_req;
  72         struct rte_ring *msgq_rsp;
  73         uint64_t timer_period; /* Measured in CPU cycles. */
  74         uint64_t time_next;
  75         uint64_t time_next_min;
  76 } __rte_cache_aligned;
  77
  78 static struct thread_data thread_data[RTE_MAX_LCORE];
  79
  80 /**
  81  * Master thread: data plane thread init
  82  */
  83 static void
  84 thread_free(void)
  85 {
  86         uint32_t i;
  87
  88         for (i = 0; i < RTE_MAX_LCORE; i++) {
  89                 struct thread *t = &thread[i];
  90
  91                 if (!rte_lcore_is_enabled(i))
  92                         continue;
  93
  94                 /* MSGQs */
  95                 if (t->msgq_req)
  96                         rte_ring_free(t->msgq_req);
  97
  98                 if (t->msgq_rsp)
  99                         rte_ring_free(t->msgq_rsp);
 100         }
 101 }
 102
 103 int
 104 thread_init(void)
 105 {
 106         uint32_t i;
 107
 108         RTE_LCORE_FOREACH_SLAVE(i) {
 109                 char name[NAME_MAX];
 110                 struct rte_ring *msgq_req, *msgq_rsp;
 111                 struct thread *t = &thread[i];
 112                 struct thread_data *t_data = &thread_data[i];
 113                 uint32_t cpu_id = rte_lcore_to_socket_id(i);
 114
 115                 /* MSGQs */
 116                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-REQ", i);
 117
 118                 msgq_req = rte_ring_create(name,
 119                         THREAD_MSGQ_SIZE,
 120                         cpu_id,
 121                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 122
 123                 if (msgq_req == NULL) {
 124                         thread_free();
 125                         return -1;
 126                 }
 127
 128                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-RSP", i);
 129
 130                 msgq_rsp = rte_ring_create(name,
 131                         THREAD_MSGQ_SIZE,
 132                         cpu_id,
 133                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 134
 135                 if (msgq_rsp == NULL) {
 136                         thread_free();
 137                         return -1;
 138                 }
 139
 140                 /* Master thread records */
 141                 t->msgq_req = msgq_req;
 142                 t->msgq_rsp = msgq_rsp;
 143                 t->enabled = 1;
 144
 145                 /* Data plane thread records */
 146                 t_data->n_pipelines = 0;
 147                 t_data->msgq_req = msgq_req;
 148                 t_data->msgq_rsp = msgq_rsp;
 149                 t_data->timer_period =
 150                         (rte_get_tsc_hz() * THREAD_TIMER_PERIOD_MS) / 1000;
 151                 t_data->time_next = rte_get_tsc_cycles() + t_data->timer_period;
 152                 t_data->time_next_min = t_data->time_next;
 153         }
 154
 155         return 0;
 156 }
 157
 158 /**
 159  * Master thread & data plane threads: message passing
 160  */
 161 enum thread_req_type {
 162         THREAD_REQ_PIPELINE_ENABLE = 0,
 163         THREAD_REQ_PIPELINE_DISABLE,
 164         THREAD_REQ_MAX
 165 };
 166
 167 struct thread_msg_req {
 168         enum thread_req_type type;
 169
 170         union {
 171                 struct {
 172                         struct rte_pipeline *p;
 173                         struct {
 174                                 struct rte_table_action *a;
 175                         } table[RTE_PIPELINE_TABLE_MAX];
 176                         struct rte_ring *msgq_req;
 177                         struct rte_ring *msgq_rsp;
 178                         uint32_t timer_period_ms;
 179                         uint32_t n_tables;
 180                 } pipeline_enable;
 181
 182                 struct {
 183                         struct rte_pipeline *p;
 184                 } pipeline_disable;
 185         };
 186 };
 187
 188 struct thread_msg_rsp {
 189         int status;
 190 };
 191
 192 /**
 193  * Master thread
 194  */
 195 static struct thread_msg_req *
 196 thread_msg_alloc(void)
 197 {
 198         size_t size = RTE_MAX(sizeof(struct thread_msg_req),
 199                 sizeof(struct thread_msg_rsp));
 200
 201         return calloc(1, size);
 202 }
 203
 204 static void
 205 thread_msg_free(struct thread_msg_rsp *rsp)
 206 {
 207         free(rsp);
 208 }
 209
 210 static struct thread_msg_rsp *
 211 thread_msg_send_recv(uint32_t thread_id,
 212         struct thread_msg_req *req)
 213 {
 214         struct thread *t = &thread[thread_id];
 215         struct rte_ring *msgq_req = t->msgq_req;
 216         struct rte_ring *msgq_rsp = t->msgq_rsp;
 217         struct thread_msg_rsp *rsp;
 218         int status;
 219
 220         /* send */
 221         do {
 222                 status = rte_ring_sp_enqueue(msgq_req, req);
 223         } while (status == -ENOBUFS);
 224
 225         /* recv */
 226         do {
 227                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 228         } while (status != 0);
 229
 230         return rsp;
 231 }
 232
 233 int
 234 thread_pipeline_enable(uint32_t thread_id,
 235         const char *pipeline_name)
 236 {
 237         struct pipeline *p = pipeline_find(pipeline_name);
 238         struct thread *t;
 239         struct thread_msg_req *req;
 240         struct thread_msg_rsp *rsp;
 241         uint32_t i;
 242         int status;
 243
 244         /* Check input params */
 245         if ((thread_id >= RTE_MAX_LCORE) ||
 246                 (p == NULL) ||
 247                 (p->n_ports_in == 0) ||
 248                 (p->n_ports_out == 0) ||
 249                 (p->n_tables == 0))
 250                 return -1;
 251
 252         t = &thread[thread_id];
 253         if ((t->enabled == 0) ||
 254                 p->enabled)
 255                 return -1;
 256
 257         /* Allocate request */
 258         req = thread_msg_alloc();
 259         if (req == NULL)
 260                 return -1;
 261
 262         /* Write request */
 263         req->type = THREAD_REQ_PIPELINE_ENABLE;
 264         req->pipeline_enable.p = p->p;
 265         for (i = 0; i < p->n_tables; i++)
 266                 req->pipeline_enable.table[i].a =
 267                         p->table[i].a;
 268         req->pipeline_enable.msgq_req = p->msgq_req;
 269         req->pipeline_enable.msgq_rsp = p->msgq_rsp;
 270         req->pipeline_enable.timer_period_ms = p->timer_period_ms;
 271         req->pipeline_enable.n_tables = p->n_tables;
 272
 273         /* Send request and wait for response */
 274         rsp = thread_msg_send_recv(thread_id, req);
 275         if (rsp == NULL)
 276                 return -1;
 277
 278         /* Read response */
 279         status = rsp->status;
 280
 281         /* Free response */
 282         thread_msg_free(rsp);
 283
 284         /* Request completion */
 285         if (status)
 286                 return status;
 287
 288         p->thread_id = thread_id;
 289         p->enabled = 1;
 290
 291         return 0;
 292 }
 293
 294 int
 295 thread_pipeline_disable(uint32_t thread_id,
 296         const char *pipeline_name)
 297 {
 298         struct pipeline *p = pipeline_find(pipeline_name);
 299         struct thread *t;
 300         struct thread_msg_req *req;
 301         struct thread_msg_rsp *rsp;
 302         int status;
 303
 304         /* Check input params */
 305         if ((thread_id >= RTE_MAX_LCORE) ||
 306                 (p == NULL))
 307                 return -1;
 308
 309         t = &thread[thread_id];
 310         if (t->enabled == 0)
 311                 return -1;
 312
 313         if (p->enabled == 0)
 314                 return 0;
 315
 316         if (p->thread_id != thread_id)
 317                 return -1;
 318
 319         /* Allocate request */
 320         req = thread_msg_alloc();
 321         if (req == NULL)
 322                 return -1;
 323
 324         /* Write request */
 325         req->type = THREAD_REQ_PIPELINE_DISABLE;
 326         req->pipeline_disable.p = p->p;
 327
 328         /* Send request and wait for response */
 329         rsp = thread_msg_send_recv(thread_id, req);
 330         if (rsp == NULL)
 331                 return -1;
 332
 333         /* Read response */
 334         status = rsp->status;
 335
 336         /* Free response */
 337         thread_msg_free(rsp);
 338
 339         /* Request completion */
 340         if (status)
 341                 return status;
 342
 343         p->enabled = 0;
 344
 345         return 0;
 346 }
 347
 348 /**
 349  * Data plane threads: message handling
 350  */
 351 static inline struct thread_msg_req *
 352 thread_msg_recv(struct rte_ring *msgq_req)
 353 {
 354         struct thread_msg_req *req;
 355
 356         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
 357
 358         if (status != 0)
 359                 return NULL;
 360
 361         return req;
 362 }
 363
 364 static inline void
 365 thread_msg_send(struct rte_ring *msgq_rsp,
 366         struct thread_msg_rsp *rsp)
 367 {
 368         int status;
 369
 370         do {
 371                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
 372         } while (status == -ENOBUFS);
 373 }
 374
 375 static struct thread_msg_rsp *
 376 thread_msg_handle_pipeline_enable(struct thread_data *t,
 377         struct thread_msg_req *req)
 378 {
 379         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 380         struct pipeline_data *p = &t->pipeline_data[t->n_pipelines];
 381         uint32_t i;
 382
 383         /* Request */
 384         if (t->n_pipelines >= THREAD_PIPELINES_MAX) {
 385                 rsp->status = -1;
 386                 return rsp;
 387         }
 388
 389         t->p[t->n_pipelines] = req->pipeline_enable.p;
 390
 391         p->p = req->pipeline_enable.p;
 392         for (i = 0; i < req->pipeline_enable.n_tables; i++)
 393                 p->table_data[i].a =
 394                         req->pipeline_enable.table[i].a;
 395
 396         p->n_tables = req->pipeline_enable.n_tables;
 397
 398         p->msgq_req = req->pipeline_enable.msgq_req;
 399         p->msgq_rsp = req->pipeline_enable.msgq_rsp;
 400         p->timer_period =
 401                 (rte_get_tsc_hz() * req->pipeline_enable.timer_period_ms) / 1000;
 402         p->time_next = rte_get_tsc_cycles() + p->timer_period;
 403
 404         t->n_pipelines++;
 405
 406         /* Response */
 407         rsp->status = 0;
 408         return rsp;
 409 }
 410
 411 static struct thread_msg_rsp *
 412 thread_msg_handle_pipeline_disable(struct thread_data *t,
 413         struct thread_msg_req *req)
 414 {
 415         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 416         uint32_t n_pipelines = t->n_pipelines;
 417         struct rte_pipeline *pipeline = req->pipeline_disable.p;
 418         uint32_t i;
 419
 420         /* find pipeline */
 421         for (i = 0; i < n_pipelines; i++) {
 422                 struct pipeline_data *p = &t->pipeline_data[i];
 423
 424                 if (p->p != pipeline)
 425                         continue;
 426
 427                 if (i < n_pipelines - 1) {
 428                         struct rte_pipeline *pipeline_last =
 429                                 t->p[n_pipelines - 1];
 430                         struct pipeline_data *p_last =
 431                                 &t->pipeline_data[n_pipelines - 1];
 432
 433                         t->p[i] = pipeline_last;
 434                         memcpy(p, p_last, sizeof(*p));
 435                 }
 436
 437                 t->n_pipelines--;
 438
 439                 rsp->status = 0;
 440                 return rsp;
 441         }
 442
 443         /* should not get here */
 444         rsp->status = 0;
 445         return rsp;
 446 }
 447
 448 static void
 449 thread_msg_handle(struct thread_data *t)
 450 {
 451         for ( ; ; ) {
 452                 struct thread_msg_req *req;
 453                 struct thread_msg_rsp *rsp;
 454
 455                 req = thread_msg_recv(t->msgq_req);
 456                 if (req == NULL)
 457                         break;
 458
 459                 switch (req->type) {
 460                 case THREAD_REQ_PIPELINE_ENABLE:
 461                         rsp = thread_msg_handle_pipeline_enable(t, req);
 462                         break;
 463
 464                 case THREAD_REQ_PIPELINE_DISABLE:
 465                         rsp = thread_msg_handle_pipeline_disable(t, req);
 466                         break;
 467
 468                 default:
 469                         rsp = (struct thread_msg_rsp *) req;
 470                         rsp->status = -1;
 471                 }
 472
 473                 thread_msg_send(t->msgq_rsp, rsp);
 474         }
 475 }
 476
 477 /**
 478  * Master thread & data plane threads: message passing
 479  */
 480 enum pipeline_req_type {
 481         /* Port IN */
 482         PIPELINE_REQ_PORT_IN_STATS_READ,
 483         PIPELINE_REQ_PORT_IN_ENABLE,
 484         PIPELINE_REQ_PORT_IN_DISABLE,
 485
 486         /* Port OUT */
 487         PIPELINE_REQ_PORT_OUT_STATS_READ,
 488
 489         /* Table */
 490         PIPELINE_REQ_TABLE_STATS_READ,
 491         PIPELINE_REQ_TABLE_RULE_ADD,
 492         PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT,
 493         PIPELINE_REQ_TABLE_RULE_ADD_BULK,
 494         PIPELINE_REQ_TABLE_RULE_DELETE,
 495         PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT,
 496         PIPELINE_REQ_MAX
 497 };
 498
 499 struct pipeline_msg_req_port_in_stats_read {
 500         int clear;
 501 };
 502
 503 struct pipeline_msg_req_port_out_stats_read {
 504         int clear;
 505 };
 506
 507 struct pipeline_msg_req_table_stats_read {
 508         int clear;
 509 };
 510
 511 struct pipeline_msg_req_table_rule_add {
 512         struct table_rule_match match;
 513         struct table_rule_action action;
 514 };
 515
 516 struct pipeline_msg_req_table_rule_add_default {
 517         struct table_rule_action action;
 518 };
 519
 520 struct pipeline_msg_req_table_rule_add_bulk {
 521         struct table_rule_match *match;
 522         struct table_rule_action *action;
 523         void **data;
 524         uint32_t n_rules;
 525         int bulk;
 526 };
 527
 528 struct pipeline_msg_req_table_rule_delete {
 529         struct table_rule_match match;
 530 };
 531
 532 struct pipeline_msg_req {
 533         enum pipeline_req_type type;
 534         uint32_t id; /* Port IN, port OUT or table ID */
 535
 536         RTE_STD_C11
 537         union {
 538                 struct pipeline_msg_req_port_in_stats_read port_in_stats_read;
 539                 struct pipeline_msg_req_port_out_stats_read port_out_stats_read;
 540                 struct pipeline_msg_req_table_stats_read table_stats_read;
 541                 struct pipeline_msg_req_table_rule_add table_rule_add;
 542                 struct pipeline_msg_req_table_rule_add_default table_rule_add_default;
 543                 struct pipeline_msg_req_table_rule_add_bulk table_rule_add_bulk;
 544                 struct pipeline_msg_req_table_rule_delete table_rule_delete;
 545         };
 546 };
 547
 548 struct pipeline_msg_rsp_port_in_stats_read {
 549         struct rte_pipeline_port_in_stats stats;
 550 };
 551
 552 struct pipeline_msg_rsp_port_out_stats_read {
 553         struct rte_pipeline_port_out_stats stats;
 554 };
 555
 556 struct pipeline_msg_rsp_table_stats_read {
 557         struct rte_pipeline_table_stats stats;
 558 };
 559
 560 struct pipeline_msg_rsp_table_rule_add {
 561         void *data;
 562 };
 563
 564 struct pipeline_msg_rsp_table_rule_add_default {
 565         void *data;
 566 };
 567
 568 struct pipeline_msg_rsp_table_rule_add_bulk {
 569         uint32_t n_rules;
 570 };
 571
 572 struct pipeline_msg_rsp {
 573         int status;
 574
 575         RTE_STD_C11
 576         union {
 577                 struct pipeline_msg_rsp_port_in_stats_read port_in_stats_read;
 578                 struct pipeline_msg_rsp_port_out_stats_read port_out_stats_read;
 579                 struct pipeline_msg_rsp_table_stats_read table_stats_read;
 580                 struct pipeline_msg_rsp_table_rule_add table_rule_add;
 581                 struct pipeline_msg_rsp_table_rule_add_default table_rule_add_default;
 582                 struct pipeline_msg_rsp_table_rule_add_bulk table_rule_add_bulk;
 583         };
 584 };
 585
 586 /**
 587  * Master thread
 588  */
 589 static struct pipeline_msg_req *
 590 pipeline_msg_alloc(void)
 591 {
 592         size_t size = RTE_MAX(sizeof(struct pipeline_msg_req),
 593                 sizeof(struct pipeline_msg_rsp));
 594
 595         return calloc(1, size);
 596 }
 597
 598 static void
 599 pipeline_msg_free(struct pipeline_msg_rsp *rsp)
 600 {
 601         free(rsp);
 602 }
 603
 604 static struct pipeline_msg_rsp *
 605 pipeline_msg_send_recv(struct pipeline *p,
 606         struct pipeline_msg_req *req)
 607 {
 608         struct rte_ring *msgq_req = p->msgq_req;
 609         struct rte_ring *msgq_rsp = p->msgq_rsp;
 610         struct pipeline_msg_rsp *rsp;
 611         int status;
 612
 613         /* send */
 614         do {
 615                 status = rte_ring_sp_enqueue(msgq_req, req);
 616         } while (status == -ENOBUFS);
 617
 618         /* recv */
 619         do {
 620                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 621         } while (status != 0);
 622
 623         return rsp;
 624 }
 625
 626 int
 627 pipeline_port_in_stats_read(const char *pipeline_name,
 628         uint32_t port_id,
 629         struct rte_pipeline_port_in_stats *stats,
 630         int clear)
 631 {
 632         struct pipeline *p;
 633         struct pipeline_msg_req *req;
 634         struct pipeline_msg_rsp *rsp;
 635         int status;
 636
 637         /* Check input params */
 638         if ((pipeline_name == NULL) ||
 639                 (stats == NULL))
 640                 return -1;
 641
 642         p = pipeline_find(pipeline_name);
 643         if ((p == NULL) ||
 644                 (p->enabled == 0) ||
 645                 (port_id >= p->n_ports_in))
 646                 return -1;
 647
 648         /* Allocate request */
 649         req = pipeline_msg_alloc();
 650         if (req == NULL)
 651                 return -1;
 652
 653         /* Write request */
 654         req->type = PIPELINE_REQ_PORT_IN_STATS_READ;
 655         req->id = port_id;
 656         req->port_in_stats_read.clear = clear;
 657
 658         /* Send request and wait for response */
 659         rsp = pipeline_msg_send_recv(p, req);
 660         if (rsp == NULL)
 661                 return -1;
 662
 663         /* Read response */
 664         status = rsp->status;
 665         if (status)
 666                 memcpy(stats, &rsp->port_in_stats_read.stats, sizeof(*stats));
 667
 668         /* Free response */
 669         pipeline_msg_free(rsp);
 670
 671         return status;
 672 }
 673
 674 int
 675 pipeline_port_in_enable(const char *pipeline_name,
 676         uint32_t port_id)
 677 {
 678         struct pipeline *p;
 679         struct pipeline_msg_req *req;
 680         struct pipeline_msg_rsp *rsp;
 681         int status;
 682
 683         /* Check input params */
 684         if (pipeline_name == NULL)
 685                 return -1;
 686
 687         p = pipeline_find(pipeline_name);
 688         if ((p == NULL) ||
 689                 (p->enabled == 0) ||
 690                 (port_id >= p->n_ports_in))
 691                 return -1;
 692
 693         /* Allocate request */
 694         req = pipeline_msg_alloc();
 695         if (req == NULL)
 696                 return -1;
 697
 698         /* Write request */
 699         req->type = PIPELINE_REQ_PORT_IN_ENABLE;
 700         req->id = port_id;
 701
 702         /* Send request and wait for response */
 703         rsp = pipeline_msg_send_recv(p, req);
 704         if (rsp == NULL)
 705                 return -1;
 706
 707         /* Read response */
 708         status = rsp->status;
 709
 710         /* Free response */
 711         pipeline_msg_free(rsp);
 712
 713         return status;
 714 }
 715
 716 int
 717 pipeline_port_in_disable(const char *pipeline_name,
 718         uint32_t port_id)
 719 {
 720         struct pipeline *p;
 721         struct pipeline_msg_req *req;
 722         struct pipeline_msg_rsp *rsp;
 723         int status;
 724
 725         /* Check input params */
 726         if (pipeline_name == NULL)
 727                 return -1;
 728
 729         p = pipeline_find(pipeline_name);
 730         if ((p == NULL) ||
 731                 (p->enabled == 0) ||
 732                 (port_id >= p->n_ports_in))
 733                 return -1;
 734
 735         /* Allocate request */
 736         req = pipeline_msg_alloc();
 737         if (req == NULL)
 738                 return -1;
 739
 740         /* Write request */
 741         req->type = PIPELINE_REQ_PORT_IN_DISABLE;
 742         req->id = port_id;
 743
 744         /* Send request and wait for response */
 745         rsp = pipeline_msg_send_recv(p, req);
 746         if (rsp == NULL)
 747                 return -1;
 748
 749         /* Read response */
 750         status = rsp->status;
 751
 752         /* Free response */
 753         pipeline_msg_free(rsp);
 754
 755         return status;
 756 }
 757
 758 int
 759 pipeline_port_out_stats_read(const char *pipeline_name,
 760         uint32_t port_id,
 761         struct rte_pipeline_port_out_stats *stats,
 762         int clear)
 763 {
 764         struct pipeline *p;
 765         struct pipeline_msg_req *req;
 766         struct pipeline_msg_rsp *rsp;
 767         int status;
 768
 769         /* Check input params */
 770         if ((pipeline_name == NULL) ||
 771                 (stats == NULL))
 772                 return -1;
 773
 774         p = pipeline_find(pipeline_name);
 775         if ((p == NULL) ||
 776                 (p->enabled == 0) ||
 777                 (port_id >= p->n_ports_out))
 778                 return -1;
 779
 780         /* Allocate request */
 781         req = pipeline_msg_alloc();
 782         if (req == NULL)
 783                 return -1;
 784
 785         /* Write request */
 786         req->type = PIPELINE_REQ_PORT_OUT_STATS_READ;
 787         req->id = port_id;
 788         req->port_out_stats_read.clear = clear;
 789
 790         /* Send request and wait for response */
 791         rsp = pipeline_msg_send_recv(p, req);
 792         if (rsp == NULL)
 793                 return -1;
 794
 795         /* Read response */
 796         status = rsp->status;
 797         if (status)
 798                 memcpy(stats, &rsp->port_out_stats_read.stats, sizeof(*stats));
 799
 800         /* Free response */
 801         pipeline_msg_free(rsp);
 802
 803         return status;
 804 }
 805
 806 int
 807 pipeline_table_stats_read(const char *pipeline_name,
 808         uint32_t table_id,
 809         struct rte_pipeline_table_stats *stats,
 810         int clear)
 811 {
 812         struct pipeline *p;
 813         struct pipeline_msg_req *req;
 814         struct pipeline_msg_rsp *rsp;
 815         int status;
 816
 817         /* Check input params */
 818         if ((pipeline_name == NULL) ||
 819                 (stats == NULL))
 820                 return -1;
 821
 822         p = pipeline_find(pipeline_name);
 823         if ((p == NULL) ||
 824                 (p->enabled == 0) ||
 825                 (table_id >= p->n_tables))
 826                 return -1;
 827
 828         /* Allocate request */
 829         req = pipeline_msg_alloc();
 830         if (req == NULL)
 831                 return -1;
 832
 833         /* Write request */
 834         req->type = PIPELINE_REQ_TABLE_STATS_READ;
 835         req->id = table_id;
 836         req->table_stats_read.clear = clear;
 837
 838         /* Send request and wait for response */
 839         rsp = pipeline_msg_send_recv(p, req);
 840         if (rsp == NULL)
 841                 return -1;
 842
 843         /* Read response */
 844         status = rsp->status;
 845         if (status)
 846                 memcpy(stats, &rsp->table_stats_read.stats, sizeof(*stats));
 847
 848         /* Free response */
 849         pipeline_msg_free(rsp);
 850
 851         return status;
 852 }
 853
 854 static int
 855 match_check(struct table_rule_match *match,
 856         struct pipeline *p,
 857         uint32_t table_id)
 858 {
 859         struct table *table;
 860
 861         if ((match == NULL) ||
 862                 (p == NULL) ||
 863                 (table_id >= p->n_tables))
 864                 return -1;
 865
 866         table = &p->table[table_id];
 867         if (match->match_type != table->params.match_type)
 868                 return -1;
 869
 870         switch (match->match_type) {
 871         case TABLE_ACL:
 872         {
 873                 struct table_acl_params *t = &table->params.match.acl;
 874                 struct table_rule_match_acl *r = &match->match.acl;
 875
 876                 if ((r->ip_version && (t->ip_version == 0)) ||
 877                         ((r->ip_version == 0) && t->ip_version))
 878                         return -1;
 879
 880                 if (r->ip_version) {
 881                         if ((r->sa_depth > 32) ||
 882                                 (r->da_depth > 32))
 883                                 return -1;
 884                 } else {
 885                         if ((r->sa_depth > 128) ||
 886                                 (r->da_depth > 128))
 887                                 return -1;
 888                 }
 889                 return 0;
 890         }
 891
 892         case TABLE_ARRAY:
 893                 return 0;
 894
 895         case TABLE_HASH:
 896                 return 0;
 897
 898         case TABLE_LPM:
 899         {
 900                 struct table_lpm_params *t = &table->params.match.lpm;
 901                 struct table_rule_match_lpm *r = &match->match.lpm;
 902
 903                 if ((r->ip_version && (t->key_size != 4)) ||
 904                         ((r->ip_version == 0) && (t->key_size != 16)))
 905                         return -1;
 906
 907                 if (r->ip_version) {
 908                         if (r->depth > 32)
 909                                 return -1;
 910                 } else {
 911                         if (r->depth > 128)
 912                                 return -1;
 913                 }
 914                 return 0;
 915         }
 916
 917         case TABLE_STUB:
 918                 return -1;
 919
 920         default:
 921                 return -1;
 922         }
 923 }
 924
 925 static int
 926 action_check(struct table_rule_action *action,
 927         struct pipeline *p,
 928         uint32_t table_id)
 929 {
 930         struct table_action_profile *ap;
 931
 932         if ((action == NULL) ||
 933                 (p == NULL) ||
 934                 (table_id >= p->n_tables))
 935                 return -1;
 936
 937         ap = p->table[table_id].ap;
 938         if (action->action_mask != ap->params.action_mask)
 939                 return -1;
 940
 941         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
 942                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
 943                         (action->fwd.id >= p->n_ports_out))
 944                         return -1;
 945
 946                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
 947                         (action->fwd.id >= p->n_tables))
 948                         return -1;
 949         }
 950
 951         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
 952                 uint32_t tc_mask0 = (1 << ap->params.mtr.n_tc) - 1;
 953                 uint32_t tc_mask1 = action->mtr.tc_mask;
 954
 955                 if (tc_mask1 != tc_mask0)
 956                         return -1;
 957         }
 958
 959         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
 960                 uint32_t n_subports_per_port =
 961                         ap->params.tm.n_subports_per_port;
 962                 uint32_t n_pipes_per_subport =
 963                         ap->params.tm.n_pipes_per_subport;
 964                 uint32_t subport_id = action->tm.subport_id;
 965                 uint32_t pipe_id = action->tm.pipe_id;
 966
 967                 if ((subport_id >= n_subports_per_port) ||
 968                         (pipe_id >= n_pipes_per_subport))
 969                         return -1;
 970         }
 971
 972         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
 973                 uint64_t encap_mask = ap->params.encap.encap_mask;
 974                 enum rte_table_action_encap_type type = action->encap.type;
 975
 976                 if ((encap_mask & (1LLU << type)) == 0)
 977                         return -1;
 978         }
 979
 980         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
 981                 int ip_version0 = ap->params.common.ip_version;
 982                 int ip_version1 = action->nat.ip_version;
 983
 984                 if ((ip_version1 && (ip_version0 == 0)) ||
 985                         ((ip_version1 == 0) && ip_version0))
 986                         return -1;
 987         }
 988
 989         return 0;
 990 }
 991
 992 static int
 993 action_default_check(struct table_rule_action *action,
 994         struct pipeline *p,
 995         uint32_t table_id)
 996 {
 997         if ((action == NULL) ||
 998                 (action->action_mask != (1LLU << RTE_TABLE_ACTION_FWD)) ||
 999                 (p == NULL) ||
1000                 (table_id >= p->n_tables))
1001                 return -1;
1002
1003         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1004                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
1005                         (action->fwd.id >= p->n_ports_out))
1006                         return -1;
1007
1008                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
1009                         (action->fwd.id >= p->n_tables))
1010                         return -1;
1011         }
1012
1013         return 0;
1014 }
1015
1016 int
1017 pipeline_table_rule_add(const char *pipeline_name,
1018         uint32_t table_id,
1019         struct table_rule_match *match,
1020         struct table_rule_action *action,
1021         void **data)
1022 {
1023         struct pipeline *p;
1024         struct pipeline_msg_req *req;
1025         struct pipeline_msg_rsp *rsp;
1026         int status;
1027
1028         /* Check input params */
1029         if ((pipeline_name == NULL) ||
1030                 (match == NULL) ||
1031                 (action == NULL) ||
1032                 (data == NULL))
1033                 return -1;
1034
1035         p = pipeline_find(pipeline_name);
1036         if ((p == NULL) ||
1037                 (p->enabled == 0) ||
1038                 (table_id >= p->n_tables) ||
1039                 match_check(match, p, table_id) ||
1040                 action_check(action, p, table_id))
1041                 return -1;
1042
1043         /* Allocate request */
1044         req = pipeline_msg_alloc();
1045         if (req == NULL)
1046                 return -1;
1047
1048         /* Write request */
1049         req->type = PIPELINE_REQ_TABLE_RULE_ADD;
1050         req->id = table_id;
1051         memcpy(&req->table_rule_add.match, match, sizeof(*match));
1052         memcpy(&req->table_rule_add.action, action, sizeof(*action));
1053
1054         /* Send request and wait for response */
1055         rsp = pipeline_msg_send_recv(p, req);
1056         if (rsp == NULL)
1057                 return -1;
1058
1059         /* Read response */
1060         status = rsp->status;
1061         if (status == 0)
1062                 *data = rsp->table_rule_add.data;
1063
1064         /* Free response */
1065         pipeline_msg_free(rsp);
1066
1067         return status;
1068 }
1069
1070 int
1071 pipeline_table_rule_add_default(const char *pipeline_name,
1072         uint32_t table_id,
1073         struct table_rule_action *action,
1074         void **data)
1075 {
1076         struct pipeline *p;
1077         struct pipeline_msg_req *req;
1078         struct pipeline_msg_rsp *rsp;
1079         int status;
1080
1081         /* Check input params */
1082         if ((pipeline_name == NULL) ||
1083                 (action == NULL) ||
1084                 (data == NULL))
1085                 return -1;
1086
1087         p = pipeline_find(pipeline_name);
1088         if ((p == NULL) ||
1089                 (p->enabled == 0) ||
1090                 (table_id >= p->n_tables) ||
1091                 action_default_check(action, p, table_id))
1092                 return -1;
1093
1094         /* Allocate request */
1095         req = pipeline_msg_alloc();
1096         if (req == NULL)
1097                 return -1;
1098
1099         /* Write request */
1100         req->type = PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT;
1101         req->id = table_id;
1102         memcpy(&req->table_rule_add_default.action, action, sizeof(*action));
1103
1104         /* Send request and wait for response */
1105         rsp = pipeline_msg_send_recv(p, req);
1106         if (rsp == NULL)
1107                 return -1;
1108
1109         /* Read response */
1110         status = rsp->status;
1111         if (status == 0)
1112                 *data = rsp->table_rule_add_default.data;
1113
1114         /* Free response */
1115         pipeline_msg_free(rsp);
1116
1117         return status;
1118 }
1119
1120 int
1121 pipeline_table_rule_add_bulk(const char *pipeline_name,
1122         uint32_t table_id,
1123         struct table_rule_match *match,
1124         struct table_rule_action *action,
1125         void **data,
1126         uint32_t *n_rules)
1127 {
1128         struct pipeline *p;
1129         struct pipeline_msg_req *req;
1130         struct pipeline_msg_rsp *rsp;
1131         uint32_t i;
1132         int status;
1133
1134         /* Check input params */
1135         if ((pipeline_name == NULL) ||
1136                 (match == NULL) ||
1137                 (action == NULL) ||
1138                 (data == NULL) ||
1139                 (n_rules == NULL) ||
1140                 (*n_rules == 0))
1141                 return -1;
1142
1143         p = pipeline_find(pipeline_name);
1144         if ((p == NULL) ||
1145                 (p->enabled == 0) ||
1146                 (table_id >= p->n_tables))
1147                 return -1;
1148
1149         for (i = 0; i < *n_rules; i++)
1150                 if (match_check(match, p, table_id) ||
1151                         action_check(action, p, table_id))
1152                         return -1;
1153
1154         /* Allocate request */
1155         req = pipeline_msg_alloc();
1156         if (req == NULL)
1157                 return -1;
1158
1159         /* Write request */
1160         req->type = PIPELINE_REQ_TABLE_RULE_ADD_BULK;
1161         req->id = table_id;
1162         req->table_rule_add_bulk.match = match;
1163         req->table_rule_add_bulk.action = action;
1164         req->table_rule_add_bulk.data = data;
1165         req->table_rule_add_bulk.n_rules = *n_rules;
1166         req->table_rule_add_bulk.bulk =
1167                 (p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
1168
1169         /* Send request and wait for response */
1170         rsp = pipeline_msg_send_recv(p, req);
1171         if (rsp == NULL)
1172                 return -1;
1173
1174         /* Read response */
1175         status = rsp->status;
1176         if (status == 0)
1177                 *n_rules = rsp->table_rule_add_bulk.n_rules;
1178
1179         /* Free response */
1180         pipeline_msg_free(rsp);
1181
1182         return status;
1183 }
1184
1185 int
1186 pipeline_table_rule_delete(const char *pipeline_name,
1187         uint32_t table_id,
1188         struct table_rule_match *match)
1189 {
1190         struct pipeline *p;
1191         struct pipeline_msg_req *req;
1192         struct pipeline_msg_rsp *rsp;
1193         int status;
1194
1195         /* Check input params */
1196         if ((pipeline_name == NULL) ||
1197                 (match == NULL))
1198                 return -1;
1199
1200         p = pipeline_find(pipeline_name);
1201         if ((p == NULL) ||
1202                 (p->enabled == 0) ||
1203                 (table_id >= p->n_tables) ||
1204                 match_check(match, p, table_id))
1205                 return -1;
1206
1207         /* Allocate request */
1208         req = pipeline_msg_alloc();
1209         if (req == NULL)
1210                 return -1;
1211
1212         /* Write request */
1213         req->type = PIPELINE_REQ_TABLE_RULE_DELETE;
1214         req->id = table_id;
1215         memcpy(&req->table_rule_delete.match, match, sizeof(*match));
1216
1217         /* Send request and wait for response */
1218         rsp = pipeline_msg_send_recv(p, req);
1219         if (rsp == NULL)
1220                 return -1;
1221
1222         /* Read response */
1223         status = rsp->status;
1224
1225         /* Free response */
1226         pipeline_msg_free(rsp);
1227
1228         return status;
1229 }
1230
1231 int
1232 pipeline_table_rule_delete_default(const char *pipeline_name,
1233         uint32_t table_id)
1234 {
1235         struct pipeline *p;
1236         struct pipeline_msg_req *req;
1237         struct pipeline_msg_rsp *rsp;
1238         int status;
1239
1240         /* Check input params */
1241         if (pipeline_name == NULL)
1242                 return -1;
1243
1244         p = pipeline_find(pipeline_name);
1245         if ((p == NULL) ||
1246                 (p->enabled == 0) ||
1247                 (table_id >= p->n_tables))
1248                 return -1;
1249
1250         /* Allocate request */
1251         req = pipeline_msg_alloc();
1252         if (req == NULL)
1253                 return -1;
1254
1255         /* Write request */
1256         req->type = PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT;
1257         req->id = table_id;
1258
1259         /* Send request and wait for response */
1260         rsp = pipeline_msg_send_recv(p, req);
1261         if (rsp == NULL)
1262                 return -1;
1263
1264         /* Read response */
1265         status = rsp->status;
1266
1267         /* Free response */
1268         pipeline_msg_free(rsp);
1269
1270         return status;
1271 }
1272
1273 /**
1274  * Data plane threads: message handling
1275  */
1276 static inline struct pipeline_msg_req *
1277 pipeline_msg_recv(struct rte_ring *msgq_req)
1278 {
1279         struct pipeline_msg_req *req;
1280
1281         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
1282
1283         if (status != 0)
1284                 return NULL;
1285
1286         return req;
1287 }
1288
1289 static inline void
1290 pipeline_msg_send(struct rte_ring *msgq_rsp,
1291         struct pipeline_msg_rsp *rsp)
1292 {
1293         int status;
1294
1295         do {
1296                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
1297         } while (status == -ENOBUFS);
1298 }
1299
1300 static struct pipeline_msg_rsp *
1301 pipeline_msg_handle_port_in_stats_read(struct pipeline_data *p,
1302         struct pipeline_msg_req *req)
1303 {
1304         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1305         uint32_t port_id = req->id;
1306         int clear = req->port_in_stats_read.clear;
1307
1308         rsp->status = rte_pipeline_port_in_stats_read(p->p,
1309                 port_id,
1310                 &rsp->port_in_stats_read.stats,
1311                 clear);
1312
1313         return rsp;
1314 }
1315
1316 static struct pipeline_msg_rsp *
1317 pipeline_msg_handle_port_in_enable(struct pipeline_data *p,
1318         struct pipeline_msg_req *req)
1319 {
1320         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1321         uint32_t port_id = req->id;
1322
1323         rsp->status = rte_pipeline_port_in_enable(p->p,
1324                 port_id);
1325
1326         return rsp;
1327 }
1328
1329 static struct pipeline_msg_rsp *
1330 pipeline_msg_handle_port_in_disable(struct pipeline_data *p,
1331         struct pipeline_msg_req *req)
1332 {
1333         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1334         uint32_t port_id = req->id;
1335
1336         rsp->status = rte_pipeline_port_in_disable(p->p,
1337                 port_id);
1338
1339         return rsp;
1340 }
1341
1342 static struct pipeline_msg_rsp *
1343 pipeline_msg_handle_port_out_stats_read(struct pipeline_data *p,
1344         struct pipeline_msg_req *req)
1345 {
1346         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1347         uint32_t port_id = req->id;
1348         int clear = req->port_out_stats_read.clear;
1349
1350         rsp->status = rte_pipeline_port_out_stats_read(p->p,
1351                 port_id,
1352                 &rsp->port_out_stats_read.stats,
1353                 clear);
1354
1355         return rsp;
1356 }
1357
1358 static struct pipeline_msg_rsp *
1359 pipeline_msg_handle_table_stats_read(struct pipeline_data *p,
1360         struct pipeline_msg_req *req)
1361 {
1362         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1363         uint32_t port_id = req->id;
1364         int clear = req->table_stats_read.clear;
1365
1366         rsp->status = rte_pipeline_table_stats_read(p->p,
1367                 port_id,
1368                 &rsp->table_stats_read.stats,
1369                 clear);
1370
1371         return rsp;
1372 }
1373
1374 union table_rule_match_low_level {
1375         struct rte_table_acl_rule_add_params acl_add;
1376         struct rte_table_acl_rule_delete_params acl_delete;
1377         struct rte_table_array_key array;
1378         uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
1379         struct rte_table_lpm_key lpm_ipv4;
1380         struct rte_table_lpm_ipv6_key lpm_ipv6;
1381 };
1382
1383 static int
1384 match_convert_ipv6_depth(uint32_t depth, uint32_t *depth32)
1385 {
1386         if (depth > 128)
1387                 return -1;
1388
1389         switch (depth / 32) {
1390         case 0:
1391                 depth32[0] = depth;
1392                 depth32[1] = 0;
1393                 depth32[2] = 0;
1394                 depth32[3] = 0;
1395                 return 0;
1396
1397         case 1:
1398                 depth32[0] = 32;
1399                 depth32[1] = depth - 32;
1400                 depth32[2] = 0;
1401                 depth32[3] = 0;
1402                 return 0;
1403
1404         case 2:
1405                 depth32[0] = 32;
1406                 depth32[1] = 32;
1407                 depth32[2] = depth - 64;
1408                 depth32[3] = 0;
1409                 return 0;
1410
1411         case 3:
1412                 depth32[0] = 32;
1413                 depth32[1] = 32;
1414                 depth32[2] = 32;
1415                 depth32[3] = depth - 96;
1416                 return 0;
1417
1418         case 4:
1419                 depth32[0] = 32;
1420                 depth32[1] = 32;
1421                 depth32[2] = 32;
1422                 depth32[3] = 32;
1423                 return 0;
1424
1425         default:
1426                 return -1;
1427         }
1428 }
1429
1430 static int
1431 match_convert(struct table_rule_match *mh,
1432         union table_rule_match_low_level *ml,
1433         int add)
1434 {
1435         memset(ml, 0, sizeof(*ml));
1436
1437         switch (mh->match_type) {
1438         case TABLE_ACL:
1439                 if (mh->match.acl.ip_version)
1440                         if (add) {
1441                                 ml->acl_add.field_value[0].value.u8 =
1442                                         mh->match.acl.proto;
1443                                 ml->acl_add.field_value[0].mask_range.u8 =
1444                                         mh->match.acl.proto_mask;
1445
1446                                 ml->acl_add.field_value[1].value.u32 =
1447                                         mh->match.acl.ipv4.sa;
1448                                 ml->acl_add.field_value[1].mask_range.u32 =
1449                                         mh->match.acl.sa_depth;
1450
1451                                 ml->acl_add.field_value[2].value.u32 =
1452                                         mh->match.acl.ipv4.da;
1453                                 ml->acl_add.field_value[2].mask_range.u32 =
1454                                         mh->match.acl.da_depth;
1455
1456                                 ml->acl_add.field_value[3].value.u16 =
1457                                         mh->match.acl.sp0;
1458                                 ml->acl_add.field_value[3].mask_range.u16 =
1459                                         mh->match.acl.sp1;
1460
1461                                 ml->acl_add.field_value[4].value.u16 =
1462                                         mh->match.acl.dp0;
1463                                 ml->acl_add.field_value[4].mask_range.u16 =
1464                                         mh->match.acl.dp1;
1465
1466                                 ml->acl_add.priority =
1467                                         (int32_t) mh->match.acl.priority;
1468                         } else {
1469                                 ml->acl_delete.field_value[0].value.u8 =
1470                                         mh->match.acl.proto;
1471                                 ml->acl_delete.field_value[0].mask_range.u8 =
1472                                         mh->match.acl.proto_mask;
1473
1474                                 ml->acl_delete.field_value[1].value.u32 =
1475                                         mh->match.acl.ipv4.sa;
1476                                 ml->acl_delete.field_value[1].mask_range.u32 =
1477                                         mh->match.acl.sa_depth;
1478
1479                                 ml->acl_delete.field_value[2].value.u32 =
1480                                         mh->match.acl.ipv4.da;
1481                                 ml->acl_delete.field_value[2].mask_range.u32 =
1482                                         mh->match.acl.da_depth;
1483
1484                                 ml->acl_delete.field_value[3].value.u16 =
1485                                         mh->match.acl.sp0;
1486                                 ml->acl_delete.field_value[3].mask_range.u16 =
1487                                         mh->match.acl.sp1;
1488
1489                                 ml->acl_delete.field_value[4].value.u16 =
1490                                         mh->match.acl.dp0;
1491                                 ml->acl_delete.field_value[4].mask_range.u16 =
1492                                         mh->match.acl.dp1;
1493                         }
1494                 else
1495                         if (add) {
1496                                 uint32_t *sa32 =
1497                                         (uint32_t *) mh->match.acl.ipv6.sa;
1498                                 uint32_t *da32 =
1499                                         (uint32_t *) mh->match.acl.ipv6.da;
1500                                 uint32_t sa32_depth[4], da32_depth[4];
1501                                 int status;
1502
1503                                 status = match_convert_ipv6_depth(
1504                                         mh->match.acl.sa_depth,
1505                                         sa32_depth);
1506                                 if (status)
1507                                         return status;
1508
1509                                 status = match_convert_ipv6_depth(
1510                                         mh->match.acl.da_depth,
1511                                         da32_depth);
1512                                 if (status)
1513                                         return status;
1514
1515                                 ml->acl_add.field_value[0].value.u8 =
1516                                         mh->match.acl.proto;
1517                                 ml->acl_add.field_value[0].mask_range.u8 =
1518                                         mh->match.acl.proto_mask;
1519
1520                                 ml->acl_add.field_value[1].value.u32 = sa32[0];
1521                                 ml->acl_add.field_value[1].mask_range.u32 =
1522                                         sa32_depth[0];
1523                                 ml->acl_add.field_value[2].value.u32 = sa32[1];
1524                                 ml->acl_add.field_value[2].mask_range.u32 =
1525                                         sa32_depth[1];
1526                                 ml->acl_add.field_value[3].value.u32 = sa32[2];
1527                                 ml->acl_add.field_value[3].mask_range.u32 =
1528                                         sa32_depth[2];
1529                                 ml->acl_add.field_value[4].value.u32 = sa32[3];
1530                                 ml->acl_add.field_value[4].mask_range.u32 =
1531                                         sa32_depth[3];
1532
1533                                 ml->acl_add.field_value[5].value.u32 = da32[0];
1534                                 ml->acl_add.field_value[5].mask_range.u32 =
1535                                         da32_depth[0];
1536                                 ml->acl_add.field_value[6].value.u32 = da32[1];
1537                                 ml->acl_add.field_value[6].mask_range.u32 =
1538                                         da32_depth[1];
1539                                 ml->acl_add.field_value[7].value.u32 = da32[2];
1540                                 ml->acl_add.field_value[7].mask_range.u32 =
1541                                         da32_depth[2];
1542                                 ml->acl_add.field_value[8].value.u32 = da32[3];
1543                                 ml->acl_add.field_value[8].mask_range.u32 =
1544                                         da32_depth[3];
1545
1546                                 ml->acl_add.field_value[9].value.u16 =
1547                                         mh->match.acl.sp0;
1548                                 ml->acl_add.field_value[9].mask_range.u16 =
1549                                         mh->match.acl.sp1;
1550
1551                                 ml->acl_add.field_value[10].value.u16 =
1552                                         mh->match.acl.dp0;
1553                                 ml->acl_add.field_value[10].mask_range.u16 =
1554                                         mh->match.acl.dp1;
1555
1556                                 ml->acl_add.priority =
1557                                         (int32_t) mh->match.acl.priority;
1558                         } else {
1559                                 uint32_t *sa32 =
1560                                         (uint32_t *) mh->match.acl.ipv6.sa;
1561                                 uint32_t *da32 =
1562                                         (uint32_t *) mh->match.acl.ipv6.da;
1563                                 uint32_t sa32_depth[4], da32_depth[4];
1564                                 int status;
1565
1566                                 status = match_convert_ipv6_depth(
1567                                         mh->match.acl.sa_depth,
1568                                         sa32_depth);
1569                                 if (status)
1570                                         return status;
1571
1572                                 status = match_convert_ipv6_depth(
1573                                         mh->match.acl.da_depth,
1574                                         da32_depth);
1575                                 if (status)
1576                                         return status;
1577
1578                                 ml->acl_delete.field_value[0].value.u8 =
1579                                         mh->match.acl.proto;
1580                                 ml->acl_delete.field_value[0].mask_range.u8 =
1581                                         mh->match.acl.proto_mask;
1582
1583                                 ml->acl_delete.field_value[1].value.u32 =
1584                                         sa32[0];
1585                                 ml->acl_delete.field_value[1].mask_range.u32 =
1586                                         sa32_depth[0];
1587                                 ml->acl_delete.field_value[2].value.u32 =
1588                                         sa32[1];
1589                                 ml->acl_delete.field_value[2].mask_range.u32 =
1590                                         sa32_depth[1];
1591                                 ml->acl_delete.field_value[3].value.u32 =
1592                                         sa32[2];
1593                                 ml->acl_delete.field_value[3].mask_range.u32 =
1594                                         sa32_depth[2];
1595                                 ml->acl_delete.field_value[4].value.u32 =
1596                                         sa32[3];
1597                                 ml->acl_delete.field_value[4].mask_range.u32 =
1598                                         sa32_depth[3];
1599
1600                                 ml->acl_delete.field_value[5].value.u32 =
1601                                         da32[0];
1602                                 ml->acl_delete.field_value[5].mask_range.u32 =
1603                                         da32_depth[0];
1604                                 ml->acl_delete.field_value[6].value.u32 =
1605                                         da32[1];
1606                                 ml->acl_delete.field_value[6].mask_range.u32 =
1607                                         da32_depth[1];
1608                                 ml->acl_delete.field_value[7].value.u32 =
1609                                         da32[2];
1610                                 ml->acl_delete.field_value[7].mask_range.u32 =
1611                                         da32_depth[2];
1612                                 ml->acl_delete.field_value[8].value.u32 =
1613                                         da32[3];
1614                                 ml->acl_delete.field_value[8].mask_range.u32 =
1615                                         da32_depth[3];
1616
1617                                 ml->acl_delete.field_value[9].value.u16 =
1618                                         mh->match.acl.sp0;
1619                                 ml->acl_delete.field_value[9].mask_range.u16 =
1620                                         mh->match.acl.sp1;
1621
1622                                 ml->acl_delete.field_value[10].value.u16 =
1623                                         mh->match.acl.dp0;
1624                                 ml->acl_delete.field_value[10].mask_range.u16 =
1625                                         mh->match.acl.dp1;
1626                         }
1627                 return 0;
1628
1629         case TABLE_ARRAY:
1630                 ml->array.pos = mh->match.array.pos;
1631                 return 0;
1632
1633         case TABLE_HASH:
1634                 memcpy(ml->hash, mh->match.hash.key, sizeof(ml->hash));
1635                 return 0;
1636
1637         case TABLE_LPM:
1638                 if (mh->match.lpm.ip_version) {
1639                         ml->lpm_ipv4.ip = mh->match.lpm.ipv4;
1640                         ml->lpm_ipv4.depth = mh->match.lpm.depth;
1641                 } else {
1642                         memcpy(ml->lpm_ipv6.ip,
1643                                 mh->match.lpm.ipv6, sizeof(ml->lpm_ipv6.ip));
1644                         ml->lpm_ipv6.depth = mh->match.lpm.depth;
1645                 }
1646
1647                 return 0;
1648
1649         default:
1650                 return -1;
1651         }
1652 }
1653
1654 static struct pipeline_msg_rsp *
1655 pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
1656         struct pipeline_msg_req *req)
1657 {
1658         union table_rule_match_low_level match_ll;
1659         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1660         struct table_rule_match *match = &req->table_rule_add.match;
1661         struct table_rule_action *action = &req->table_rule_add.action;
1662         struct rte_pipeline_table_entry *data_in, *data_out;
1663         uint32_t table_id = req->id;
1664         int key_found, status;
1665         struct rte_table_action *a = p->table_data[table_id].a;
1666
1667         /* Apply actions */
1668         memset(p->buffer, 0, sizeof(p->buffer));
1669         data_in = (struct rte_pipeline_table_entry *) p->buffer;
1670
1671         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1672                 status = rte_table_action_apply(a,
1673                         data_in,
1674                         RTE_TABLE_ACTION_FWD,
1675                         &action->fwd);
1676
1677                 if (status) {
1678                         rsp->status = -1;
1679                         return rsp;
1680                 }
1681         }
1682
1683         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1684                 status = rte_table_action_apply(a,
1685                         data_in,
1686                         RTE_TABLE_ACTION_MTR,
1687                         &action->mtr);
1688
1689                 if (status) {
1690                         rsp->status = -1;
1691                         return rsp;
1692                 }
1693         }
1694
1695         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1696                 status = rte_table_action_apply(a,
1697                         data_in,
1698                         RTE_TABLE_ACTION_TM,
1699                         &action->tm);
1700
1701                 if (status) {
1702                         rsp->status = -1;
1703                         return rsp;
1704                 }
1705         }
1706
1707         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1708                 status = rte_table_action_apply(a,
1709                         data_in,
1710                         RTE_TABLE_ACTION_ENCAP,
1711                         &action->encap);
1712
1713                 if (status) {
1714                         rsp->status = -1;
1715                         return rsp;
1716                 }
1717         }
1718
1719         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1720                 status = rte_table_action_apply(a,
1721                         data_in,
1722                         RTE_TABLE_ACTION_NAT,
1723                         &action->nat);
1724
1725                 if (status) {
1726                         rsp->status = -1;
1727                         return rsp;
1728                 }
1729         }
1730
1731         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
1732                 status = rte_table_action_apply(a,
1733                         data_in,
1734                         RTE_TABLE_ACTION_TTL,
1735                         &action->ttl);
1736
1737                 if (status) {
1738                         rsp->status = -1;
1739                         return rsp;
1740                 }
1741         }
1742
1743         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
1744                 status = rte_table_action_apply(a,
1745                         data_in,
1746                         RTE_TABLE_ACTION_STATS,
1747                         &action->stats);
1748
1749                 if (status) {
1750                         rsp->status = -1;
1751                         return rsp;
1752                 }
1753         }
1754
1755         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
1756                 status = rte_table_action_apply(a,
1757                         data_in,
1758                         RTE_TABLE_ACTION_TIME,
1759                         &action->time);
1760
1761                 if (status) {
1762                         rsp->status = -1;
1763                         return rsp;
1764                 }
1765         }
1766
1767         /* Add rule (match, action) to table */
1768         status = match_convert(match, &match_ll, 1);
1769         if (status) {
1770                 rsp->status = -1;
1771                 return rsp;
1772         }
1773
1774         status = rte_pipeline_table_entry_add(p->p,
1775                 table_id,
1776                 &match_ll,
1777                 data_in,
1778                 &key_found,
1779                 &data_out);
1780         if (status) {
1781                 rsp->status = -1;
1782                 return rsp;
1783         }
1784
1785         /* Write response */
1786         rsp->status = 0;
1787         rsp->table_rule_add.data = data_out;
1788
1789         return rsp;
1790 }
1791
1792 static struct pipeline_msg_rsp *
1793 pipeline_msg_handle_table_rule_add_default(struct pipeline_data *p,
1794         struct pipeline_msg_req *req)
1795 {
1796         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1797         struct table_rule_action *action = &req->table_rule_add_default.action;
1798         struct rte_pipeline_table_entry *data_in, *data_out;
1799         uint32_t table_id = req->id;
1800         int status;
1801
1802         /* Apply actions */
1803         memset(p->buffer, 0, sizeof(p->buffer));
1804         data_in = (struct rte_pipeline_table_entry *) p->buffer;
1805
1806         data_in->action = action->fwd.action;
1807         if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
1808                 data_in->port_id = action->fwd.id;
1809         if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
1810                 data_in->table_id = action->fwd.id;
1811
1812         /* Add default rule to table */
1813         status = rte_pipeline_table_default_entry_add(p->p,
1814                 table_id,
1815                 data_in,
1816                 &data_out);
1817         if (status) {
1818                 rsp->status = -1;
1819                 return rsp;
1820         }
1821
1822         /* Write response */
1823         rsp->status = 0;
1824         rsp->table_rule_add_default.data = data_out;
1825
1826         return rsp;
1827 }
1828
1829 static struct pipeline_msg_rsp *
1830 pipeline_msg_handle_table_rule_add_bulk(struct pipeline_data *p,
1831         struct pipeline_msg_req *req)
1832 {
1833
1834         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
1835
1836         uint32_t table_id = req->id;
1837         struct table_rule_match *match = req->table_rule_add_bulk.match;
1838         struct table_rule_action *action = req->table_rule_add_bulk.action;
1839         struct rte_pipeline_table_entry **data =
1840                 (struct rte_pipeline_table_entry **)req->table_rule_add_bulk.data;
1841         uint32_t n_rules = req->table_rule_add_bulk.n_rules;
1842         uint32_t bulk = req->table_rule_add_bulk.bulk;
1843
1844         struct rte_table_action *a = p->table_data[table_id].a;
1845         union table_rule_match_low_level *match_ll;
1846         uint8_t *action_ll;
1847         void **match_ll_ptr;
1848         struct rte_pipeline_table_entry **action_ll_ptr;
1849         int *found, status;
1850         uint32_t i;
1851
1852         /* Memory allocation */
1853         match_ll = calloc(n_rules, sizeof(union table_rule_match_low_level));
1854         action_ll = calloc(n_rules, TABLE_RULE_ACTION_SIZE_MAX);
1855         match_ll_ptr = calloc(n_rules, sizeof(void *));
1856         action_ll_ptr =
1857                 calloc(n_rules, sizeof(struct rte_pipeline_table_entry *));
1858         found = calloc(n_rules, sizeof(int));
1859
1860         if ((match_ll == NULL) ||
1861                 (action_ll == NULL) ||
1862                 (match_ll_ptr == NULL) ||
1863                 (action_ll_ptr == NULL) ||
1864                 (found == NULL))
1865                 goto fail;
1866
1867         for (i = 0; i < n_rules; i++) {
1868                 match_ll_ptr[i] = (void *)&match_ll[i];
1869                 action_ll_ptr[i] =
1870                         (struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
1871         }
1872
1873         /* Rule match conversion */
1874         for (i = 0; i < n_rules; i++) {
1875                 status = match_convert(&match[i], match_ll_ptr[i], 1);
1876                 if (status)
1877                         goto fail;
1878         }
1879
1880         /* Rule action conversion */
1881         for (i = 0; i < n_rules; i++) {
1882                 void *data_in = action_ll_ptr[i];
1883                 struct table_rule_action *act = &action[i];
1884
1885                 if (act->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1886                         status = rte_table_action_apply(a,
1887                                 data_in,
1888                                 RTE_TABLE_ACTION_FWD,
1889                                 &act->fwd);
1890
1891                         if (status)
1892                                 goto fail;
1893                 }
1894
1895                 if (act->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1896                         status = rte_table_action_apply(a,
1897                                 data_in,
1898                                 RTE_TABLE_ACTION_MTR,
1899                                 &act->mtr);
1900
1901                         if (status)
1902                                 goto fail;
1903                 }
1904
1905                 if (act->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1906                         status = rte_table_action_apply(a,
1907                                 data_in,
1908                                 RTE_TABLE_ACTION_TM,
1909                                 &act->tm);
1910
1911                         if (status)
1912                                 goto fail;
1913                 }
1914
1915                 if (act->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1916                         status = rte_table_action_apply(a,
1917                                 data_in,
1918                                 RTE_TABLE_ACTION_ENCAP,
1919                                 &act->encap);
1920
1921                         if (status)
1922                                 goto fail;
1923                 }
1924
1925                 if (act->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1926                         status = rte_table_action_apply(a,
1927                                 data_in,
1928                                 RTE_TABLE_ACTION_NAT,
1929                                 &act->nat);
1930
1931                         if (status)
1932                                 goto fail;
1933                 }
1934
1935                 if (act->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
1936                         status = rte_table_action_apply(a,
1937                                 data_in,
1938                                 RTE_TABLE_ACTION_TTL,
1939                                 &act->ttl);
1940
1941                         if (status)
1942                                 goto fail;
1943                 }
1944
1945                 if (act->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
1946                         status = rte_table_action_apply(a,
1947                                 data_in,
1948                                 RTE_TABLE_ACTION_STATS,
1949                                 &act->stats);
1950
1951                         if (status)
1952                                 goto fail;
1953                 }
1954
1955                 if (act->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
1956                         status = rte_table_action_apply(a,
1957                                 data_in,
1958                                 RTE_TABLE_ACTION_TIME,
1959                                 &act->time);
1960
1961                         if (status)
1962                                 goto fail;
1963                 }
1964         }
1965
1966         /* Add rule (match, action) to table */
1967         if (bulk) {
1968                 status = rte_pipeline_table_entry_add_bulk(p->p,
1969                         table_id,
1970                         match_ll_ptr,
1971                         action_ll_ptr,
1972                         n_rules,
1973                         found,
1974                         data);
1975                 if (status)
1976                         n_rules = 0;
1977         } else
1978                 for (i = 0; i < n_rules; i++) {
1979                         status = rte_pipeline_table_entry_add(p->p,
1980                                 table_id,
1981                                 match_ll_ptr[i],
1982                                 action_ll_ptr[i],
1983                                 &found[i],
1984                                 &data[i]);
1985                         if (status) {
1986                                 n_rules = i;
1987                                 break;
1988                         }
1989                 }
1990
1991         /* Write response */
1992         rsp->status = 0;
1993         rsp->table_rule_add_bulk.n_rules = n_rules;
1994
1995         /* Free */
1996         free(found);
1997         free(action_ll_ptr);
1998         free(match_ll_ptr);
1999         free(action_ll);
2000         free(match_ll);
2001
2002         return rsp;
2003
2004 fail:
2005         free(found);
2006         free(action_ll_ptr);
2007         free(match_ll_ptr);
2008         free(action_ll);
2009         free(match_ll);
2010
2011         rsp->status = -1;
2012         rsp->table_rule_add_bulk.n_rules = 0;
2013         return rsp;
2014 }
2015
2016 static struct pipeline_msg_rsp *
2017 pipeline_msg_handle_table_rule_delete(struct pipeline_data *p,
2018         struct pipeline_msg_req *req)
2019 {
2020         union table_rule_match_low_level match_ll;
2021         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2022         struct table_rule_match *match = &req->table_rule_delete.match;
2023         uint32_t table_id = req->id;
2024         int key_found, status;
2025
2026         status = match_convert(match, &match_ll, 0);
2027         if (status) {
2028                 rsp->status = -1;
2029                 return rsp;
2030         }
2031
2032         rsp->status = rte_pipeline_table_entry_delete(p->p,
2033                 table_id,
2034                 &match_ll,
2035                 &key_found,
2036                 NULL);
2037
2038         return rsp;
2039 }
2040
2041 static struct pipeline_msg_rsp *
2042 pipeline_msg_handle_table_rule_delete_default(struct pipeline_data *p,
2043         struct pipeline_msg_req *req)
2044 {
2045         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2046         uint32_t table_id = req->id;
2047
2048         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
2049                 table_id,
2050                 NULL);
2051
2052         return rsp;
2053 }
2054
2055 static void
2056 pipeline_msg_handle(struct pipeline_data *p)
2057 {
2058         for ( ; ; ) {
2059                 struct pipeline_msg_req *req;
2060                 struct pipeline_msg_rsp *rsp;
2061
2062                 req = pipeline_msg_recv(p->msgq_req);
2063                 if (req == NULL)
2064                         break;
2065
2066                 switch (req->type) {
2067                 case PIPELINE_REQ_PORT_IN_STATS_READ:
2068                         rsp = pipeline_msg_handle_port_in_stats_read(p, req);
2069                         break;
2070
2071                 case PIPELINE_REQ_PORT_IN_ENABLE:
2072                         rsp = pipeline_msg_handle_port_in_enable(p, req);
2073                         break;
2074
2075                 case PIPELINE_REQ_PORT_IN_DISABLE:
2076                         rsp = pipeline_msg_handle_port_in_disable(p, req);
2077                         break;
2078
2079                 case PIPELINE_REQ_PORT_OUT_STATS_READ:
2080                         rsp = pipeline_msg_handle_port_out_stats_read(p, req);
2081                         break;
2082
2083                 case PIPELINE_REQ_TABLE_STATS_READ:
2084                         rsp = pipeline_msg_handle_table_stats_read(p, req);
2085                         break;
2086
2087                 case PIPELINE_REQ_TABLE_RULE_ADD:
2088                         rsp = pipeline_msg_handle_table_rule_add(p, req);
2089                         break;
2090
2091                 case PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT:
2092                         rsp = pipeline_msg_handle_table_rule_add_default(p,     req);
2093                         break;
2094
2095                 case PIPELINE_REQ_TABLE_RULE_ADD_BULK:
2096                         rsp = pipeline_msg_handle_table_rule_add_bulk(p, req);
2097                         break;
2098
2099                 case PIPELINE_REQ_TABLE_RULE_DELETE:
2100                         rsp = pipeline_msg_handle_table_rule_delete(p, req);
2101                         break;
2102
2103                 case PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT:
2104                         rsp = pipeline_msg_handle_table_rule_delete_default(p, req);
2105                         break;
2106
2107                 default:
2108                         rsp = (struct pipeline_msg_rsp *) req;
2109                         rsp->status = -1;
2110                 }
2111
2112                 pipeline_msg_send(p->msgq_rsp, rsp);
2113         }
2114 }
2115
2116 /**
2117  * Data plane threads: main
2118  */
2119 int
2120 thread_main(void *arg __rte_unused)
2121 {
2122         struct thread_data *t;
2123         uint32_t thread_id, i;
2124
2125         thread_id = rte_lcore_id();
2126         t = &thread_data[thread_id];
2127
2128         /* Dispatch loop */
2129         for (i = 0; ; i++) {
2130                 uint32_t j;
2131
2132                 /* Data Plane */
2133                 for (j = 0; j < t->n_pipelines; j++)
2134                         rte_pipeline_run(t->p[j]);
2135
2136                 /* Control Plane */
2137                 if ((i & 0xF) == 0) {
2138                         uint64_t time = rte_get_tsc_cycles();
2139                         uint64_t time_next_min = UINT64_MAX;
2140
2141                         if (time < t->time_next_min)
2142                                 continue;
2143
2144                         /* Pipeline message queues */
2145                         for (j = 0; j < t->n_pipelines; j++) {
2146                                 struct pipeline_data *p =
2147                                         &t->pipeline_data[j];
2148                                 uint64_t time_next = p->time_next;
2149
2150                                 if (time_next <= time) {
2151                                         pipeline_msg_handle(p);
2152                                         rte_pipeline_flush(p->p);
2153                                         time_next = time + p->timer_period;
2154                                         p->time_next = time_next;
2155                                 }
2156
2157                                 if (time_next < time_next_min)
2158                                         time_next_min = time_next;
2159                         }
2160
2161                         /* Thread message queues */
2162                         {
2163                                 uint64_t time_next = t->time_next;
2164
2165                                 if (time_next <= time) {
2166                                         thread_msg_handle(t);
2167                                         time_next = time + t->timer_period;
2168                                         t->time_next = time_next;
2169                                 }
2170
2171                                 if (time_next < time_next_min)
2172                                         time_next_min = time_next;
2173                         }
2174
2175                         t->time_next_min = time_next_min;
2176                 }
2177         }
2178
2179         return 0;
2180 }