drivers/net/softnic/rte_eth_softnic_thread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdlib.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_cycles.h>
   9 #include <rte_lcore.h>
  10 #include <rte_service_component.h>
  11 #include <rte_ring.h>
  12
  13 #include <rte_table_acl.h>
  14 #include <rte_table_array.h>
  15 #include <rte_table_hash.h>
  16 #include <rte_table_lpm.h>
  17 #include <rte_table_lpm_ipv6.h>
  18 #include "rte_eth_softnic_internals.h"
  19
  20 /**
  21  * Master thread: data plane thread init
  22  */
  23 void
  24 softnic_thread_free(struct pmd_internals *softnic)
  25 {
  26         uint32_t i;
  27
  28         RTE_LCORE_FOREACH_SLAVE(i) {
  29                 struct softnic_thread *t = &softnic->thread[i];
  30
  31                 /* MSGQs */
  32                 if (t->msgq_req)
  33                         rte_ring_free(t->msgq_req);
  34
  35                 if (t->msgq_rsp)
  36                         rte_ring_free(t->msgq_rsp);
  37         }
  38 }
  39
  40 int
  41 softnic_thread_init(struct pmd_internals *softnic)
  42 {
  43         uint32_t i;
  44
  45         for (i = 0; i < RTE_MAX_LCORE; i++) {
  46                 char ring_name[NAME_MAX];
  47                 struct rte_ring *msgq_req, *msgq_rsp;
  48                 struct softnic_thread *t = &softnic->thread[i];
  49                 struct softnic_thread_data *t_data = &softnic->thread_data[i];
  50                 uint32_t cpu_id = rte_lcore_to_socket_id(i);
  51
  52                 /* MSGQs */
  53                 snprintf(ring_name, sizeof(ring_name), "%s-TH%u-REQ",
  54                         softnic->params.name,
  55                         i);
  56
  57                 msgq_req = rte_ring_create(ring_name,
  58                         THREAD_MSGQ_SIZE,
  59                         cpu_id,
  60                         RING_F_SP_ENQ | RING_F_SC_DEQ);
  61
  62                 if (msgq_req == NULL) {
  63                         softnic_thread_free(softnic);
  64                         return -1;
  65                 }
  66
  67                 snprintf(ring_name, sizeof(ring_name), "%s-TH%u-RSP",
  68                         softnic->params.name,
  69                         i);
  70
  71                 msgq_rsp = rte_ring_create(ring_name,
  72                         THREAD_MSGQ_SIZE,
  73                         cpu_id,
  74                         RING_F_SP_ENQ | RING_F_SC_DEQ);
  75
  76                 if (msgq_rsp == NULL) {
  77                         softnic_thread_free(softnic);
  78                         return -1;
  79                 }
  80
  81                 /* Master thread records */
  82                 t->msgq_req = msgq_req;
  83                 t->msgq_rsp = msgq_rsp;
  84                 t->service_id = UINT32_MAX;
  85
  86                 /* Data plane thread records */
  87                 t_data->n_pipelines = 0;
  88                 t_data->msgq_req = msgq_req;
  89                 t_data->msgq_rsp = msgq_rsp;
  90                 t_data->timer_period =
  91                         (rte_get_tsc_hz() * THREAD_TIMER_PERIOD_MS) / 1000;
  92                 t_data->time_next = rte_get_tsc_cycles() + t_data->timer_period;
  93                 t_data->time_next_min = t_data->time_next;
  94         }
  95
  96         return 0;
  97 }
  98
  99 static inline int
 100 thread_is_valid(struct pmd_internals *softnic, uint32_t thread_id)
 101 {
 102         if (thread_id == rte_get_master_lcore())
 103                 return 0; /* FALSE */
 104
 105         if (softnic->params.sc && rte_lcore_has_role(thread_id, ROLE_SERVICE))
 106                 return 1; /* TRUE */
 107         if (!softnic->params.sc && rte_lcore_has_role(thread_id, ROLE_RTE))
 108                 return 1; /* TRUE */
 109
 110         return 0; /* FALSE */
 111 }
 112
 113 static inline int
 114 thread_is_running(uint32_t thread_id)
 115 {
 116         enum rte_lcore_state_t thread_state;
 117
 118         thread_state = rte_eal_get_lcore_state(thread_id);
 119         return (thread_state == RUNNING)? 1 : 0;
 120 }
 121
 122 static int32_t
 123 rte_pmd_softnic_run_internal(void *arg);
 124
 125 static inline int
 126 thread_sc_service_up(struct pmd_internals *softnic, uint32_t thread_id)
 127 {
 128         struct rte_service_spec service_params;
 129         struct softnic_thread *t = &softnic->thread[thread_id];
 130         struct rte_eth_dev *dev;
 131         int status;
 132         uint16_t port_id;
 133
 134         /* service params */
 135         status = rte_eth_dev_get_port_by_name(softnic->params.name, &port_id);
 136         if (status)
 137                 return status;
 138
 139         dev = &rte_eth_devices[port_id];
 140         snprintf(service_params.name, sizeof(service_params.name), "%s_%u",
 141                 softnic->params.name,
 142                 thread_id);
 143         service_params.callback = rte_pmd_softnic_run_internal;
 144         service_params.callback_userdata = dev;
 145         service_params.capabilities = 0;
 146         service_params.socket_id = (int)softnic->params.cpu_id;
 147
 148         /* service register */
 149         status = rte_service_component_register(&service_params, &t->service_id);
 150         if (status)
 151                 return status;
 152
 153         status = rte_service_component_runstate_set(t->service_id, 1);
 154         if (status) {
 155                 rte_service_component_unregister(t->service_id);
 156                 t->service_id = UINT32_MAX;
 157                 return status;
 158         }
 159
 160         status = rte_service_runstate_set(t->service_id, 1);
 161         if (status) {
 162                 rte_service_component_runstate_set(t->service_id, 0);
 163                 rte_service_component_unregister(t->service_id);
 164                 t->service_id = UINT32_MAX;
 165                 return status;
 166         }
 167
 168         /* service map to thread */
 169         status = rte_service_map_lcore_set(t->service_id, thread_id, 1);
 170         if (status) {
 171                 rte_service_runstate_set(t->service_id, 0);
 172                 rte_service_component_runstate_set(t->service_id, 0);
 173                 rte_service_component_unregister(t->service_id);
 174                 t->service_id = UINT32_MAX;
 175                 return status;
 176         }
 177
 178         return 0;
 179 }
 180
 181 static inline void
 182 thread_sc_service_down(struct pmd_internals *softnic, uint32_t thread_id)
 183 {
 184         struct softnic_thread *t = &softnic->thread[thread_id];
 185
 186         /* service unmap from thread */
 187         rte_service_map_lcore_set(t->service_id, thread_id, 0);
 188
 189         /* service unregister */
 190         rte_service_runstate_set(t->service_id, 0);
 191         rte_service_component_runstate_set(t->service_id, 0);
 192         rte_service_component_unregister(t->service_id);
 193
 194         t->service_id = UINT32_MAX;
 195 }
 196
 197 /**
 198  * Pipeline is running when:
 199  *    (A) Pipeline is mapped to a data plane thread AND
 200  *    (B) Its data plane thread is in RUNNING state.
 201  */
 202 static inline int
 203 pipeline_is_running(struct pipeline *p)
 204 {
 205         if (p->enabled == 0)
 206                 return 0;
 207
 208         return thread_is_running(p->thread_id);
 209 }
 210
 211 /**
 212  * Master thread & data plane threads: message passing
 213  */
 214 enum thread_req_type {
 215         THREAD_REQ_PIPELINE_ENABLE = 0,
 216         THREAD_REQ_PIPELINE_DISABLE,
 217         THREAD_REQ_MAX
 218 };
 219
 220 struct thread_msg_req {
 221         enum thread_req_type type;
 222
 223         union {
 224                 struct {
 225                         struct rte_pipeline *p;
 226                         struct {
 227                                 struct rte_table_action *a;
 228                         } table[RTE_PIPELINE_TABLE_MAX];
 229                         struct rte_ring *msgq_req;
 230                         struct rte_ring *msgq_rsp;
 231                         uint32_t timer_period_ms;
 232                         uint32_t n_tables;
 233                 } pipeline_enable;
 234
 235                 struct {
 236                         struct rte_pipeline *p;
 237                 } pipeline_disable;
 238         };
 239 };
 240
 241 struct thread_msg_rsp {
 242         int status;
 243 };
 244
 245 /**
 246  * Master thread
 247  */
 248 static struct thread_msg_req *
 249 thread_msg_alloc(void)
 250 {
 251         size_t size = RTE_MAX(sizeof(struct thread_msg_req),
 252                 sizeof(struct thread_msg_rsp));
 253
 254         return calloc(1, size);
 255 }
 256
 257 static void
 258 thread_msg_free(struct thread_msg_rsp *rsp)
 259 {
 260         free(rsp);
 261 }
 262
 263 static struct thread_msg_rsp *
 264 thread_msg_send_recv(struct pmd_internals *softnic,
 265         uint32_t thread_id,
 266         struct thread_msg_req *req)
 267 {
 268         struct softnic_thread *t = &softnic->thread[thread_id];
 269         struct rte_ring *msgq_req = t->msgq_req;
 270         struct rte_ring *msgq_rsp = t->msgq_rsp;
 271         struct thread_msg_rsp *rsp;
 272         int status;
 273
 274         /* send */
 275         do {
 276                 status = rte_ring_sp_enqueue(msgq_req, req);
 277         } while (status == -ENOBUFS);
 278
 279         /* recv */
 280         do {
 281                 status = rte_ring_sc_dequeue(msgq_rsp, (void **)&rsp);
 282         } while (status != 0);
 283
 284         return rsp;
 285 }
 286
 287 int
 288 softnic_thread_pipeline_enable(struct pmd_internals *softnic,
 289         uint32_t thread_id,
 290         const char *pipeline_name)
 291 {
 292         struct pipeline *p = softnic_pipeline_find(softnic, pipeline_name);
 293         struct thread_msg_req *req;
 294         struct thread_msg_rsp *rsp;
 295         uint32_t n_pipelines, i;
 296         int status;
 297
 298         /* Check input params */
 299         if (!thread_is_valid(softnic, thread_id) ||
 300                 (p == NULL) ||
 301                 (p->n_ports_in == 0) ||
 302                 (p->n_ports_out == 0) ||
 303                 (p->n_tables == 0) ||
 304                 p->enabled)
 305                 return -1;
 306
 307         n_pipelines = softnic_pipeline_thread_count(softnic, thread_id);
 308         if (n_pipelines >= THREAD_PIPELINES_MAX)
 309                 return -1;
 310
 311         if (softnic->params.sc && (n_pipelines == 0)) {
 312                 status = thread_sc_service_up(softnic, thread_id);
 313                 if (status)
 314                         return status;
 315         }
 316
 317         if (!thread_is_running(thread_id)) {
 318                 struct softnic_thread_data *td = &softnic->thread_data[thread_id];
 319                 struct pipeline_data *tdp = &td->pipeline_data[td->n_pipelines];
 320
 321                 /* Data plane thread */
 322                 td->p[td->n_pipelines] = p->p;
 323
 324                 tdp->p = p->p;
 325                 for (i = 0; i < p->n_tables; i++)
 326                         tdp->table_data[i].a =
 327                                 p->table[i].a;
 328                 tdp->n_tables = p->n_tables;
 329
 330                 tdp->msgq_req = p->msgq_req;
 331                 tdp->msgq_rsp = p->msgq_rsp;
 332                 tdp->timer_period = (rte_get_tsc_hz() * p->timer_period_ms) / 1000;
 333                 tdp->time_next = rte_get_tsc_cycles() + tdp->timer_period;
 334
 335                 td->n_pipelines++;
 336
 337                 /* Pipeline */
 338                 p->thread_id = thread_id;
 339                 p->enabled = 1;
 340
 341                 return 0;
 342         }
 343
 344         /* Allocate request */
 345         req = thread_msg_alloc();
 346         if (req == NULL)
 347                 return -1;
 348
 349         /* Write request */
 350         req->type = THREAD_REQ_PIPELINE_ENABLE;
 351         req->pipeline_enable.p = p->p;
 352         for (i = 0; i < p->n_tables; i++)
 353                 req->pipeline_enable.table[i].a =
 354                         p->table[i].a;
 355         req->pipeline_enable.msgq_req = p->msgq_req;
 356         req->pipeline_enable.msgq_rsp = p->msgq_rsp;
 357         req->pipeline_enable.timer_period_ms = p->timer_period_ms;
 358         req->pipeline_enable.n_tables = p->n_tables;
 359
 360         /* Send request and wait for response */
 361         rsp = thread_msg_send_recv(softnic, thread_id, req);
 362
 363         /* Read response */
 364         status = rsp->status;
 365
 366         /* Free response */
 367         thread_msg_free(rsp);
 368
 369         /* Request completion */
 370         if (status)
 371                 return status;
 372
 373         p->thread_id = thread_id;
 374         p->enabled = 1;
 375
 376         return 0;
 377 }
 378
 379 int
 380 softnic_thread_pipeline_disable(struct pmd_internals *softnic,
 381         uint32_t thread_id,
 382         const char *pipeline_name)
 383 {
 384         struct pipeline *p = softnic_pipeline_find(softnic, pipeline_name);
 385         struct thread_msg_req *req;
 386         struct thread_msg_rsp *rsp;
 387         uint32_t n_pipelines;
 388         int status;
 389
 390         /* Check input params */
 391         if (!thread_is_valid(softnic, thread_id) ||
 392                 (p == NULL) ||
 393                 (p->enabled && (p->thread_id != thread_id)))
 394                 return -1;
 395
 396         if (p->enabled == 0)
 397                 return 0;
 398
 399         if (!thread_is_running(thread_id)) {
 400                 struct softnic_thread_data *td = &softnic->thread_data[thread_id];
 401                 uint32_t i;
 402
 403                 for (i = 0; i < td->n_pipelines; i++) {
 404                         struct pipeline_data *tdp = &td->pipeline_data[i];
 405
 406                         if (tdp->p != p->p)
 407                                 continue;
 408
 409                         /* Data plane thread */
 410                         if (i < td->n_pipelines - 1) {
 411                                 struct rte_pipeline *pipeline_last =
 412                                         td->p[td->n_pipelines - 1];
 413                                 struct pipeline_data *tdp_last =
 414                                         &td->pipeline_data[td->n_pipelines - 1];
 415
 416                                 td->p[i] = pipeline_last;
 417                                 memcpy(tdp, tdp_last, sizeof(*tdp));
 418                         }
 419
 420                         td->n_pipelines--;
 421
 422                         /* Pipeline */
 423                         p->enabled = 0;
 424
 425                         break;
 426                 }
 427
 428                 if (softnic->params.sc && (td->n_pipelines == 0))
 429                         thread_sc_service_down(softnic, thread_id);
 430
 431                 return 0;
 432         }
 433
 434         /* Allocate request */
 435         req = thread_msg_alloc();
 436         if (req == NULL)
 437                 return -1;
 438
 439         /* Write request */
 440         req->type = THREAD_REQ_PIPELINE_DISABLE;
 441         req->pipeline_disable.p = p->p;
 442
 443         /* Send request and wait for response */
 444         rsp = thread_msg_send_recv(softnic, thread_id, req);
 445
 446         /* Read response */
 447         status = rsp->status;
 448
 449         /* Free response */
 450         thread_msg_free(rsp);
 451
 452         /* Request completion */
 453         if (status)
 454                 return status;
 455
 456         p->enabled = 0;
 457
 458         n_pipelines = softnic_pipeline_thread_count(softnic, thread_id);
 459         if (softnic->params.sc && (n_pipelines == 0))
 460                 thread_sc_service_down(softnic, thread_id);
 461
 462         return 0;
 463 }
 464
 465 /**
 466  * Data plane threads: message handling
 467  */
 468 static inline struct thread_msg_req *
 469 thread_msg_recv(struct rte_ring *msgq_req)
 470 {
 471         struct thread_msg_req *req;
 472
 473         int status = rte_ring_sc_dequeue(msgq_req, (void **)&req);
 474
 475         if (status != 0)
 476                 return NULL;
 477
 478         return req;
 479 }
 480
 481 static inline void
 482 thread_msg_send(struct rte_ring *msgq_rsp,
 483         struct thread_msg_rsp *rsp)
 484 {
 485         int status;
 486
 487         do {
 488                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
 489         } while (status == -ENOBUFS);
 490 }
 491
 492 static struct thread_msg_rsp *
 493 thread_msg_handle_pipeline_enable(struct softnic_thread_data *t,
 494         struct thread_msg_req *req)
 495 {
 496         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *)req;
 497         struct pipeline_data *p = &t->pipeline_data[t->n_pipelines];
 498         uint32_t i;
 499
 500         /* Request */
 501         t->p[t->n_pipelines] = req->pipeline_enable.p;
 502
 503         p->p = req->pipeline_enable.p;
 504         for (i = 0; i < req->pipeline_enable.n_tables; i++)
 505                 p->table_data[i].a =
 506                         req->pipeline_enable.table[i].a;
 507
 508         p->n_tables = req->pipeline_enable.n_tables;
 509
 510         p->msgq_req = req->pipeline_enable.msgq_req;
 511         p->msgq_rsp = req->pipeline_enable.msgq_rsp;
 512         p->timer_period =
 513                 (rte_get_tsc_hz() * req->pipeline_enable.timer_period_ms) / 1000;
 514         p->time_next = rte_get_tsc_cycles() + p->timer_period;
 515
 516         t->n_pipelines++;
 517
 518         /* Response */
 519         rsp->status = 0;
 520         return rsp;
 521 }
 522
 523 static struct thread_msg_rsp *
 524 thread_msg_handle_pipeline_disable(struct softnic_thread_data *t,
 525         struct thread_msg_req *req)
 526 {
 527         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *)req;
 528         uint32_t n_pipelines = t->n_pipelines;
 529         struct rte_pipeline *pipeline = req->pipeline_disable.p;
 530         uint32_t i;
 531
 532         /* find pipeline */
 533         for (i = 0; i < n_pipelines; i++) {
 534                 struct pipeline_data *p = &t->pipeline_data[i];
 535
 536                 if (p->p != pipeline)
 537                         continue;
 538
 539                 if (i < n_pipelines - 1) {
 540                         struct rte_pipeline *pipeline_last =
 541                                 t->p[n_pipelines - 1];
 542                         struct pipeline_data *p_last =
 543                                 &t->pipeline_data[n_pipelines - 1];
 544
 545                         t->p[i] = pipeline_last;
 546                         memcpy(p, p_last, sizeof(*p));
 547                 }
 548
 549                 t->n_pipelines--;
 550
 551                 rsp->status = 0;
 552                 return rsp;
 553         }
 554
 555         /* should not get here */
 556         rsp->status = 0;
 557         return rsp;
 558 }
 559
 560 static void
 561 thread_msg_handle(struct softnic_thread_data *t)
 562 {
 563         for ( ; ; ) {
 564                 struct thread_msg_req *req;
 565                 struct thread_msg_rsp *rsp;
 566
 567                 req = thread_msg_recv(t->msgq_req);
 568                 if (req == NULL)
 569                         break;
 570
 571                 switch (req->type) {
 572                 case THREAD_REQ_PIPELINE_ENABLE:
 573                         rsp = thread_msg_handle_pipeline_enable(t, req);
 574                         break;
 575
 576                 case THREAD_REQ_PIPELINE_DISABLE:
 577                         rsp = thread_msg_handle_pipeline_disable(t, req);
 578                         break;
 579
 580                 default:
 581                         rsp = (struct thread_msg_rsp *)req;
 582                         rsp->status = -1;
 583                 }
 584
 585                 thread_msg_send(t->msgq_rsp, rsp);
 586         }
 587 }
 588
 589 /**
 590  * Master thread & data plane threads: message passing
 591  */
 592 enum pipeline_req_type {
 593         /* Port IN */
 594         PIPELINE_REQ_PORT_IN_STATS_READ,
 595         PIPELINE_REQ_PORT_IN_ENABLE,
 596         PIPELINE_REQ_PORT_IN_DISABLE,
 597
 598         /* Port OUT */
 599         PIPELINE_REQ_PORT_OUT_STATS_READ,
 600
 601         /* Table */
 602         PIPELINE_REQ_TABLE_STATS_READ,
 603         PIPELINE_REQ_TABLE_RULE_ADD,
 604         PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT,
 605         PIPELINE_REQ_TABLE_RULE_ADD_BULK,
 606         PIPELINE_REQ_TABLE_RULE_DELETE,
 607         PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT,
 608         PIPELINE_REQ_TABLE_RULE_STATS_READ,
 609         PIPELINE_REQ_TABLE_MTR_PROFILE_ADD,
 610         PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE,
 611         PIPELINE_REQ_TABLE_RULE_MTR_READ,
 612         PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE,
 613         PIPELINE_REQ_TABLE_RULE_TTL_READ,
 614         PIPELINE_REQ_MAX
 615 };
 616
 617 struct pipeline_msg_req_port_in_stats_read {
 618         int clear;
 619 };
 620
 621 struct pipeline_msg_req_port_out_stats_read {
 622         int clear;
 623 };
 624
 625 struct pipeline_msg_req_table_stats_read {
 626         int clear;
 627 };
 628
 629 struct pipeline_msg_req_table_rule_add {
 630         struct softnic_table_rule_match match;
 631         struct softnic_table_rule_action action;
 632 };
 633
 634 struct pipeline_msg_req_table_rule_add_default {
 635         struct softnic_table_rule_action action;
 636 };
 637
 638 struct pipeline_msg_req_table_rule_add_bulk {
 639         struct softnic_table_rule_match *match;
 640         struct softnic_table_rule_action *action;
 641         void **data;
 642         uint32_t n_rules;
 643         int bulk;
 644 };
 645
 646 struct pipeline_msg_req_table_rule_delete {
 647         struct softnic_table_rule_match match;
 648 };
 649
 650 struct pipeline_msg_req_table_rule_stats_read {
 651         void *data;
 652         int clear;
 653 };
 654
 655 struct pipeline_msg_req_table_mtr_profile_add {
 656         uint32_t meter_profile_id;
 657         struct rte_table_action_meter_profile profile;
 658 };
 659
 660 struct pipeline_msg_req_table_mtr_profile_delete {
 661         uint32_t meter_profile_id;
 662 };
 663
 664 struct pipeline_msg_req_table_rule_mtr_read {
 665         void *data;
 666         uint32_t tc_mask;
 667         int clear;
 668 };
 669
 670 struct pipeline_msg_req_table_dscp_table_update {
 671         uint64_t dscp_mask;
 672         struct rte_table_action_dscp_table dscp_table;
 673 };
 674
 675 struct pipeline_msg_req_table_rule_ttl_read {
 676         void *data;
 677         int clear;
 678 };
 679
 680 struct pipeline_msg_req {
 681         enum pipeline_req_type type;
 682         uint32_t id; /* Port IN, port OUT or table ID */
 683
 684         RTE_STD_C11
 685         union {
 686                 struct pipeline_msg_req_port_in_stats_read port_in_stats_read;
 687                 struct pipeline_msg_req_port_out_stats_read port_out_stats_read;
 688                 struct pipeline_msg_req_table_stats_read table_stats_read;
 689                 struct pipeline_msg_req_table_rule_add table_rule_add;
 690                 struct pipeline_msg_req_table_rule_add_default table_rule_add_default;
 691                 struct pipeline_msg_req_table_rule_add_bulk table_rule_add_bulk;
 692                 struct pipeline_msg_req_table_rule_delete table_rule_delete;
 693                 struct pipeline_msg_req_table_rule_stats_read table_rule_stats_read;
 694                 struct pipeline_msg_req_table_mtr_profile_add table_mtr_profile_add;
 695                 struct pipeline_msg_req_table_mtr_profile_delete table_mtr_profile_delete;
 696                 struct pipeline_msg_req_table_rule_mtr_read table_rule_mtr_read;
 697                 struct pipeline_msg_req_table_dscp_table_update table_dscp_table_update;
 698                 struct pipeline_msg_req_table_rule_ttl_read table_rule_ttl_read;
 699         };
 700 };
 701
 702 struct pipeline_msg_rsp_port_in_stats_read {
 703         struct rte_pipeline_port_in_stats stats;
 704 };
 705
 706 struct pipeline_msg_rsp_port_out_stats_read {
 707         struct rte_pipeline_port_out_stats stats;
 708 };
 709
 710 struct pipeline_msg_rsp_table_stats_read {
 711         struct rte_pipeline_table_stats stats;
 712 };
 713
 714 struct pipeline_msg_rsp_table_rule_add {
 715         void *data;
 716 };
 717
 718 struct pipeline_msg_rsp_table_rule_add_default {
 719         void *data;
 720 };
 721
 722 struct pipeline_msg_rsp_table_rule_add_bulk {
 723         uint32_t n_rules;
 724 };
 725
 726 struct pipeline_msg_rsp_table_rule_stats_read {
 727         struct rte_table_action_stats_counters stats;
 728 };
 729
 730 struct pipeline_msg_rsp_table_rule_mtr_read {
 731         struct rte_table_action_mtr_counters stats;
 732 };
 733
 734 struct pipeline_msg_rsp_table_rule_ttl_read {
 735         struct rte_table_action_ttl_counters stats;
 736 };
 737
 738 struct pipeline_msg_rsp {
 739         int status;
 740
 741         RTE_STD_C11
 742         union {
 743                 struct pipeline_msg_rsp_port_in_stats_read port_in_stats_read;
 744                 struct pipeline_msg_rsp_port_out_stats_read port_out_stats_read;
 745                 struct pipeline_msg_rsp_table_stats_read table_stats_read;
 746                 struct pipeline_msg_rsp_table_rule_add table_rule_add;
 747                 struct pipeline_msg_rsp_table_rule_add_default table_rule_add_default;
 748                 struct pipeline_msg_rsp_table_rule_add_bulk table_rule_add_bulk;
 749                 struct pipeline_msg_rsp_table_rule_stats_read table_rule_stats_read;
 750                 struct pipeline_msg_rsp_table_rule_mtr_read table_rule_mtr_read;
 751                 struct pipeline_msg_rsp_table_rule_ttl_read table_rule_ttl_read;
 752         };
 753 };
 754
 755 /**
 756  * Master thread
 757  */
 758 static struct pipeline_msg_req *
 759 pipeline_msg_alloc(void)
 760 {
 761         size_t size = RTE_MAX(sizeof(struct pipeline_msg_req),
 762                 sizeof(struct pipeline_msg_rsp));
 763
 764         return calloc(1, size);
 765 }
 766
 767 static void
 768 pipeline_msg_free(struct pipeline_msg_rsp *rsp)
 769 {
 770         free(rsp);
 771 }
 772
 773 static struct pipeline_msg_rsp *
 774 pipeline_msg_send_recv(struct pipeline *p,
 775         struct pipeline_msg_req *req)
 776 {
 777         struct rte_ring *msgq_req = p->msgq_req;
 778         struct rte_ring *msgq_rsp = p->msgq_rsp;
 779         struct pipeline_msg_rsp *rsp;
 780         int status;
 781
 782         /* send */
 783         do {
 784                 status = rte_ring_sp_enqueue(msgq_req, req);
 785         } while (status == -ENOBUFS);
 786
 787         /* recv */
 788         do {
 789                 status = rte_ring_sc_dequeue(msgq_rsp, (void **)&rsp);
 790         } while (status != 0);
 791
 792         return rsp;
 793 }
 794
 795 int
 796 softnic_pipeline_port_in_stats_read(struct pmd_internals *softnic,
 797         const char *pipeline_name,
 798         uint32_t port_id,
 799         struct rte_pipeline_port_in_stats *stats,
 800         int clear)
 801 {
 802         struct pipeline *p;
 803         struct pipeline_msg_req *req;
 804         struct pipeline_msg_rsp *rsp;
 805         int status;
 806
 807         /* Check input params */
 808         if (pipeline_name == NULL ||
 809                 stats == NULL)
 810                 return -1;
 811
 812         p = softnic_pipeline_find(softnic, pipeline_name);
 813         if (p == NULL ||
 814                 port_id >= p->n_ports_in)
 815                 return -1;
 816
 817         if (!pipeline_is_running(p)) {
 818                 status = rte_pipeline_port_in_stats_read(p->p,
 819                         port_id,
 820                         stats,
 821                         clear);
 822
 823                 return status;
 824         }
 825
 826         /* Allocate request */
 827         req = pipeline_msg_alloc();
 828         if (req == NULL)
 829                 return -1;
 830
 831         /* Write request */
 832         req->type = PIPELINE_REQ_PORT_IN_STATS_READ;
 833         req->id = port_id;
 834         req->port_in_stats_read.clear = clear;
 835
 836         /* Send request and wait for response */
 837         rsp = pipeline_msg_send_recv(p, req);
 838         if (rsp == NULL)
 839                 return -1;
 840
 841         /* Read response */
 842         status = rsp->status;
 843         if (status)
 844                 memcpy(stats, &rsp->port_in_stats_read.stats, sizeof(*stats));
 845
 846         /* Free response */
 847         pipeline_msg_free(rsp);
 848
 849         return status;
 850 }
 851
 852 int
 853 softnic_pipeline_port_in_enable(struct pmd_internals *softnic,
 854         const char *pipeline_name,
 855         uint32_t port_id)
 856 {
 857         struct pipeline *p;
 858         struct pipeline_msg_req *req;
 859         struct pipeline_msg_rsp *rsp;
 860         int status;
 861
 862         /* Check input params */
 863         if (pipeline_name == NULL)
 864                 return -1;
 865
 866         p = softnic_pipeline_find(softnic, pipeline_name);
 867         if (p == NULL ||
 868                 port_id >= p->n_ports_in)
 869                 return -1;
 870
 871         if (!pipeline_is_running(p)) {
 872                 status = rte_pipeline_port_in_enable(p->p, port_id);
 873                 return status;
 874         }
 875
 876         /* Allocate request */
 877         req = pipeline_msg_alloc();
 878         if (req == NULL)
 879                 return -1;
 880
 881         /* Write request */
 882         req->type = PIPELINE_REQ_PORT_IN_ENABLE;
 883         req->id = port_id;
 884
 885         /* Send request and wait for response */
 886         rsp = pipeline_msg_send_recv(p, req);
 887         if (rsp == NULL)
 888                 return -1;
 889
 890         /* Read response */
 891         status = rsp->status;
 892
 893         /* Free response */
 894         pipeline_msg_free(rsp);
 895
 896         return status;
 897 }
 898
 899 int
 900 softnic_pipeline_port_in_disable(struct pmd_internals *softnic,
 901         const char *pipeline_name,
 902         uint32_t port_id)
 903 {
 904         struct pipeline *p;
 905         struct pipeline_msg_req *req;
 906         struct pipeline_msg_rsp *rsp;
 907         int status;
 908
 909         /* Check input params */
 910         if (pipeline_name == NULL)
 911                 return -1;
 912
 913         p = softnic_pipeline_find(softnic, pipeline_name);
 914         if (p == NULL ||
 915                 port_id >= p->n_ports_in)
 916                 return -1;
 917
 918         if (!pipeline_is_running(p)) {
 919                 status = rte_pipeline_port_in_disable(p->p, port_id);
 920                 return status;
 921         }
 922
 923         /* Allocate request */
 924         req = pipeline_msg_alloc();
 925         if (req == NULL)
 926                 return -1;
 927
 928         /* Write request */
 929         req->type = PIPELINE_REQ_PORT_IN_DISABLE;
 930         req->id = port_id;
 931
 932         /* Send request and wait for response */
 933         rsp = pipeline_msg_send_recv(p, req);
 934         if (rsp == NULL)
 935                 return -1;
 936
 937         /* Read response */
 938         status = rsp->status;
 939
 940         /* Free response */
 941         pipeline_msg_free(rsp);
 942
 943         return status;
 944 }
 945
 946 int
 947 softnic_pipeline_port_out_stats_read(struct pmd_internals *softnic,
 948         const char *pipeline_name,
 949         uint32_t port_id,
 950         struct rte_pipeline_port_out_stats *stats,
 951         int clear)
 952 {
 953         struct pipeline *p;
 954         struct pipeline_msg_req *req;
 955         struct pipeline_msg_rsp *rsp;
 956         int status;
 957
 958         /* Check input params */
 959         if (pipeline_name == NULL ||
 960                 stats == NULL)
 961                 return -1;
 962
 963         p = softnic_pipeline_find(softnic, pipeline_name);
 964         if (p == NULL ||
 965                 port_id >= p->n_ports_out)
 966                 return -1;
 967
 968         if (!pipeline_is_running(p)) {
 969                 status = rte_pipeline_port_out_stats_read(p->p,
 970                         port_id,
 971                         stats,
 972                         clear);
 973
 974                 return status;
 975         }
 976
 977         /* Allocate request */
 978         req = pipeline_msg_alloc();
 979         if (req == NULL)
 980                 return -1;
 981
 982         /* Write request */
 983         req->type = PIPELINE_REQ_PORT_OUT_STATS_READ;
 984         req->id = port_id;
 985         req->port_out_stats_read.clear = clear;
 986
 987         /* Send request and wait for response */
 988         rsp = pipeline_msg_send_recv(p, req);
 989         if (rsp == NULL)
 990                 return -1;
 991
 992         /* Read response */
 993         status = rsp->status;
 994         if (status)
 995                 memcpy(stats, &rsp->port_out_stats_read.stats, sizeof(*stats));
 996
 997         /* Free response */
 998         pipeline_msg_free(rsp);
 999
1000         return status;
1001 }
1002
1003 int
1004 softnic_pipeline_table_stats_read(struct pmd_internals *softnic,
1005         const char *pipeline_name,
1006         uint32_t table_id,
1007         struct rte_pipeline_table_stats *stats,
1008         int clear)
1009 {
1010         struct pipeline *p;
1011         struct pipeline_msg_req *req;
1012         struct pipeline_msg_rsp *rsp;
1013         int status;
1014
1015         /* Check input params */
1016         if (pipeline_name == NULL ||
1017                 stats == NULL)
1018                 return -1;
1019
1020         p = softnic_pipeline_find(softnic, pipeline_name);
1021         if (p == NULL ||
1022                 table_id >= p->n_tables)
1023                 return -1;
1024
1025         if (!pipeline_is_running(p)) {
1026                 status = rte_pipeline_table_stats_read(p->p,
1027                         table_id,
1028                         stats,
1029                         clear);
1030
1031                 return status;
1032         }
1033
1034         /* Allocate request */
1035         req = pipeline_msg_alloc();
1036         if (req == NULL)
1037                 return -1;
1038
1039         /* Write request */
1040         req->type = PIPELINE_REQ_TABLE_STATS_READ;
1041         req->id = table_id;
1042         req->table_stats_read.clear = clear;
1043
1044         /* Send request and wait for response */
1045         rsp = pipeline_msg_send_recv(p, req);
1046         if (rsp == NULL)
1047                 return -1;
1048
1049         /* Read response */
1050         status = rsp->status;
1051         if (status)
1052                 memcpy(stats, &rsp->table_stats_read.stats, sizeof(*stats));
1053
1054         /* Free response */
1055         pipeline_msg_free(rsp);
1056
1057         return status;
1058 }
1059
1060 static int
1061 match_check(struct softnic_table_rule_match *match,
1062         struct pipeline *p,
1063         uint32_t table_id)
1064 {
1065         struct softnic_table *table;
1066
1067         if (match == NULL ||
1068                 p == NULL ||
1069                 table_id >= p->n_tables)
1070                 return -1;
1071
1072         table = &p->table[table_id];
1073         if (match->match_type != table->params.match_type)
1074                 return -1;
1075
1076         switch (match->match_type) {
1077         case TABLE_ACL:
1078         {
1079                 struct softnic_table_acl_params *t = &table->params.match.acl;
1080                 struct softnic_table_rule_match_acl *r = &match->match.acl;
1081
1082                 if ((r->ip_version && (t->ip_version == 0)) ||
1083                         ((r->ip_version == 0) && t->ip_version))
1084                         return -1;
1085
1086                 if (r->ip_version) {
1087                         if (r->sa_depth > 32 ||
1088                                 r->da_depth > 32)
1089                                 return -1;
1090                 } else {
1091                         if (r->sa_depth > 128 ||
1092                                 r->da_depth > 128)
1093                                 return -1;
1094                 }
1095                 return 0;
1096         }
1097
1098         case TABLE_ARRAY:
1099                 return 0;
1100
1101         case TABLE_HASH:
1102                 return 0;
1103
1104         case TABLE_LPM:
1105         {
1106                 struct softnic_table_lpm_params *t = &table->params.match.lpm;
1107                 struct softnic_table_rule_match_lpm *r = &match->match.lpm;
1108
1109                 if ((r->ip_version && (t->key_size != 4)) ||
1110                         ((r->ip_version == 0) && (t->key_size != 16)))
1111                         return -1;
1112
1113                 if (r->ip_version) {
1114                         if (r->depth > 32)
1115                                 return -1;
1116                 } else {
1117                         if (r->depth > 128)
1118                                 return -1;
1119                 }
1120                 return 0;
1121         }
1122
1123         case TABLE_STUB:
1124                 return -1;
1125
1126         default:
1127                 return -1;
1128         }
1129 }
1130
1131 static int
1132 action_check(struct softnic_table_rule_action *action,
1133         struct pipeline *p,
1134         uint32_t table_id)
1135 {
1136         struct softnic_table_action_profile *ap;
1137
1138         if (action == NULL ||
1139                 p == NULL ||
1140                 table_id >= p->n_tables)
1141                 return -1;
1142
1143         ap = p->table[table_id].ap;
1144         if (action->action_mask != ap->params.action_mask)
1145                 return -1;
1146
1147         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1148                 if (action->fwd.action == RTE_PIPELINE_ACTION_PORT &&
1149                         action->fwd.id >= p->n_ports_out)
1150                         return -1;
1151
1152                 if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE &&
1153                         action->fwd.id >= p->n_tables)
1154                         return -1;
1155         }
1156
1157         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1158                 uint32_t tc_mask0 = (1 << ap->params.mtr.n_tc) - 1;
1159                 uint32_t tc_mask1 = action->mtr.tc_mask;
1160
1161                 if (tc_mask1 != tc_mask0)
1162                         return -1;
1163         }
1164
1165         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1166                 uint32_t n_subports_per_port =
1167                         ap->params.tm.n_subports_per_port;
1168                 uint32_t n_pipes_per_subport =
1169                         ap->params.tm.n_pipes_per_subport;
1170                 uint32_t subport_id = action->tm.subport_id;
1171                 uint32_t pipe_id = action->tm.pipe_id;
1172
1173                 if (subport_id >= n_subports_per_port ||
1174                         pipe_id >= n_pipes_per_subport)
1175                         return -1;
1176         }
1177
1178         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1179                 uint64_t encap_mask = ap->params.encap.encap_mask;
1180                 enum rte_table_action_encap_type type = action->encap.type;
1181
1182                 if ((encap_mask & (1LLU << type)) == 0)
1183                         return -1;
1184         }
1185
1186         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1187                 int ip_version0 = ap->params.common.ip_version;
1188                 int ip_version1 = action->nat.ip_version;
1189
1190                 if ((ip_version1 && (ip_version0 == 0)) ||
1191                         ((ip_version1 == 0) && ip_version0))
1192                         return -1;
1193         }
1194
1195         return 0;
1196 }
1197
1198 static int
1199 action_default_check(struct softnic_table_rule_action *action,
1200         struct pipeline *p,
1201         uint32_t table_id)
1202 {
1203         if (action == NULL ||
1204                 action->action_mask != (1LLU << RTE_TABLE_ACTION_FWD) ||
1205                 p == NULL ||
1206                 table_id >= p->n_tables)
1207                 return -1;
1208
1209         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1210                 if (action->fwd.action == RTE_PIPELINE_ACTION_PORT &&
1211                         action->fwd.id >= p->n_ports_out)
1212                         return -1;
1213
1214                 if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE &&
1215                         action->fwd.id >= p->n_tables)
1216                         return -1;
1217         }
1218
1219         return 0;
1220 }
1221
1222 union table_rule_match_low_level {
1223         struct rte_table_acl_rule_add_params acl_add;
1224         struct rte_table_acl_rule_delete_params acl_delete;
1225         struct rte_table_array_key array;
1226         uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
1227         struct rte_table_lpm_key lpm_ipv4;
1228         struct rte_table_lpm_ipv6_key lpm_ipv6;
1229 };
1230
1231 static int
1232 match_convert(struct softnic_table_rule_match *mh,
1233         union table_rule_match_low_level *ml,
1234         int add);
1235
1236 static int
1237 action_convert(struct rte_table_action *a,
1238         struct softnic_table_rule_action *action,
1239         struct rte_pipeline_table_entry *data);
1240
1241 int
1242 softnic_pipeline_table_rule_add(struct pmd_internals *softnic,
1243         const char *pipeline_name,
1244         uint32_t table_id,
1245         struct softnic_table_rule_match *match,
1246         struct softnic_table_rule_action *action,
1247         void **data)
1248 {
1249         struct pipeline *p;
1250         struct pipeline_msg_req *req;
1251         struct pipeline_msg_rsp *rsp;
1252         int status;
1253
1254         /* Check input params */
1255         if (pipeline_name == NULL ||
1256                 match == NULL ||
1257                 action == NULL ||
1258                 data == NULL)
1259                 return -1;
1260
1261         p = softnic_pipeline_find(softnic, pipeline_name);
1262         if (p == NULL ||
1263                 table_id >= p->n_tables ||
1264                 match_check(match, p, table_id) ||
1265                 action_check(action, p, table_id))
1266                 return -1;
1267
1268         if (!pipeline_is_running(p)) {
1269                 struct rte_table_action *a = p->table[table_id].a;
1270                 union table_rule_match_low_level match_ll;
1271                 struct rte_pipeline_table_entry *data_in, *data_out;
1272                 int key_found;
1273                 uint8_t *buffer;
1274
1275                 buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
1276                 if (buffer == NULL)
1277                         return -1;
1278
1279                 /* Table match-action rule conversion */
1280                 data_in = (struct rte_pipeline_table_entry *)buffer;
1281
1282                 status = match_convert(match, &match_ll, 1);
1283                 if (status) {
1284                         free(buffer);
1285                         return -1;
1286                 }
1287
1288                 status = action_convert(a, action, data_in);
1289                 if (status) {
1290                         free(buffer);
1291                         return -1;
1292                 }
1293
1294                 /* Add rule (match, action) to table */
1295                 status = rte_pipeline_table_entry_add(p->p,
1296                                 table_id,
1297                                 &match_ll,
1298                                 data_in,
1299                                 &key_found,
1300                                 &data_out);
1301                 if (status) {
1302                         free(buffer);
1303                         return -1;
1304                 }
1305
1306                 /* Write Response */
1307                 *data = data_out;
1308
1309                 free(buffer);
1310                 return 0;
1311         }
1312
1313         /* Allocate request */
1314         req = pipeline_msg_alloc();
1315         if (req == NULL)
1316                 return -1;
1317
1318         /* Write request */
1319         req->type = PIPELINE_REQ_TABLE_RULE_ADD;
1320         req->id = table_id;
1321         memcpy(&req->table_rule_add.match, match, sizeof(*match));
1322         memcpy(&req->table_rule_add.action, action, sizeof(*action));
1323
1324         /* Send request and wait for response */
1325         rsp = pipeline_msg_send_recv(p, req);
1326         if (rsp == NULL)
1327                 return -1;
1328
1329         /* Read response */
1330         status = rsp->status;
1331         if (status == 0)
1332                 *data = rsp->table_rule_add.data;
1333
1334         /* Free response */
1335         pipeline_msg_free(rsp);
1336
1337         return status;
1338 }
1339
1340 int
1341 softnic_pipeline_table_rule_add_default(struct pmd_internals *softnic,
1342         const char *pipeline_name,
1343         uint32_t table_id,
1344         struct softnic_table_rule_action *action,
1345         void **data)
1346 {
1347         struct pipeline *p;
1348         struct pipeline_msg_req *req;
1349         struct pipeline_msg_rsp *rsp;
1350         int status;
1351
1352         /* Check input params */
1353         if (pipeline_name == NULL ||
1354                 action == NULL ||
1355                 data == NULL)
1356                 return -1;
1357
1358         p = softnic_pipeline_find(softnic, pipeline_name);
1359         if (p == NULL ||
1360                 table_id >= p->n_tables ||
1361                 action_default_check(action, p, table_id))
1362                 return -1;
1363
1364         if (!pipeline_is_running(p)) {
1365                 struct rte_pipeline_table_entry *data_in, *data_out;
1366                 uint8_t *buffer;
1367
1368                 buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
1369                 if (buffer == NULL)
1370                         return -1;
1371
1372                 /* Apply actions */
1373                 data_in = (struct rte_pipeline_table_entry *)buffer;
1374
1375                 data_in->action = action->fwd.action;
1376                 if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
1377                         data_in->port_id = action->fwd.id;
1378                 if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
1379                         data_in->table_id = action->fwd.id;
1380
1381                 /* Add default rule to table */
1382                 status = rte_pipeline_table_default_entry_add(p->p,
1383                                 table_id,
1384                                 data_in,
1385                                 &data_out);
1386                 if (status) {
1387                         free(buffer);
1388                         return -1;
1389                 }
1390
1391                 /* Write Response */
1392                 *data = data_out;
1393
1394                 free(buffer);
1395                 return 0;
1396         }
1397
1398         /* Allocate request */
1399         req = pipeline_msg_alloc();
1400         if (req == NULL)
1401                 return -1;
1402
1403         /* Write request */
1404         req->type = PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT;
1405         req->id = table_id;
1406         memcpy(&req->table_rule_add_default.action, action, sizeof(*action));
1407
1408         /* Send request and wait for response */
1409         rsp = pipeline_msg_send_recv(p, req);
1410         if (rsp == NULL)
1411                 return -1;
1412
1413         /* Read response */
1414         status = rsp->status;
1415         if (status == 0)
1416                 *data = rsp->table_rule_add_default.data;
1417
1418         /* Free response */
1419         pipeline_msg_free(rsp);
1420
1421         return status;
1422 }
1423
1424 int
1425 softnic_pipeline_table_rule_add_bulk(struct pmd_internals *softnic,
1426         const char *pipeline_name,
1427         uint32_t table_id,
1428         struct softnic_table_rule_match *match,
1429         struct softnic_table_rule_action *action,
1430         void **data,
1431         uint32_t *n_rules)
1432 {
1433         struct pipeline *p;
1434         struct pipeline_msg_req *req;
1435         struct pipeline_msg_rsp *rsp;
1436         uint32_t i;
1437         int status;
1438
1439         /* Check input params */
1440         if (pipeline_name == NULL ||
1441                 match == NULL ||
1442                 action == NULL ||
1443                 data == NULL ||
1444                 n_rules == NULL ||
1445                 (*n_rules == 0))
1446                 return -1;
1447
1448         p = softnic_pipeline_find(softnic, pipeline_name);
1449         if (p == NULL ||
1450                 table_id >= p->n_tables)
1451                 return -1;
1452
1453         for (i = 0; i < *n_rules; i++)
1454                 if (match_check(match, p, table_id) ||
1455                         action_check(action, p, table_id))
1456                         return -1;
1457
1458         if (!pipeline_is_running(p)) {
1459                 struct rte_table_action *a = p->table[table_id].a;
1460                 union table_rule_match_low_level *match_ll;
1461                 uint8_t *action_ll;
1462                 void **match_ll_ptr;
1463                 struct rte_pipeline_table_entry **action_ll_ptr;
1464                 struct rte_pipeline_table_entry **entries_ptr =
1465                         (struct rte_pipeline_table_entry **)data;
1466                 uint32_t bulk =
1467                         (p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
1468                 int *found;
1469
1470                 /* Memory allocation */
1471                 match_ll = calloc(*n_rules, sizeof(union table_rule_match_low_level));
1472                 action_ll = calloc(*n_rules, TABLE_RULE_ACTION_SIZE_MAX);
1473                 match_ll_ptr = calloc(*n_rules, sizeof(void *));
1474                 action_ll_ptr =
1475                         calloc(*n_rules, sizeof(struct rte_pipeline_table_entry *));
1476                 found = calloc(*n_rules, sizeof(int));
1477
1478                 if (match_ll == NULL ||
1479                         action_ll == NULL ||
1480                         match_ll_ptr == NULL ||
1481                         action_ll_ptr == NULL ||
1482                         found == NULL)
1483                         goto fail;
1484
1485                 for (i = 0; i < *n_rules; i++) {
1486                         match_ll_ptr[i] = (void *)&match_ll[i];
1487                         action_ll_ptr[i] =
1488                                 (struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
1489                 }
1490
1491                 /* Rule match conversion */
1492                 for (i = 0; i < *n_rules; i++) {
1493                         status = match_convert(&match[i], match_ll_ptr[i], 1);
1494                         if (status)
1495                                 goto fail;
1496                 }
1497
1498                 /* Rule action conversion */
1499                 for (i = 0; i < *n_rules; i++) {
1500                         status = action_convert(a, &action[i], action_ll_ptr[i]);
1501                         if (status)
1502                                 goto fail;
1503                 }
1504
1505                 /* Add rule (match, action) to table */
1506                 if (bulk) {
1507                         status = rte_pipeline_table_entry_add_bulk(p->p,
1508                                 table_id,
1509                                 match_ll_ptr,
1510                                 action_ll_ptr,
1511                                 *n_rules,
1512                                 found,
1513                                 entries_ptr);
1514                         if (status)
1515                                 *n_rules = 0;
1516                 } else {
1517                         for (i = 0; i < *n_rules; i++) {
1518                                 status = rte_pipeline_table_entry_add(p->p,
1519                                         table_id,
1520                                         match_ll_ptr[i],
1521                                         action_ll_ptr[i],
1522                                         &found[i],
1523                                         &entries_ptr[i]);
1524                                 if (status) {
1525                                         *n_rules = i;
1526                                         break;
1527                                 }
1528                         }
1529                 }
1530
1531                 /* Free */
1532                 free(found);
1533                 free(action_ll_ptr);
1534                 free(match_ll_ptr);
1535                 free(action_ll);
1536                 free(match_ll);
1537
1538                 return status;
1539
1540 fail:
1541                 free(found);
1542                 free(action_ll_ptr);
1543                 free(match_ll_ptr);
1544                 free(action_ll);
1545                 free(match_ll);
1546
1547                 *n_rules = 0;
1548                 return -1;
1549         }
1550
1551         /* Allocate request */
1552         req = pipeline_msg_alloc();
1553         if (req == NULL)
1554                 return -1;
1555
1556         /* Write request */
1557         req->type = PIPELINE_REQ_TABLE_RULE_ADD_BULK;
1558         req->id = table_id;
1559         req->table_rule_add_bulk.match = match;
1560         req->table_rule_add_bulk.action = action;
1561         req->table_rule_add_bulk.data = data;
1562         req->table_rule_add_bulk.n_rules = *n_rules;
1563         req->table_rule_add_bulk.bulk =
1564                 (p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
1565
1566         /* Send request and wait for response */
1567         rsp = pipeline_msg_send_recv(p, req);
1568         if (rsp == NULL)
1569                 return -1;
1570
1571         /* Read response */
1572         status = rsp->status;
1573         if (status == 0)
1574                 *n_rules = rsp->table_rule_add_bulk.n_rules;
1575
1576         /* Free response */
1577         pipeline_msg_free(rsp);
1578
1579         return status;
1580 }
1581
1582 int
1583 softnic_pipeline_table_rule_delete(struct pmd_internals *softnic,
1584         const char *pipeline_name,
1585         uint32_t table_id,
1586         struct softnic_table_rule_match *match)
1587 {
1588         struct pipeline *p;
1589         struct pipeline_msg_req *req;
1590         struct pipeline_msg_rsp *rsp;
1591         int status;
1592
1593         /* Check input params */
1594         if (pipeline_name == NULL ||
1595                 match == NULL)
1596                 return -1;
1597
1598         p = softnic_pipeline_find(softnic, pipeline_name);
1599         if (p == NULL ||
1600                 table_id >= p->n_tables ||
1601                 match_check(match, p, table_id))
1602                 return -1;
1603
1604         if (!pipeline_is_running(p)) {
1605                 union table_rule_match_low_level match_ll;
1606                 int key_found;
1607
1608                 status = match_convert(match, &match_ll, 0);
1609                 if (status)
1610                         return -1;
1611
1612                 status = rte_pipeline_table_entry_delete(p->p,
1613                                 table_id,
1614                                 &match_ll,
1615                                 &key_found,
1616                                 NULL);
1617
1618                 return status;
1619         }
1620
1621         /* Allocate request */
1622         req = pipeline_msg_alloc();
1623         if (req == NULL)
1624                 return -1;
1625
1626         /* Write request */
1627         req->type = PIPELINE_REQ_TABLE_RULE_DELETE;
1628         req->id = table_id;
1629         memcpy(&req->table_rule_delete.match, match, sizeof(*match));
1630
1631         /* Send request and wait for response */
1632         rsp = pipeline_msg_send_recv(p, req);
1633         if (rsp == NULL)
1634                 return -1;
1635
1636         /* Read response */
1637         status = rsp->status;
1638
1639         /* Free response */
1640         pipeline_msg_free(rsp);
1641
1642         return status;
1643 }
1644
1645 int
1646 softnic_pipeline_table_rule_delete_default(struct pmd_internals *softnic,
1647         const char *pipeline_name,
1648         uint32_t table_id)
1649 {
1650         struct pipeline *p;
1651         struct pipeline_msg_req *req;
1652         struct pipeline_msg_rsp *rsp;
1653         int status;
1654
1655         /* Check input params */
1656         if (pipeline_name == NULL)
1657                 return -1;
1658
1659         p = softnic_pipeline_find(softnic, pipeline_name);
1660         if (p == NULL ||
1661                 table_id >= p->n_tables)
1662                 return -1;
1663
1664         if (!pipeline_is_running(p)) {
1665                 status = rte_pipeline_table_default_entry_delete(p->p,
1666                         table_id,
1667                         NULL);
1668
1669                 return status;
1670         }
1671
1672         /* Allocate request */
1673         req = pipeline_msg_alloc();
1674         if (req == NULL)
1675                 return -1;
1676
1677         /* Write request */
1678         req->type = PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT;
1679         req->id = table_id;
1680
1681         /* Send request and wait for response */
1682         rsp = pipeline_msg_send_recv(p, req);
1683         if (rsp == NULL)
1684                 return -1;
1685
1686         /* Read response */
1687         status = rsp->status;
1688
1689         /* Free response */
1690         pipeline_msg_free(rsp);
1691
1692         return status;
1693 }
1694
1695 int
1696 softnic_pipeline_table_rule_stats_read(struct pmd_internals *softnic,
1697         const char *pipeline_name,
1698         uint32_t table_id,
1699         void *data,
1700         struct rte_table_action_stats_counters *stats,
1701         int clear)
1702 {
1703         struct pipeline *p;
1704         struct pipeline_msg_req *req;
1705         struct pipeline_msg_rsp *rsp;
1706         int status;
1707
1708         /* Check input params */
1709         if (pipeline_name == NULL ||
1710                 data == NULL ||
1711                 stats == NULL)
1712                 return -1;
1713
1714         p = softnic_pipeline_find(softnic, pipeline_name);
1715         if (p == NULL ||
1716                 table_id >= p->n_tables)
1717                 return -1;
1718
1719         if (!pipeline_is_running(p)) {
1720                 struct rte_table_action *a = p->table[table_id].a;
1721
1722                 status = rte_table_action_stats_read(a,
1723                         data,
1724                         stats,
1725                         clear);
1726
1727                 return status;
1728         }
1729
1730         /* Allocate request */
1731         req = pipeline_msg_alloc();
1732         if (req == NULL)
1733                 return -1;
1734
1735         /* Write request */
1736         req->type = PIPELINE_REQ_TABLE_RULE_STATS_READ;
1737         req->id = table_id;
1738         req->table_rule_stats_read.data = data;
1739         req->table_rule_stats_read.clear = clear;
1740
1741         /* Send request and wait for response */
1742         rsp = pipeline_msg_send_recv(p, req);
1743         if (rsp == NULL)
1744                 return -1;
1745
1746         /* Read response */
1747         status = rsp->status;
1748         if (status)
1749                 memcpy(stats, &rsp->table_rule_stats_read.stats, sizeof(*stats));
1750
1751         /* Free response */
1752         pipeline_msg_free(rsp);
1753
1754         return status;
1755 }
1756
1757 int
1758 softnic_pipeline_table_mtr_profile_add(struct pmd_internals *softnic,
1759         const char *pipeline_name,
1760         uint32_t table_id,
1761         uint32_t meter_profile_id,
1762         struct rte_table_action_meter_profile *profile)
1763 {
1764         struct pipeline *p;
1765         struct pipeline_msg_req *req;
1766         struct pipeline_msg_rsp *rsp;
1767         struct softnic_table *table;
1768         struct softnic_table_meter_profile *mp;
1769         int status;
1770
1771         /* Check input params */
1772         if (pipeline_name == NULL ||
1773                 profile == NULL)
1774                 return -1;
1775
1776         p = softnic_pipeline_find(softnic, pipeline_name);
1777         if (p == NULL ||
1778                 table_id >= p->n_tables)
1779                 return -1;
1780
1781         table = &p->table[table_id];
1782         mp = softnic_pipeline_table_meter_profile_find(table, meter_profile_id);
1783         if (mp)
1784                 return -1;
1785
1786         /* Resource Allocation */
1787         mp = calloc(1, sizeof(struct softnic_table_meter_profile));
1788         if (mp == NULL)
1789                 return -1;
1790
1791         mp->meter_profile_id = meter_profile_id;
1792         memcpy(&mp->profile, profile, sizeof(mp->profile));
1793
1794         if (!pipeline_is_running(p)) {
1795                 status = rte_table_action_meter_profile_add(table->a,
1796                         meter_profile_id,
1797                         profile);
1798                 if (status) {
1799                         free(mp);
1800                         return status;
1801                 }
1802
1803                 /* Add profile to the table. */
1804                 TAILQ_INSERT_TAIL(&table->meter_profiles, mp, node);
1805
1806                 return status;
1807         }
1808
1809         /* Allocate request */
1810         req = pipeline_msg_alloc();
1811         if (req == NULL) {
1812                 free(mp);
1813                 return -1;
1814         }
1815
1816         /* Write request */
1817         req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_ADD;
1818         req->id = table_id;
1819         req->table_mtr_profile_add.meter_profile_id = meter_profile_id;
1820         memcpy(&req->table_mtr_profile_add.profile, profile, sizeof(*profile));
1821
1822         /* Send request and wait for response */
1823         rsp = pipeline_msg_send_recv(p, req);
1824         if (rsp == NULL) {
1825                 free(mp);
1826                 return -1;
1827         }
1828
1829         /* Read response */
1830         status = rsp->status;
1831         if (status == 0)
1832                 TAILQ_INSERT_TAIL(&table->meter_profiles, mp, node);
1833         else
1834                 free(mp);
1835
1836         /* Free response */
1837         pipeline_msg_free(rsp);
1838
1839         return status;
1840 }
1841
1842 int
1843 softnic_pipeline_table_mtr_profile_delete(struct pmd_internals *softnic,
1844         const char *pipeline_name,
1845         uint32_t table_id,
1846         uint32_t meter_profile_id)
1847 {
1848         struct pipeline *p;
1849         struct pipeline_msg_req *req;
1850         struct pipeline_msg_rsp *rsp;
1851         int status;
1852
1853         /* Check input params */
1854         if (pipeline_name == NULL)
1855                 return -1;
1856
1857         p = softnic_pipeline_find(softnic, pipeline_name);
1858         if (p == NULL ||
1859                 table_id >= p->n_tables)
1860                 return -1;
1861
1862         if (!pipeline_is_running(p)) {
1863                 struct rte_table_action *a = p->table[table_id].a;
1864
1865                 status = rte_table_action_meter_profile_delete(a,
1866                                 meter_profile_id);
1867
1868                 return status;
1869         }
1870
1871         /* Allocate request */
1872         req = pipeline_msg_alloc();
1873         if (req == NULL)
1874                 return -1;
1875
1876         /* Write request */
1877         req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE;
1878         req->id = table_id;
1879         req->table_mtr_profile_delete.meter_profile_id = meter_profile_id;
1880
1881         /* Send request and wait for response */
1882         rsp = pipeline_msg_send_recv(p, req);
1883         if (rsp == NULL)
1884                 return -1;
1885
1886         /* Read response */
1887         status = rsp->status;
1888
1889         /* Free response */
1890         pipeline_msg_free(rsp);
1891
1892         return status;
1893 }
1894
1895 int
1896 softnic_pipeline_table_rule_mtr_read(struct pmd_internals *softnic,
1897         const char *pipeline_name,
1898         uint32_t table_id,
1899         void *data,
1900         uint32_t tc_mask,
1901         struct rte_table_action_mtr_counters *stats,
1902         int clear)
1903 {
1904         struct pipeline *p;
1905         struct pipeline_msg_req *req;
1906         struct pipeline_msg_rsp *rsp;
1907         int status;
1908
1909         /* Check input params */
1910         if (pipeline_name == NULL ||
1911                 data == NULL ||
1912                 stats == NULL)
1913                 return -1;
1914
1915         p = softnic_pipeline_find(softnic, pipeline_name);
1916         if (p == NULL ||
1917                 table_id >= p->n_tables)
1918                 return -1;
1919
1920         if (!pipeline_is_running(p)) {
1921                 struct rte_table_action *a = p->table[table_id].a;
1922
1923                 status = rte_table_action_meter_read(a,
1924                                 data,
1925                                 tc_mask,
1926                                 stats,
1927                                 clear);
1928
1929                 return status;
1930         }
1931
1932         /* Allocate request */
1933         req = pipeline_msg_alloc();
1934         if (req == NULL)
1935                 return -1;
1936
1937         /* Write request */
1938         req->type = PIPELINE_REQ_TABLE_RULE_MTR_READ;
1939         req->id = table_id;
1940         req->table_rule_mtr_read.data = data;
1941         req->table_rule_mtr_read.tc_mask = tc_mask;
1942         req->table_rule_mtr_read.clear = clear;
1943
1944         /* Send request and wait for response */
1945         rsp = pipeline_msg_send_recv(p, req);
1946         if (rsp == NULL)
1947                 return -1;
1948
1949         /* Read response */
1950         status = rsp->status;
1951         if (status)
1952                 memcpy(stats, &rsp->table_rule_mtr_read.stats, sizeof(*stats));
1953
1954         /* Free response */
1955         pipeline_msg_free(rsp);
1956
1957         return status;
1958 }
1959
1960 int
1961 softnic_pipeline_table_dscp_table_update(struct pmd_internals *softnic,
1962         const char *pipeline_name,
1963         uint32_t table_id,
1964         uint64_t dscp_mask,
1965         struct rte_table_action_dscp_table *dscp_table)
1966 {
1967         struct pipeline *p;
1968         struct pipeline_msg_req *req;
1969         struct pipeline_msg_rsp *rsp;
1970         int status;
1971
1972         /* Check input params */
1973         if (pipeline_name == NULL ||
1974                 dscp_table == NULL)
1975                 return -1;
1976
1977         p = softnic_pipeline_find(softnic, pipeline_name);
1978         if (p == NULL ||
1979                 table_id >= p->n_tables)
1980                 return -1;
1981
1982         if (!pipeline_is_running(p)) {
1983                 struct rte_table_action *a = p->table[table_id].a;
1984
1985                 status = rte_table_action_dscp_table_update(a,
1986                                 dscp_mask,
1987                                 dscp_table);
1988
1989                 /* Update table dscp table */
1990                 if (!status)
1991                         memcpy(&p->table[table_id].dscp_table, dscp_table,
1992                                 sizeof(p->table[table_id].dscp_table));
1993
1994                 return status;
1995         }
1996
1997         /* Allocate request */
1998         req = pipeline_msg_alloc();
1999         if (req == NULL)
2000                 return -1;
2001
2002         /* Write request */
2003         req->type = PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE;
2004         req->id = table_id;
2005         req->table_dscp_table_update.dscp_mask = dscp_mask;
2006         memcpy(&req->table_dscp_table_update.dscp_table,
2007                 dscp_table, sizeof(*dscp_table));
2008
2009         /* Send request and wait for response */
2010         rsp = pipeline_msg_send_recv(p, req);
2011         if (rsp == NULL)
2012                 return -1;
2013
2014         /* Read response */
2015         status = rsp->status;
2016
2017         /* Update table dscp table */
2018         if (!status)
2019                 memcpy(&p->table[table_id].dscp_table, dscp_table,
2020                         sizeof(p->table[table_id].dscp_table));
2021
2022         /* Free response */
2023         pipeline_msg_free(rsp);
2024
2025         return status;
2026 }
2027
2028 int
2029 softnic_pipeline_table_rule_ttl_read(struct pmd_internals *softnic,
2030         const char *pipeline_name,
2031         uint32_t table_id,
2032         void *data,
2033         struct rte_table_action_ttl_counters *stats,
2034         int clear)
2035 {
2036         struct pipeline *p;
2037         struct pipeline_msg_req *req;
2038         struct pipeline_msg_rsp *rsp;
2039         int status;
2040
2041         /* Check input params */
2042         if (pipeline_name == NULL ||
2043                 data == NULL ||
2044                 stats == NULL)
2045                 return -1;
2046
2047         p = softnic_pipeline_find(softnic, pipeline_name);
2048         if (p == NULL ||
2049                 table_id >= p->n_tables)
2050                 return -1;
2051
2052         if (!pipeline_is_running(p)) {
2053                 struct rte_table_action *a = p->table[table_id].a;
2054
2055                 status = rte_table_action_ttl_read(a,
2056                                 data,
2057                                 stats,
2058                                 clear);
2059
2060                 return status;
2061         }
2062
2063         /* Allocate request */
2064         req = pipeline_msg_alloc();
2065         if (req == NULL)
2066                 return -1;
2067
2068         /* Write request */
2069         req->type = PIPELINE_REQ_TABLE_RULE_TTL_READ;
2070         req->id = table_id;
2071         req->table_rule_ttl_read.data = data;
2072         req->table_rule_ttl_read.clear = clear;
2073
2074         /* Send request and wait for response */
2075         rsp = pipeline_msg_send_recv(p, req);
2076         if (rsp == NULL)
2077                 return -1;
2078
2079         /* Read response */
2080         status = rsp->status;
2081         if (status)
2082                 memcpy(stats, &rsp->table_rule_ttl_read.stats, sizeof(*stats));
2083
2084         /* Free response */
2085         pipeline_msg_free(rsp);
2086
2087         return status;
2088 }
2089
2090 /**
2091  * Data plane threads: message handling
2092  */
2093 static inline struct pipeline_msg_req *
2094 pipeline_msg_recv(struct rte_ring *msgq_req)
2095 {
2096         struct pipeline_msg_req *req;
2097
2098         int status = rte_ring_sc_dequeue(msgq_req, (void **)&req);
2099
2100         if (status != 0)
2101                 return NULL;
2102
2103         return req;
2104 }
2105
2106 static inline void
2107 pipeline_msg_send(struct rte_ring *msgq_rsp,
2108         struct pipeline_msg_rsp *rsp)
2109 {
2110         int status;
2111
2112         do {
2113                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
2114         } while (status == -ENOBUFS);
2115 }
2116
2117 static struct pipeline_msg_rsp *
2118 pipeline_msg_handle_port_in_stats_read(struct pipeline_data *p,
2119         struct pipeline_msg_req *req)
2120 {
2121         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2122         uint32_t port_id = req->id;
2123         int clear = req->port_in_stats_read.clear;
2124
2125         rsp->status = rte_pipeline_port_in_stats_read(p->p,
2126                 port_id,
2127                 &rsp->port_in_stats_read.stats,
2128                 clear);
2129
2130         return rsp;
2131 }
2132
2133 static struct pipeline_msg_rsp *
2134 pipeline_msg_handle_port_in_enable(struct pipeline_data *p,
2135         struct pipeline_msg_req *req)
2136 {
2137         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2138         uint32_t port_id = req->id;
2139
2140         rsp->status = rte_pipeline_port_in_enable(p->p,
2141                 port_id);
2142
2143         return rsp;
2144 }
2145
2146 static struct pipeline_msg_rsp *
2147 pipeline_msg_handle_port_in_disable(struct pipeline_data *p,
2148         struct pipeline_msg_req *req)
2149 {
2150         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2151         uint32_t port_id = req->id;
2152
2153         rsp->status = rte_pipeline_port_in_disable(p->p,
2154                 port_id);
2155
2156         return rsp;
2157 }
2158
2159 static struct pipeline_msg_rsp *
2160 pipeline_msg_handle_port_out_stats_read(struct pipeline_data *p,
2161         struct pipeline_msg_req *req)
2162 {
2163         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2164         uint32_t port_id = req->id;
2165         int clear = req->port_out_stats_read.clear;
2166
2167         rsp->status = rte_pipeline_port_out_stats_read(p->p,
2168                 port_id,
2169                 &rsp->port_out_stats_read.stats,
2170                 clear);
2171
2172         return rsp;
2173 }
2174
2175 static struct pipeline_msg_rsp *
2176 pipeline_msg_handle_table_stats_read(struct pipeline_data *p,
2177         struct pipeline_msg_req *req)
2178 {
2179         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2180         uint32_t port_id = req->id;
2181         int clear = req->table_stats_read.clear;
2182
2183         rsp->status = rte_pipeline_table_stats_read(p->p,
2184                 port_id,
2185                 &rsp->table_stats_read.stats,
2186                 clear);
2187
2188         return rsp;
2189 }
2190
2191 static int
2192 match_convert_ipv6_depth(uint32_t depth, uint32_t *depth32)
2193 {
2194         if (depth > 128)
2195                 return -1;
2196
2197         switch (depth / 32) {
2198         case 0:
2199                 depth32[0] = depth;
2200                 depth32[1] = 0;
2201                 depth32[2] = 0;
2202                 depth32[3] = 0;
2203                 return 0;
2204
2205         case 1:
2206                 depth32[0] = 32;
2207                 depth32[1] = depth - 32;
2208                 depth32[2] = 0;
2209                 depth32[3] = 0;
2210                 return 0;
2211
2212         case 2:
2213                 depth32[0] = 32;
2214                 depth32[1] = 32;
2215                 depth32[2] = depth - 64;
2216                 depth32[3] = 0;
2217                 return 0;
2218
2219         case 3:
2220                 depth32[0] = 32;
2221                 depth32[1] = 32;
2222                 depth32[2] = 32;
2223                 depth32[3] = depth - 96;
2224                 return 0;
2225
2226         case 4:
2227                 depth32[0] = 32;
2228                 depth32[1] = 32;
2229                 depth32[2] = 32;
2230                 depth32[3] = 32;
2231                 return 0;
2232
2233         default:
2234                 return -1;
2235         }
2236 }
2237
2238 static int
2239 match_convert(struct softnic_table_rule_match *mh,
2240         union table_rule_match_low_level *ml,
2241         int add)
2242 {
2243         memset(ml, 0, sizeof(*ml));
2244
2245         switch (mh->match_type) {
2246         case TABLE_ACL:
2247                 if (mh->match.acl.ip_version)
2248                         if (add) {
2249                                 ml->acl_add.field_value[0].value.u8 =
2250                                         mh->match.acl.proto;
2251                                 ml->acl_add.field_value[0].mask_range.u8 =
2252                                         mh->match.acl.proto_mask;
2253
2254                                 ml->acl_add.field_value[1].value.u32 =
2255                                         mh->match.acl.ipv4.sa;
2256                                 ml->acl_add.field_value[1].mask_range.u32 =
2257                                         mh->match.acl.sa_depth;
2258
2259                                 ml->acl_add.field_value[2].value.u32 =
2260                                         mh->match.acl.ipv4.da;
2261                                 ml->acl_add.field_value[2].mask_range.u32 =
2262                                         mh->match.acl.da_depth;
2263
2264                                 ml->acl_add.field_value[3].value.u16 =
2265                                         mh->match.acl.sp0;
2266                                 ml->acl_add.field_value[3].mask_range.u16 =
2267                                         mh->match.acl.sp1;
2268
2269                                 ml->acl_add.field_value[4].value.u16 =
2270                                         mh->match.acl.dp0;
2271                                 ml->acl_add.field_value[4].mask_range.u16 =
2272                                         mh->match.acl.dp1;
2273
2274                                 ml->acl_add.priority =
2275                                         (int32_t)mh->match.acl.priority;
2276                         } else {
2277                                 ml->acl_delete.field_value[0].value.u8 =
2278                                         mh->match.acl.proto;
2279                                 ml->acl_delete.field_value[0].mask_range.u8 =
2280                                         mh->match.acl.proto_mask;
2281
2282                                 ml->acl_delete.field_value[1].value.u32 =
2283                                         mh->match.acl.ipv4.sa;
2284                                 ml->acl_delete.field_value[1].mask_range.u32 =
2285                                         mh->match.acl.sa_depth;
2286
2287                                 ml->acl_delete.field_value[2].value.u32 =
2288                                         mh->match.acl.ipv4.da;
2289                                 ml->acl_delete.field_value[2].mask_range.u32 =
2290                                         mh->match.acl.da_depth;
2291
2292                                 ml->acl_delete.field_value[3].value.u16 =
2293                                         mh->match.acl.sp0;
2294                                 ml->acl_delete.field_value[3].mask_range.u16 =
2295                                         mh->match.acl.sp1;
2296
2297                                 ml->acl_delete.field_value[4].value.u16 =
2298                                         mh->match.acl.dp0;
2299                                 ml->acl_delete.field_value[4].mask_range.u16 =
2300                                         mh->match.acl.dp1;
2301                         }
2302                 else
2303                         if (add) {
2304                                 uint32_t *sa32 =
2305                                         (uint32_t *)mh->match.acl.ipv6.sa;
2306                                 uint32_t *da32 =
2307                                         (uint32_t *)mh->match.acl.ipv6.da;
2308                                 uint32_t sa32_depth[4], da32_depth[4];
2309                                 int status;
2310
2311                                 status = match_convert_ipv6_depth(mh->match.acl.sa_depth,
2312                                         sa32_depth);
2313                                 if (status)
2314                                         return status;
2315
2316                                 status = match_convert_ipv6_depth(
2317                                         mh->match.acl.da_depth,
2318                                         da32_depth);
2319                                 if (status)
2320                                         return status;
2321
2322                                 ml->acl_add.field_value[0].value.u8 =
2323                                         mh->match.acl.proto;
2324                                 ml->acl_add.field_value[0].mask_range.u8 =
2325                                         mh->match.acl.proto_mask;
2326
2327                                 ml->acl_add.field_value[1].value.u32 =
2328                                         rte_be_to_cpu_32(sa32[0]);
2329                                 ml->acl_add.field_value[1].mask_range.u32 =
2330                                         sa32_depth[0];
2331                                 ml->acl_add.field_value[2].value.u32 =
2332                                         rte_be_to_cpu_32(sa32[1]);
2333                                 ml->acl_add.field_value[2].mask_range.u32 =
2334                                         sa32_depth[1];
2335                                 ml->acl_add.field_value[3].value.u32 =
2336                                         rte_be_to_cpu_32(sa32[2]);
2337                                 ml->acl_add.field_value[3].mask_range.u32 =
2338                                         sa32_depth[2];
2339                                 ml->acl_add.field_value[4].value.u32 =
2340                                         rte_be_to_cpu_32(sa32[3]);
2341                                 ml->acl_add.field_value[4].mask_range.u32 =
2342                                         sa32_depth[3];
2343
2344                                 ml->acl_add.field_value[5].value.u32 =
2345                                         rte_be_to_cpu_32(da32[0]);
2346                                 ml->acl_add.field_value[5].mask_range.u32 =
2347                                         da32_depth[0];
2348                                 ml->acl_add.field_value[6].value.u32 =
2349                                         rte_be_to_cpu_32(da32[1]);
2350                                 ml->acl_add.field_value[6].mask_range.u32 =
2351                                         da32_depth[1];
2352                                 ml->acl_add.field_value[7].value.u32 =
2353                                         rte_be_to_cpu_32(da32[2]);
2354                                 ml->acl_add.field_value[7].mask_range.u32 =
2355                                         da32_depth[2];
2356                                 ml->acl_add.field_value[8].value.u32 =
2357                                         rte_be_to_cpu_32(da32[3]);
2358                                 ml->acl_add.field_value[8].mask_range.u32 =
2359                                         da32_depth[3];
2360
2361                                 ml->acl_add.field_value[9].value.u16 =
2362                                         mh->match.acl.sp0;
2363                                 ml->acl_add.field_value[9].mask_range.u16 =
2364                                         mh->match.acl.sp1;
2365
2366                                 ml->acl_add.field_value[10].value.u16 =
2367                                         mh->match.acl.dp0;
2368                                 ml->acl_add.field_value[10].mask_range.u16 =
2369                                         mh->match.acl.dp1;
2370
2371                                 ml->acl_add.priority =
2372                                         (int32_t)mh->match.acl.priority;
2373                         } else {
2374                                 uint32_t *sa32 =
2375                                         (uint32_t *)mh->match.acl.ipv6.sa;
2376                                 uint32_t *da32 =
2377                                         (uint32_t *)mh->match.acl.ipv6.da;
2378                                 uint32_t sa32_depth[4], da32_depth[4];
2379                                 int status;
2380
2381                                 status = match_convert_ipv6_depth(mh->match.acl.sa_depth,
2382                                         sa32_depth);
2383                                 if (status)
2384                                         return status;
2385
2386                                 status = match_convert_ipv6_depth(mh->match.acl.da_depth,
2387                                         da32_depth);
2388                                 if (status)
2389                                         return status;
2390
2391                                 ml->acl_delete.field_value[0].value.u8 =
2392                                         mh->match.acl.proto;
2393                                 ml->acl_delete.field_value[0].mask_range.u8 =
2394                                         mh->match.acl.proto_mask;
2395
2396                                 ml->acl_delete.field_value[1].value.u32 =
2397                                         rte_be_to_cpu_32(sa32[0]);
2398                                 ml->acl_delete.field_value[1].mask_range.u32 =
2399                                         sa32_depth[0];
2400                                 ml->acl_delete.field_value[2].value.u32 =
2401                                         rte_be_to_cpu_32(sa32[1]);
2402                                 ml->acl_delete.field_value[2].mask_range.u32 =
2403                                         sa32_depth[1];
2404                                 ml->acl_delete.field_value[3].value.u32 =
2405                                         rte_be_to_cpu_32(sa32[2]);
2406                                 ml->acl_delete.field_value[3].mask_range.u32 =
2407                                         sa32_depth[2];
2408                                 ml->acl_delete.field_value[4].value.u32 =
2409                                         rte_be_to_cpu_32(sa32[3]);
2410                                 ml->acl_delete.field_value[4].mask_range.u32 =
2411                                         sa32_depth[3];
2412
2413                                 ml->acl_delete.field_value[5].value.u32 =
2414                                         rte_be_to_cpu_32(da32[0]);
2415                                 ml->acl_delete.field_value[5].mask_range.u32 =
2416                                         da32_depth[0];
2417                                 ml->acl_delete.field_value[6].value.u32 =
2418                                         rte_be_to_cpu_32(da32[1]);
2419                                 ml->acl_delete.field_value[6].mask_range.u32 =
2420                                         da32_depth[1];
2421                                 ml->acl_delete.field_value[7].value.u32 =
2422                                         rte_be_to_cpu_32(da32[2]);
2423                                 ml->acl_delete.field_value[7].mask_range.u32 =
2424                                         da32_depth[2];
2425                                 ml->acl_delete.field_value[8].value.u32 =
2426                                         rte_be_to_cpu_32(da32[3]);
2427                                 ml->acl_delete.field_value[8].mask_range.u32 =
2428                                         da32_depth[3];
2429
2430                                 ml->acl_delete.field_value[9].value.u16 =
2431                                         mh->match.acl.sp0;
2432                                 ml->acl_delete.field_value[9].mask_range.u16 =
2433                                         mh->match.acl.sp1;
2434
2435                                 ml->acl_delete.field_value[10].value.u16 =
2436                                         mh->match.acl.dp0;
2437                                 ml->acl_delete.field_value[10].mask_range.u16 =
2438                                         mh->match.acl.dp1;
2439                         }
2440                 return 0;
2441
2442         case TABLE_ARRAY:
2443                 ml->array.pos = mh->match.array.pos;
2444                 return 0;
2445
2446         case TABLE_HASH:
2447                 memcpy(ml->hash, mh->match.hash.key, sizeof(ml->hash));
2448                 return 0;
2449
2450         case TABLE_LPM:
2451                 if (mh->match.lpm.ip_version) {
2452                         ml->lpm_ipv4.ip = mh->match.lpm.ipv4;
2453                         ml->lpm_ipv4.depth = mh->match.lpm.depth;
2454                 } else {
2455                         memcpy(ml->lpm_ipv6.ip,
2456                                 mh->match.lpm.ipv6, sizeof(ml->lpm_ipv6.ip));
2457                         ml->lpm_ipv6.depth = mh->match.lpm.depth;
2458                 }
2459
2460                 return 0;
2461
2462         default:
2463                 return -1;
2464         }
2465 }
2466
2467 static int
2468 action_convert(struct rte_table_action *a,
2469         struct softnic_table_rule_action *action,
2470         struct rte_pipeline_table_entry *data)
2471 {
2472         int status;
2473
2474         /* Apply actions */
2475         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
2476                 status = rte_table_action_apply(a,
2477                         data,
2478                         RTE_TABLE_ACTION_FWD,
2479                         &action->fwd);
2480
2481                 if (status)
2482                         return status;
2483         }
2484
2485         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) {
2486                 status = rte_table_action_apply(a,
2487                         data,
2488                         RTE_TABLE_ACTION_LB,
2489                         &action->lb);
2490
2491                 if (status)
2492                         return status;
2493         }
2494
2495         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
2496                 status = rte_table_action_apply(a,
2497                         data,
2498                         RTE_TABLE_ACTION_MTR,
2499                         &action->mtr);
2500
2501                 if (status)
2502                         return status;
2503         }
2504
2505         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
2506                 status = rte_table_action_apply(a,
2507                         data,
2508                         RTE_TABLE_ACTION_TM,
2509                         &action->tm);
2510
2511                 if (status)
2512                         return status;
2513         }
2514
2515         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
2516                 status = rte_table_action_apply(a,
2517                         data,
2518                         RTE_TABLE_ACTION_ENCAP,
2519                         &action->encap);
2520
2521                 if (status)
2522                         return status;
2523         }
2524
2525         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
2526                 status = rte_table_action_apply(a,
2527                         data,
2528                         RTE_TABLE_ACTION_NAT,
2529                         &action->nat);
2530
2531                 if (status)
2532                         return status;
2533         }
2534
2535         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
2536                 status = rte_table_action_apply(a,
2537                         data,
2538                         RTE_TABLE_ACTION_TTL,
2539                         &action->ttl);
2540
2541                 if (status)
2542                         return status;
2543         }
2544
2545         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
2546                 status = rte_table_action_apply(a,
2547                         data,
2548                         RTE_TABLE_ACTION_STATS,
2549                         &action->stats);
2550
2551                 if (status)
2552                         return status;
2553         }
2554
2555         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
2556                 status = rte_table_action_apply(a,
2557                         data,
2558                         RTE_TABLE_ACTION_TIME,
2559                         &action->time);
2560
2561                 if (status)
2562                         return status;
2563         }
2564
2565         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) {
2566                 status = rte_table_action_apply(a,
2567                         data,
2568                         RTE_TABLE_ACTION_TAG,
2569                         &action->tag);
2570
2571                 if (status)
2572                         return status;
2573         }
2574
2575         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) {
2576                 status = rte_table_action_apply(a,
2577                         data,
2578                         RTE_TABLE_ACTION_DECAP,
2579                         &action->decap);
2580
2581                 if (status)
2582                         return status;
2583         }
2584
2585         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) {
2586                 status = rte_table_action_apply(a,
2587                         data,
2588                         RTE_TABLE_ACTION_SYM_CRYPTO,
2589                         &action->sym_crypto);
2590
2591                 if (status)
2592                         return status;
2593         }
2594
2595         return 0;
2596 }
2597
2598 static struct pipeline_msg_rsp *
2599 pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
2600         struct pipeline_msg_req *req)
2601 {
2602         union table_rule_match_low_level match_ll;
2603         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2604         struct softnic_table_rule_match *match = &req->table_rule_add.match;
2605         struct softnic_table_rule_action *action = &req->table_rule_add.action;
2606         struct rte_pipeline_table_entry *data_in, *data_out;
2607         uint32_t table_id = req->id;
2608         int key_found, status;
2609         struct rte_table_action *a = p->table_data[table_id].a;
2610
2611         /* Apply actions */
2612         memset(p->buffer, 0, sizeof(p->buffer));
2613         data_in = (struct rte_pipeline_table_entry *)p->buffer;
2614
2615         status = match_convert(match, &match_ll, 1);
2616         if (status) {
2617                 rsp->status = -1;
2618                 return rsp;
2619         }
2620
2621         status = action_convert(a, action, data_in);
2622         if (status) {
2623                 rsp->status = -1;
2624                 return rsp;
2625         }
2626
2627         status = rte_pipeline_table_entry_add(p->p,
2628                 table_id,
2629                 &match_ll,
2630                 data_in,
2631                 &key_found,
2632                 &data_out);
2633         if (status) {
2634                 rsp->status = -1;
2635                 return rsp;
2636         }
2637
2638         /* Write response */
2639         rsp->status = 0;
2640         rsp->table_rule_add.data = data_out;
2641
2642         return rsp;
2643 }
2644
2645 static struct pipeline_msg_rsp *
2646 pipeline_msg_handle_table_rule_add_default(struct pipeline_data *p,
2647         struct pipeline_msg_req *req)
2648 {
2649         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2650         struct softnic_table_rule_action *action = &req->table_rule_add_default.action;
2651         struct rte_pipeline_table_entry *data_in, *data_out;
2652         uint32_t table_id = req->id;
2653         int status;
2654
2655         /* Apply actions */
2656         memset(p->buffer, 0, sizeof(p->buffer));
2657         data_in = (struct rte_pipeline_table_entry *)p->buffer;
2658
2659         data_in->action = action->fwd.action;
2660         if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
2661                 data_in->port_id = action->fwd.id;
2662         if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
2663                 data_in->table_id = action->fwd.id;
2664
2665         /* Add default rule to table */
2666         status = rte_pipeline_table_default_entry_add(p->p,
2667                 table_id,
2668                 data_in,
2669                 &data_out);
2670         if (status) {
2671                 rsp->status = -1;
2672                 return rsp;
2673         }
2674
2675         /* Write response */
2676         rsp->status = 0;
2677         rsp->table_rule_add_default.data = data_out;
2678
2679         return rsp;
2680 }
2681
2682 static struct pipeline_msg_rsp *
2683 pipeline_msg_handle_table_rule_add_bulk(struct pipeline_data *p,
2684         struct pipeline_msg_req *req)
2685 {
2686         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2687
2688         uint32_t table_id = req->id;
2689         struct softnic_table_rule_match *match = req->table_rule_add_bulk.match;
2690         struct softnic_table_rule_action *action = req->table_rule_add_bulk.action;
2691         struct rte_pipeline_table_entry **data =
2692                 (struct rte_pipeline_table_entry **)req->table_rule_add_bulk.data;
2693         uint32_t n_rules = req->table_rule_add_bulk.n_rules;
2694         uint32_t bulk = req->table_rule_add_bulk.bulk;
2695
2696         struct rte_table_action *a = p->table_data[table_id].a;
2697         union table_rule_match_low_level *match_ll;
2698         uint8_t *action_ll;
2699         void **match_ll_ptr;
2700         struct rte_pipeline_table_entry **action_ll_ptr;
2701         int *found, status;
2702         uint32_t i;
2703
2704         /* Memory allocation */
2705         match_ll = calloc(n_rules, sizeof(union table_rule_match_low_level));
2706         action_ll = calloc(n_rules, TABLE_RULE_ACTION_SIZE_MAX);
2707         match_ll_ptr = calloc(n_rules, sizeof(void *));
2708         action_ll_ptr =
2709                 calloc(n_rules, sizeof(struct rte_pipeline_table_entry *));
2710         found = calloc(n_rules, sizeof(int));
2711
2712         if (match_ll == NULL ||
2713                 action_ll == NULL ||
2714                 match_ll_ptr == NULL ||
2715                 action_ll_ptr == NULL ||
2716                 found == NULL)
2717                 goto fail;
2718
2719         for (i = 0; i < n_rules; i++) {
2720                 match_ll_ptr[i] = (void *)&match_ll[i];
2721                 action_ll_ptr[i] =
2722                         (struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
2723         }
2724
2725         /* Rule match conversion */
2726         for (i = 0; i < n_rules; i++) {
2727                 status = match_convert(&match[i], match_ll_ptr[i], 1);
2728                 if (status)
2729                         goto fail;
2730         }
2731
2732         /* Rule action conversion */
2733         for (i = 0; i < n_rules; i++) {
2734                 status = action_convert(a, &action[i], action_ll_ptr[i]);
2735                 if (status)
2736                         goto fail;
2737         }
2738
2739         /* Add rule (match, action) to table */
2740         if (bulk) {
2741                 status = rte_pipeline_table_entry_add_bulk(p->p,
2742                         table_id,
2743                         match_ll_ptr,
2744                         action_ll_ptr,
2745                         n_rules,
2746                         found,
2747                         data);
2748                 if (status)
2749                         n_rules = 0;
2750         } else {
2751                 for (i = 0; i < n_rules; i++) {
2752                         status = rte_pipeline_table_entry_add(p->p,
2753                                 table_id,
2754                                 match_ll_ptr[i],
2755                                 action_ll_ptr[i],
2756                                 &found[i],
2757                                 &data[i]);
2758                         if (status) {
2759                                 n_rules = i;
2760                                 break;
2761                         }
2762                 }
2763         }
2764
2765         /* Write response */
2766         rsp->status = 0;
2767         rsp->table_rule_add_bulk.n_rules = n_rules;
2768
2769         /* Free */
2770         free(found);
2771         free(action_ll_ptr);
2772         free(match_ll_ptr);
2773         free(action_ll);
2774         free(match_ll);
2775
2776         return rsp;
2777
2778 fail:
2779         free(found);
2780         free(action_ll_ptr);
2781         free(match_ll_ptr);
2782         free(action_ll);
2783         free(match_ll);
2784
2785         rsp->status = -1;
2786         rsp->table_rule_add_bulk.n_rules = 0;
2787         return rsp;
2788 }
2789
2790 static struct pipeline_msg_rsp *
2791 pipeline_msg_handle_table_rule_delete(struct pipeline_data *p,
2792         struct pipeline_msg_req *req)
2793 {
2794         union table_rule_match_low_level match_ll;
2795         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2796         struct softnic_table_rule_match *match = &req->table_rule_delete.match;
2797         uint32_t table_id = req->id;
2798         int key_found, status;
2799
2800         status = match_convert(match, &match_ll, 0);
2801         if (status) {
2802                 rsp->status = -1;
2803                 return rsp;
2804         }
2805
2806         rsp->status = rte_pipeline_table_entry_delete(p->p,
2807                 table_id,
2808                 &match_ll,
2809                 &key_found,
2810                 NULL);
2811
2812         return rsp;
2813 }
2814
2815 static struct pipeline_msg_rsp *
2816 pipeline_msg_handle_table_rule_delete_default(struct pipeline_data *p,
2817         struct pipeline_msg_req *req)
2818 {
2819         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2820         uint32_t table_id = req->id;
2821
2822         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
2823                 table_id,
2824                 NULL);
2825
2826         return rsp;
2827 }
2828
2829 static struct pipeline_msg_rsp *
2830 pipeline_msg_handle_table_rule_stats_read(struct pipeline_data *p,
2831         struct pipeline_msg_req *req)
2832 {
2833         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2834         uint32_t table_id = req->id;
2835         void *data = req->table_rule_stats_read.data;
2836         int clear = req->table_rule_stats_read.clear;
2837         struct rte_table_action *a = p->table_data[table_id].a;
2838
2839         rsp->status = rte_table_action_stats_read(a,
2840                 data,
2841                 &rsp->table_rule_stats_read.stats,
2842                 clear);
2843
2844         return rsp;
2845 }
2846
2847 static struct pipeline_msg_rsp *
2848 pipeline_msg_handle_table_mtr_profile_add(struct pipeline_data *p,
2849         struct pipeline_msg_req *req)
2850 {
2851         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2852         uint32_t table_id = req->id;
2853         uint32_t meter_profile_id = req->table_mtr_profile_add.meter_profile_id;
2854         struct rte_table_action_meter_profile *profile =
2855                 &req->table_mtr_profile_add.profile;
2856         struct rte_table_action *a = p->table_data[table_id].a;
2857
2858         rsp->status = rte_table_action_meter_profile_add(a,
2859                 meter_profile_id,
2860                 profile);
2861
2862         return rsp;
2863 }
2864
2865 static struct pipeline_msg_rsp *
2866 pipeline_msg_handle_table_mtr_profile_delete(struct pipeline_data *p,
2867         struct pipeline_msg_req *req)
2868 {
2869         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2870         uint32_t table_id = req->id;
2871         uint32_t meter_profile_id =
2872                 req->table_mtr_profile_delete.meter_profile_id;
2873         struct rte_table_action *a = p->table_data[table_id].a;
2874
2875         rsp->status = rte_table_action_meter_profile_delete(a,
2876                 meter_profile_id);
2877
2878         return rsp;
2879 }
2880
2881 static struct pipeline_msg_rsp *
2882 pipeline_msg_handle_table_rule_mtr_read(struct pipeline_data *p,
2883         struct pipeline_msg_req *req)
2884 {
2885         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2886         uint32_t table_id = req->id;
2887         void *data = req->table_rule_mtr_read.data;
2888         uint32_t tc_mask = req->table_rule_mtr_read.tc_mask;
2889         int clear = req->table_rule_mtr_read.clear;
2890         struct rte_table_action *a = p->table_data[table_id].a;
2891
2892         rsp->status = rte_table_action_meter_read(a,
2893                 data,
2894                 tc_mask,
2895                 &rsp->table_rule_mtr_read.stats,
2896                 clear);
2897
2898         return rsp;
2899 }
2900
2901 static struct pipeline_msg_rsp *
2902 pipeline_msg_handle_table_dscp_table_update(struct pipeline_data *p,
2903         struct pipeline_msg_req *req)
2904 {
2905         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2906         uint32_t table_id = req->id;
2907         uint64_t dscp_mask = req->table_dscp_table_update.dscp_mask;
2908         struct rte_table_action_dscp_table *dscp_table =
2909                 &req->table_dscp_table_update.dscp_table;
2910         struct rte_table_action *a = p->table_data[table_id].a;
2911
2912         rsp->status = rte_table_action_dscp_table_update(a,
2913                 dscp_mask,
2914                 dscp_table);
2915
2916         return rsp;
2917 }
2918
2919 static struct pipeline_msg_rsp *
2920 pipeline_msg_handle_table_rule_ttl_read(struct pipeline_data *p,
2921         struct pipeline_msg_req *req)
2922 {
2923         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
2924         uint32_t table_id = req->id;
2925         void *data = req->table_rule_ttl_read.data;
2926         int clear = req->table_rule_ttl_read.clear;
2927         struct rte_table_action *a = p->table_data[table_id].a;
2928
2929         rsp->status = rte_table_action_ttl_read(a,
2930                 data,
2931                 &rsp->table_rule_ttl_read.stats,
2932                 clear);
2933
2934         return rsp;
2935 }
2936
2937 static void
2938 pipeline_msg_handle(struct pipeline_data *p)
2939 {
2940         for ( ; ; ) {
2941                 struct pipeline_msg_req *req;
2942                 struct pipeline_msg_rsp *rsp;
2943
2944                 req = pipeline_msg_recv(p->msgq_req);
2945                 if (req == NULL)
2946                         break;
2947
2948                 switch (req->type) {
2949                 case PIPELINE_REQ_PORT_IN_STATS_READ:
2950                         rsp = pipeline_msg_handle_port_in_stats_read(p, req);
2951                         break;
2952
2953                 case PIPELINE_REQ_PORT_IN_ENABLE:
2954                         rsp = pipeline_msg_handle_port_in_enable(p, req);
2955                         break;
2956
2957                 case PIPELINE_REQ_PORT_IN_DISABLE:
2958                         rsp = pipeline_msg_handle_port_in_disable(p, req);
2959                         break;
2960
2961                 case PIPELINE_REQ_PORT_OUT_STATS_READ:
2962                         rsp = pipeline_msg_handle_port_out_stats_read(p, req);
2963                         break;
2964
2965                 case PIPELINE_REQ_TABLE_STATS_READ:
2966                         rsp = pipeline_msg_handle_table_stats_read(p, req);
2967                         break;
2968
2969                 case PIPELINE_REQ_TABLE_RULE_ADD:
2970                         rsp = pipeline_msg_handle_table_rule_add(p, req);
2971                         break;
2972
2973                 case PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT:
2974                         rsp = pipeline_msg_handle_table_rule_add_default(p,     req);
2975                         break;
2976
2977                 case PIPELINE_REQ_TABLE_RULE_ADD_BULK:
2978                         rsp = pipeline_msg_handle_table_rule_add_bulk(p, req);
2979                         break;
2980
2981                 case PIPELINE_REQ_TABLE_RULE_DELETE:
2982                         rsp = pipeline_msg_handle_table_rule_delete(p, req);
2983                         break;
2984
2985                 case PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT:
2986                         rsp = pipeline_msg_handle_table_rule_delete_default(p, req);
2987                         break;
2988
2989                 case PIPELINE_REQ_TABLE_RULE_STATS_READ:
2990                         rsp = pipeline_msg_handle_table_rule_stats_read(p, req);
2991                         break;
2992
2993                 case PIPELINE_REQ_TABLE_MTR_PROFILE_ADD:
2994                         rsp = pipeline_msg_handle_table_mtr_profile_add(p, req);
2995                         break;
2996
2997                 case PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE:
2998                         rsp = pipeline_msg_handle_table_mtr_profile_delete(p, req);
2999                         break;
3000
3001                 case PIPELINE_REQ_TABLE_RULE_MTR_READ:
3002                         rsp = pipeline_msg_handle_table_rule_mtr_read(p, req);
3003                         break;
3004
3005                 case PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE:
3006                         rsp = pipeline_msg_handle_table_dscp_table_update(p, req);
3007                         break;
3008
3009                 case PIPELINE_REQ_TABLE_RULE_TTL_READ:
3010                         rsp = pipeline_msg_handle_table_rule_ttl_read(p, req);
3011                         break;
3012
3013                 default:
3014                         rsp = (struct pipeline_msg_rsp *)req;
3015                         rsp->status = -1;
3016                 }
3017
3018                 pipeline_msg_send(p->msgq_rsp, rsp);
3019         }
3020 }
3021
3022 /**
3023  * Data plane threads: main
3024  */
3025 static int32_t
3026 rte_pmd_softnic_run_internal(void *arg)
3027 {
3028         struct rte_eth_dev *dev = arg;
3029         struct pmd_internals *softnic;
3030         struct softnic_thread_data *t;
3031         uint32_t thread_id, j;
3032
3033         softnic = dev->data->dev_private;
3034         thread_id = rte_lcore_id();
3035         t = &softnic->thread_data[thread_id];
3036         t->iter++;
3037
3038         /* Data Plane */
3039         for (j = 0; j < t->n_pipelines; j++)
3040                 rte_pipeline_run(t->p[j]);
3041
3042         /* Control Plane */
3043         if ((t->iter & 0xFLLU) == 0) {
3044                 uint64_t time = rte_get_tsc_cycles();
3045                 uint64_t time_next_min = UINT64_MAX;
3046
3047                 if (time < t->time_next_min)
3048                         return 0;
3049
3050                 /* Pipeline message queues */
3051                 for (j = 0; j < t->n_pipelines; j++) {
3052                         struct pipeline_data *p =
3053                                 &t->pipeline_data[j];
3054                         uint64_t time_next = p->time_next;
3055
3056                         if (time_next <= time) {
3057                                 pipeline_msg_handle(p);
3058                                 rte_pipeline_flush(p->p);
3059                                 time_next = time + p->timer_period;
3060                                 p->time_next = time_next;
3061                         }
3062
3063                         if (time_next < time_next_min)
3064                                 time_next_min = time_next;
3065                 }
3066
3067                 /* Thread message queues */
3068                 {
3069                         uint64_t time_next = t->time_next;
3070
3071                         if (time_next <= time) {
3072                                 thread_msg_handle(t);
3073                                 time_next = time + t->timer_period;
3074                                 t->time_next = time_next;
3075                         }
3076
3077                         if (time_next < time_next_min)
3078                                 time_next_min = time_next;
3079                 }
3080
3081                 t->time_next_min = time_next_min;
3082         }
3083
3084         return 0;
3085 }
3086
3087 int
3088 rte_pmd_softnic_run(uint16_t port_id)
3089 {
3090         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3091
3092 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
3093         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
3094 #endif
3095
3096         return (int)rte_pmd_softnic_run_internal(dev);
3097 }