lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9
  10 #include <rte_common.h>
  11
  12 #include "rte_swx_pipeline.h"
  13 #include "rte_swx_ctl.h"
  14
  15 #define CHECK(condition, err_code)                                             \
  16 do {                                                                           \
  17         if (!(condition))                                                      \
  18                 return -(err_code);                                            \
  19 } while (0)
  20
  21 #define CHECK_NAME(name, err_code)                                             \
  22         CHECK((name) && (name)[0], err_code)
  23
  24 /*
  25  * Struct.
  26  */
  27 struct field {
  28         char name[RTE_SWX_NAME_SIZE];
  29         uint32_t n_bits;
  30         uint32_t offset;
  31 };
  32
  33 struct struct_type {
  34         TAILQ_ENTRY(struct_type) node;
  35         char name[RTE_SWX_NAME_SIZE];
  36         struct field *fields;
  37         uint32_t n_fields;
  38         uint32_t n_bits;
  39 };
  40
  41 TAILQ_HEAD(struct_type_tailq, struct_type);
  42
  43 /*
  44  * Input port.
  45  */
  46 struct port_in_type {
  47         TAILQ_ENTRY(port_in_type) node;
  48         char name[RTE_SWX_NAME_SIZE];
  49         struct rte_swx_port_in_ops ops;
  50 };
  51
  52 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  53
  54 struct port_in {
  55         TAILQ_ENTRY(port_in) node;
  56         struct port_in_type *type;
  57         void *obj;
  58         uint32_t id;
  59 };
  60
  61 TAILQ_HEAD(port_in_tailq, port_in);
  62
  63 struct port_in_runtime {
  64         rte_swx_port_in_pkt_rx_t pkt_rx;
  65         void *obj;
  66 };
  67
  68 /*
  69  * Output port.
  70  */
  71 struct port_out_type {
  72         TAILQ_ENTRY(port_out_type) node;
  73         char name[RTE_SWX_NAME_SIZE];
  74         struct rte_swx_port_out_ops ops;
  75 };
  76
  77 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  78
  79 struct port_out {
  80         TAILQ_ENTRY(port_out) node;
  81         struct port_out_type *type;
  82         void *obj;
  83         uint32_t id;
  84 };
  85
  86 TAILQ_HEAD(port_out_tailq, port_out);
  87
  88 struct port_out_runtime {
  89         rte_swx_port_out_pkt_tx_t pkt_tx;
  90         rte_swx_port_out_flush_t flush;
  91         void *obj;
  92 };
  93
  94 /*
  95  * Extern object.
  96  */
  97 struct extern_type_member_func {
  98         TAILQ_ENTRY(extern_type_member_func) node;
  99         char name[RTE_SWX_NAME_SIZE];
 100         rte_swx_extern_type_member_func_t func;
 101         uint32_t id;
 102 };
 103
 104 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 105
 106 struct extern_type {
 107         TAILQ_ENTRY(extern_type) node;
 108         char name[RTE_SWX_NAME_SIZE];
 109         struct struct_type *mailbox_struct_type;
 110         rte_swx_extern_type_constructor_t constructor;
 111         rte_swx_extern_type_destructor_t destructor;
 112         struct extern_type_member_func_tailq funcs;
 113         uint32_t n_funcs;
 114 };
 115
 116 TAILQ_HEAD(extern_type_tailq, extern_type);
 117
 118 struct extern_obj {
 119         TAILQ_ENTRY(extern_obj) node;
 120         char name[RTE_SWX_NAME_SIZE];
 121         struct extern_type *type;
 122         void *obj;
 123         uint32_t struct_id;
 124         uint32_t id;
 125 };
 126
 127 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 128
 129 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 130 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 131 #endif
 132
 133 struct extern_obj_runtime {
 134         void *obj;
 135         uint8_t *mailbox;
 136         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 137 };
 138
 139 /*
 140  * Extern function.
 141  */
 142 struct extern_func {
 143         TAILQ_ENTRY(extern_func) node;
 144         char name[RTE_SWX_NAME_SIZE];
 145         struct struct_type *mailbox_struct_type;
 146         rte_swx_extern_func_t func;
 147         uint32_t struct_id;
 148         uint32_t id;
 149 };
 150
 151 TAILQ_HEAD(extern_func_tailq, extern_func);
 152
 153 struct extern_func_runtime {
 154         uint8_t *mailbox;
 155         rte_swx_extern_func_t func;
 156 };
 157
 158 /*
 159  * Header.
 160  */
 161 struct header {
 162         TAILQ_ENTRY(header) node;
 163         char name[RTE_SWX_NAME_SIZE];
 164         struct struct_type *st;
 165         uint32_t struct_id;
 166         uint32_t id;
 167 };
 168
 169 TAILQ_HEAD(header_tailq, header);
 170
 171 struct header_runtime {
 172         uint8_t *ptr0;
 173 };
 174
 175 struct header_out_runtime {
 176         uint8_t *ptr0;
 177         uint8_t *ptr;
 178         uint32_t n_bytes;
 179 };
 180
 181 /*
 182  * Instruction.
 183  */
 184 struct instruction {
 185 };
 186
 187 /*
 188  * Action.
 189  */
 190 struct action {
 191         TAILQ_ENTRY(action) node;
 192         char name[RTE_SWX_NAME_SIZE];
 193         struct struct_type *st;
 194         struct instruction *instructions;
 195         uint32_t n_instructions;
 196         uint32_t id;
 197 };
 198
 199 TAILQ_HEAD(action_tailq, action);
 200
 201 /*
 202  * Table.
 203  */
 204 struct table_type {
 205         TAILQ_ENTRY(table_type) node;
 206         char name[RTE_SWX_NAME_SIZE];
 207         enum rte_swx_table_match_type match_type;
 208         struct rte_swx_table_ops ops;
 209 };
 210
 211 TAILQ_HEAD(table_type_tailq, table_type);
 212
 213 struct match_field {
 214         enum rte_swx_table_match_type match_type;
 215         struct field *field;
 216 };
 217
 218 struct table {
 219         TAILQ_ENTRY(table) node;
 220         char name[RTE_SWX_NAME_SIZE];
 221         char args[RTE_SWX_NAME_SIZE];
 222         struct table_type *type; /* NULL when n_fields == 0. */
 223
 224         /* Match. */
 225         struct match_field *fields;
 226         uint32_t n_fields;
 227         int is_header; /* Only valid when n_fields > 0. */
 228         struct header *header; /* Only valid when n_fields > 0. */
 229
 230         /* Action. */
 231         struct action **actions;
 232         struct action *default_action;
 233         uint8_t *default_action_data;
 234         uint32_t n_actions;
 235         int default_action_is_const;
 236         uint32_t action_data_size_max;
 237
 238         uint32_t size;
 239         uint32_t id;
 240 };
 241
 242 TAILQ_HEAD(table_tailq, table);
 243
 244 struct table_runtime {
 245         rte_swx_table_lookup_t func;
 246         void *mailbox;
 247         uint8_t **key;
 248 };
 249
 250 /*
 251  * Pipeline.
 252  */
 253 struct thread {
 254         /* Structures. */
 255         uint8_t **structs;
 256
 257         /* Packet headers. */
 258         struct header_runtime *headers; /* Extracted or generated headers. */
 259         struct header_out_runtime *headers_out; /* Emitted headers. */
 260         uint8_t *header_storage;
 261         uint8_t *header_out_storage;
 262         uint64_t valid_headers;
 263         uint32_t n_headers_out;
 264
 265         /* Packet meta-data. */
 266         uint8_t *metadata;
 267
 268         /* Tables. */
 269         struct table_runtime *tables;
 270         struct rte_swx_table_state *table_state;
 271         uint64_t action_id;
 272         int hit; /* 0 = Miss, 1 = Hit. */
 273
 274         /* Extern objects and functions. */
 275         struct extern_obj_runtime *extern_objs;
 276         struct extern_func_runtime *extern_funcs;
 277 };
 278
 279 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 280 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 281 #endif
 282
 283 struct rte_swx_pipeline {
 284         struct struct_type_tailq struct_types;
 285         struct port_in_type_tailq port_in_types;
 286         struct port_in_tailq ports_in;
 287         struct port_out_type_tailq port_out_types;
 288         struct port_out_tailq ports_out;
 289         struct extern_type_tailq extern_types;
 290         struct extern_obj_tailq extern_objs;
 291         struct extern_func_tailq extern_funcs;
 292         struct header_tailq headers;
 293         struct struct_type *metadata_st;
 294         uint32_t metadata_struct_id;
 295         struct action_tailq actions;
 296         struct table_type_tailq table_types;
 297         struct table_tailq tables;
 298
 299         struct port_in_runtime *in;
 300         struct port_out_runtime *out;
 301         struct instruction **action_instructions;
 302         struct rte_swx_table_state *table_state;
 303         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 304
 305         uint32_t n_structs;
 306         uint32_t n_ports_in;
 307         uint32_t n_ports_out;
 308         uint32_t n_extern_objs;
 309         uint32_t n_extern_funcs;
 310         uint32_t n_actions;
 311         uint32_t n_tables;
 312         uint32_t n_headers;
 313         int build_done;
 314         int numa_node;
 315 };
 316
 317 /*
 318  * Struct.
 319  */
 320 static struct struct_type *
 321 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 322 {
 323         struct struct_type *elem;
 324
 325         TAILQ_FOREACH(elem, &p->struct_types, node)
 326                 if (strcmp(elem->name, name) == 0)
 327                         return elem;
 328
 329         return NULL;
 330 }
 331
 332 static struct field *
 333 struct_type_field_find(struct struct_type *st, const char *name)
 334 {
 335         uint32_t i;
 336
 337         for (i = 0; i < st->n_fields; i++) {
 338                 struct field *f = &st->fields[i];
 339
 340                 if (strcmp(f->name, name) == 0)
 341                         return f;
 342         }
 343
 344         return NULL;
 345 }
 346
 347 int
 348 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 349                                       const char *name,
 350                                       struct rte_swx_field_params *fields,
 351                                       uint32_t n_fields)
 352 {
 353         struct struct_type *st;
 354         uint32_t i;
 355
 356         CHECK(p, EINVAL);
 357         CHECK_NAME(name, EINVAL);
 358         CHECK(fields, EINVAL);
 359         CHECK(n_fields, EINVAL);
 360
 361         for (i = 0; i < n_fields; i++) {
 362                 struct rte_swx_field_params *f = &fields[i];
 363                 uint32_t j;
 364
 365                 CHECK_NAME(f->name, EINVAL);
 366                 CHECK(f->n_bits, EINVAL);
 367                 CHECK(f->n_bits <= 64, EINVAL);
 368                 CHECK((f->n_bits & 7) == 0, EINVAL);
 369
 370                 for (j = 0; j < i; j++) {
 371                         struct rte_swx_field_params *f_prev = &fields[j];
 372
 373                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 374                 }
 375         }
 376
 377         CHECK(!struct_type_find(p, name), EEXIST);
 378
 379         /* Node allocation. */
 380         st = calloc(1, sizeof(struct struct_type));
 381         CHECK(st, ENOMEM);
 382
 383         st->fields = calloc(n_fields, sizeof(struct field));
 384         if (!st->fields) {
 385                 free(st);
 386                 CHECK(0, ENOMEM);
 387         }
 388
 389         /* Node initialization. */
 390         strcpy(st->name, name);
 391         for (i = 0; i < n_fields; i++) {
 392                 struct field *dst = &st->fields[i];
 393                 struct rte_swx_field_params *src = &fields[i];
 394
 395                 strcpy(dst->name, src->name);
 396                 dst->n_bits = src->n_bits;
 397                 dst->offset = st->n_bits;
 398
 399                 st->n_bits += src->n_bits;
 400         }
 401         st->n_fields = n_fields;
 402
 403         /* Node add to tailq. */
 404         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 405
 406         return 0;
 407 }
 408
 409 static int
 410 struct_build(struct rte_swx_pipeline *p)
 411 {
 412         uint32_t i;
 413
 414         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 415                 struct thread *t = &p->threads[i];
 416
 417                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 418                 CHECK(t->structs, ENOMEM);
 419         }
 420
 421         return 0;
 422 }
 423
 424 static void
 425 struct_build_free(struct rte_swx_pipeline *p)
 426 {
 427         uint32_t i;
 428
 429         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 430                 struct thread *t = &p->threads[i];
 431
 432                 free(t->structs);
 433                 t->structs = NULL;
 434         }
 435 }
 436
 437 static void
 438 struct_free(struct rte_swx_pipeline *p)
 439 {
 440         struct_build_free(p);
 441
 442         /* Struct types. */
 443         for ( ; ; ) {
 444                 struct struct_type *elem;
 445
 446                 elem = TAILQ_FIRST(&p->struct_types);
 447                 if (!elem)
 448                         break;
 449
 450                 TAILQ_REMOVE(&p->struct_types, elem, node);
 451                 free(elem->fields);
 452                 free(elem);
 453         }
 454 }
 455
 456 /*
 457  * Input port.
 458  */
 459 static struct port_in_type *
 460 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 461 {
 462         struct port_in_type *elem;
 463
 464         if (!name)
 465                 return NULL;
 466
 467         TAILQ_FOREACH(elem, &p->port_in_types, node)
 468                 if (strcmp(elem->name, name) == 0)
 469                         return elem;
 470
 471         return NULL;
 472 }
 473
 474 int
 475 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 476                                        const char *name,
 477                                        struct rte_swx_port_in_ops *ops)
 478 {
 479         struct port_in_type *elem;
 480
 481         CHECK(p, EINVAL);
 482         CHECK_NAME(name, EINVAL);
 483         CHECK(ops, EINVAL);
 484         CHECK(ops->create, EINVAL);
 485         CHECK(ops->free, EINVAL);
 486         CHECK(ops->pkt_rx, EINVAL);
 487         CHECK(ops->stats_read, EINVAL);
 488
 489         CHECK(!port_in_type_find(p, name), EEXIST);
 490
 491         /* Node allocation. */
 492         elem = calloc(1, sizeof(struct port_in_type));
 493         CHECK(elem, ENOMEM);
 494
 495         /* Node initialization. */
 496         strcpy(elem->name, name);
 497         memcpy(&elem->ops, ops, sizeof(*ops));
 498
 499         /* Node add to tailq. */
 500         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 501
 502         return 0;
 503 }
 504
 505 static struct port_in *
 506 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 507 {
 508         struct port_in *port;
 509
 510         TAILQ_FOREACH(port, &p->ports_in, node)
 511                 if (port->id == port_id)
 512                         return port;
 513
 514         return NULL;
 515 }
 516
 517 int
 518 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 519                                 uint32_t port_id,
 520                                 const char *port_type_name,
 521                                 void *args)
 522 {
 523         struct port_in_type *type = NULL;
 524         struct port_in *port = NULL;
 525         void *obj = NULL;
 526
 527         CHECK(p, EINVAL);
 528
 529         CHECK(!port_in_find(p, port_id), EINVAL);
 530
 531         CHECK_NAME(port_type_name, EINVAL);
 532         type = port_in_type_find(p, port_type_name);
 533         CHECK(type, EINVAL);
 534
 535         obj = type->ops.create(args);
 536         CHECK(obj, ENODEV);
 537
 538         /* Node allocation. */
 539         port = calloc(1, sizeof(struct port_in));
 540         CHECK(port, ENOMEM);
 541
 542         /* Node initialization. */
 543         port->type = type;
 544         port->obj = obj;
 545         port->id = port_id;
 546
 547         /* Node add to tailq. */
 548         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 549         if (p->n_ports_in < port_id + 1)
 550                 p->n_ports_in = port_id + 1;
 551
 552         return 0;
 553 }
 554
 555 static int
 556 port_in_build(struct rte_swx_pipeline *p)
 557 {
 558         struct port_in *port;
 559         uint32_t i;
 560
 561         CHECK(p->n_ports_in, EINVAL);
 562         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 563
 564         for (i = 0; i < p->n_ports_in; i++)
 565                 CHECK(port_in_find(p, i), EINVAL);
 566
 567         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 568         CHECK(p->in, ENOMEM);
 569
 570         TAILQ_FOREACH(port, &p->ports_in, node) {
 571                 struct port_in_runtime *in = &p->in[port->id];
 572
 573                 in->pkt_rx = port->type->ops.pkt_rx;
 574                 in->obj = port->obj;
 575         }
 576
 577         return 0;
 578 }
 579
 580 static void
 581 port_in_build_free(struct rte_swx_pipeline *p)
 582 {
 583         free(p->in);
 584         p->in = NULL;
 585 }
 586
 587 static void
 588 port_in_free(struct rte_swx_pipeline *p)
 589 {
 590         port_in_build_free(p);
 591
 592         /* Input ports. */
 593         for ( ; ; ) {
 594                 struct port_in *port;
 595
 596                 port = TAILQ_FIRST(&p->ports_in);
 597                 if (!port)
 598                         break;
 599
 600                 TAILQ_REMOVE(&p->ports_in, port, node);
 601                 port->type->ops.free(port->obj);
 602                 free(port);
 603         }
 604
 605         /* Input port types. */
 606         for ( ; ; ) {
 607                 struct port_in_type *elem;
 608
 609                 elem = TAILQ_FIRST(&p->port_in_types);
 610                 if (!elem)
 611                         break;
 612
 613                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 614                 free(elem);
 615         }
 616 }
 617
 618 /*
 619  * Output port.
 620  */
 621 static struct port_out_type *
 622 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 623 {
 624         struct port_out_type *elem;
 625
 626         if (!name)
 627                 return NULL;
 628
 629         TAILQ_FOREACH(elem, &p->port_out_types, node)
 630                 if (!strcmp(elem->name, name))
 631                         return elem;
 632
 633         return NULL;
 634 }
 635
 636 int
 637 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 638                                         const char *name,
 639                                         struct rte_swx_port_out_ops *ops)
 640 {
 641         struct port_out_type *elem;
 642
 643         CHECK(p, EINVAL);
 644         CHECK_NAME(name, EINVAL);
 645         CHECK(ops, EINVAL);
 646         CHECK(ops->create, EINVAL);
 647         CHECK(ops->free, EINVAL);
 648         CHECK(ops->pkt_tx, EINVAL);
 649         CHECK(ops->stats_read, EINVAL);
 650
 651         CHECK(!port_out_type_find(p, name), EEXIST);
 652
 653         /* Node allocation. */
 654         elem = calloc(1, sizeof(struct port_out_type));
 655         CHECK(elem, ENOMEM);
 656
 657         /* Node initialization. */
 658         strcpy(elem->name, name);
 659         memcpy(&elem->ops, ops, sizeof(*ops));
 660
 661         /* Node add to tailq. */
 662         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 663
 664         return 0;
 665 }
 666
 667 static struct port_out *
 668 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 669 {
 670         struct port_out *port;
 671
 672         TAILQ_FOREACH(port, &p->ports_out, node)
 673                 if (port->id == port_id)
 674                         return port;
 675
 676         return NULL;
 677 }
 678
 679 int
 680 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 681                                  uint32_t port_id,
 682                                  const char *port_type_name,
 683                                  void *args)
 684 {
 685         struct port_out_type *type = NULL;
 686         struct port_out *port = NULL;
 687         void *obj = NULL;
 688
 689         CHECK(p, EINVAL);
 690
 691         CHECK(!port_out_find(p, port_id), EINVAL);
 692
 693         CHECK_NAME(port_type_name, EINVAL);
 694         type = port_out_type_find(p, port_type_name);
 695         CHECK(type, EINVAL);
 696
 697         obj = type->ops.create(args);
 698         CHECK(obj, ENODEV);
 699
 700         /* Node allocation. */
 701         port = calloc(1, sizeof(struct port_out));
 702         CHECK(port, ENOMEM);
 703
 704         /* Node initialization. */
 705         port->type = type;
 706         port->obj = obj;
 707         port->id = port_id;
 708
 709         /* Node add to tailq. */
 710         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 711         if (p->n_ports_out < port_id + 1)
 712                 p->n_ports_out = port_id + 1;
 713
 714         return 0;
 715 }
 716
 717 static int
 718 port_out_build(struct rte_swx_pipeline *p)
 719 {
 720         struct port_out *port;
 721         uint32_t i;
 722
 723         CHECK(p->n_ports_out, EINVAL);
 724
 725         for (i = 0; i < p->n_ports_out; i++)
 726                 CHECK(port_out_find(p, i), EINVAL);
 727
 728         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 729         CHECK(p->out, ENOMEM);
 730
 731         TAILQ_FOREACH(port, &p->ports_out, node) {
 732                 struct port_out_runtime *out = &p->out[port->id];
 733
 734                 out->pkt_tx = port->type->ops.pkt_tx;
 735                 out->flush = port->type->ops.flush;
 736                 out->obj = port->obj;
 737         }
 738
 739         return 0;
 740 }
 741
 742 static void
 743 port_out_build_free(struct rte_swx_pipeline *p)
 744 {
 745         free(p->out);
 746         p->out = NULL;
 747 }
 748
 749 static void
 750 port_out_free(struct rte_swx_pipeline *p)
 751 {
 752         port_out_build_free(p);
 753
 754         /* Output ports. */
 755         for ( ; ; ) {
 756                 struct port_out *port;
 757
 758                 port = TAILQ_FIRST(&p->ports_out);
 759                 if (!port)
 760                         break;
 761
 762                 TAILQ_REMOVE(&p->ports_out, port, node);
 763                 port->type->ops.free(port->obj);
 764                 free(port);
 765         }
 766
 767         /* Output port types. */
 768         for ( ; ; ) {
 769                 struct port_out_type *elem;
 770
 771                 elem = TAILQ_FIRST(&p->port_out_types);
 772                 if (!elem)
 773                         break;
 774
 775                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 776                 free(elem);
 777         }
 778 }
 779
 780 /*
 781  * Extern object.
 782  */
 783 static struct extern_type *
 784 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 785 {
 786         struct extern_type *elem;
 787
 788         TAILQ_FOREACH(elem, &p->extern_types, node)
 789                 if (strcmp(elem->name, name) == 0)
 790                         return elem;
 791
 792         return NULL;
 793 }
 794
 795 static struct extern_type_member_func *
 796 extern_type_member_func_find(struct extern_type *type, const char *name)
 797 {
 798         struct extern_type_member_func *elem;
 799
 800         TAILQ_FOREACH(elem, &type->funcs, node)
 801                 if (strcmp(elem->name, name) == 0)
 802                         return elem;
 803
 804         return NULL;
 805 }
 806
 807 static struct extern_obj *
 808 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 809 {
 810         struct extern_obj *elem;
 811
 812         TAILQ_FOREACH(elem, &p->extern_objs, node)
 813                 if (strcmp(elem->name, name) == 0)
 814                         return elem;
 815
 816         return NULL;
 817 }
 818
 819 int
 820 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 821         const char *name,
 822         const char *mailbox_struct_type_name,
 823         rte_swx_extern_type_constructor_t constructor,
 824         rte_swx_extern_type_destructor_t destructor)
 825 {
 826         struct extern_type *elem;
 827         struct struct_type *mailbox_struct_type;
 828
 829         CHECK(p, EINVAL);
 830
 831         CHECK_NAME(name, EINVAL);
 832         CHECK(!extern_type_find(p, name), EEXIST);
 833
 834         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 835         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 836         CHECK(mailbox_struct_type, EINVAL);
 837
 838         CHECK(constructor, EINVAL);
 839         CHECK(destructor, EINVAL);
 840
 841         /* Node allocation. */
 842         elem = calloc(1, sizeof(struct extern_type));
 843         CHECK(elem, ENOMEM);
 844
 845         /* Node initialization. */
 846         strcpy(elem->name, name);
 847         elem->mailbox_struct_type = mailbox_struct_type;
 848         elem->constructor = constructor;
 849         elem->destructor = destructor;
 850         TAILQ_INIT(&elem->funcs);
 851
 852         /* Node add to tailq. */
 853         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 854
 855         return 0;
 856 }
 857
 858 int
 859 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 860         const char *extern_type_name,
 861         const char *name,
 862         rte_swx_extern_type_member_func_t member_func)
 863 {
 864         struct extern_type *type;
 865         struct extern_type_member_func *type_member;
 866
 867         CHECK(p, EINVAL);
 868
 869         CHECK(extern_type_name, EINVAL);
 870         type = extern_type_find(p, extern_type_name);
 871         CHECK(type, EINVAL);
 872         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 873
 874         CHECK(name, EINVAL);
 875         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 876
 877         CHECK(member_func, EINVAL);
 878
 879         /* Node allocation. */
 880         type_member = calloc(1, sizeof(struct extern_type_member_func));
 881         CHECK(type_member, ENOMEM);
 882
 883         /* Node initialization. */
 884         strcpy(type_member->name, name);
 885         type_member->func = member_func;
 886         type_member->id = type->n_funcs;
 887
 888         /* Node add to tailq. */
 889         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 890         type->n_funcs++;
 891
 892         return 0;
 893 }
 894
 895 int
 896 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 897                                       const char *extern_type_name,
 898                                       const char *name,
 899                                       const char *args)
 900 {
 901         struct extern_type *type;
 902         struct extern_obj *obj;
 903         void *obj_handle;
 904
 905         CHECK(p, EINVAL);
 906
 907         CHECK_NAME(extern_type_name, EINVAL);
 908         type = extern_type_find(p, extern_type_name);
 909         CHECK(type, EINVAL);
 910
 911         CHECK_NAME(name, EINVAL);
 912         CHECK(!extern_obj_find(p, name), EEXIST);
 913
 914         /* Node allocation. */
 915         obj = calloc(1, sizeof(struct extern_obj));
 916         CHECK(obj, ENOMEM);
 917
 918         /* Object construction. */
 919         obj_handle = type->constructor(args);
 920         if (!obj_handle) {
 921                 free(obj);
 922                 CHECK(0, ENODEV);
 923         }
 924
 925         /* Node initialization. */
 926         strcpy(obj->name, name);
 927         obj->type = type;
 928         obj->obj = obj_handle;
 929         obj->struct_id = p->n_structs;
 930         obj->id = p->n_extern_objs;
 931
 932         /* Node add to tailq. */
 933         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 934         p->n_extern_objs++;
 935         p->n_structs++;
 936
 937         return 0;
 938 }
 939
 940 static int
 941 extern_obj_build(struct rte_swx_pipeline *p)
 942 {
 943         uint32_t i;
 944
 945         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 946                 struct thread *t = &p->threads[i];
 947                 struct extern_obj *obj;
 948
 949                 t->extern_objs = calloc(p->n_extern_objs,
 950                                         sizeof(struct extern_obj_runtime));
 951                 CHECK(t->extern_objs, ENOMEM);
 952
 953                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 954                         struct extern_obj_runtime *r =
 955                                 &t->extern_objs[obj->id];
 956                         struct extern_type_member_func *func;
 957                         uint32_t mailbox_size =
 958                                 obj->type->mailbox_struct_type->n_bits / 8;
 959
 960                         r->obj = obj->obj;
 961
 962                         r->mailbox = calloc(1, mailbox_size);
 963                         CHECK(r->mailbox, ENOMEM);
 964
 965                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 966                                 r->funcs[func->id] = func->func;
 967
 968                         t->structs[obj->struct_id] = r->mailbox;
 969                 }
 970         }
 971
 972         return 0;
 973 }
 974
 975 static void
 976 extern_obj_build_free(struct rte_swx_pipeline *p)
 977 {
 978         uint32_t i;
 979
 980         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 981                 struct thread *t = &p->threads[i];
 982                 uint32_t j;
 983
 984                 if (!t->extern_objs)
 985                         continue;
 986
 987                 for (j = 0; j < p->n_extern_objs; j++) {
 988                         struct extern_obj_runtime *r = &t->extern_objs[j];
 989
 990                         free(r->mailbox);
 991                 }
 992
 993                 free(t->extern_objs);
 994                 t->extern_objs = NULL;
 995         }
 996 }
 997
 998 static void
 999 extern_obj_free(struct rte_swx_pipeline *p)
1000 {
1001         extern_obj_build_free(p);
1002
1003         /* Extern objects. */
1004         for ( ; ; ) {
1005                 struct extern_obj *elem;
1006
1007                 elem = TAILQ_FIRST(&p->extern_objs);
1008                 if (!elem)
1009                         break;
1010
1011                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1012                 if (elem->obj)
1013                         elem->type->destructor(elem->obj);
1014                 free(elem);
1015         }
1016
1017         /* Extern types. */
1018         for ( ; ; ) {
1019                 struct extern_type *elem;
1020
1021                 elem = TAILQ_FIRST(&p->extern_types);
1022                 if (!elem)
1023                         break;
1024
1025                 TAILQ_REMOVE(&p->extern_types, elem, node);
1026
1027                 for ( ; ; ) {
1028                         struct extern_type_member_func *func;
1029
1030                         func = TAILQ_FIRST(&elem->funcs);
1031                         if (!func)
1032                                 break;
1033
1034                         TAILQ_REMOVE(&elem->funcs, func, node);
1035                         free(func);
1036                 }
1037
1038                 free(elem);
1039         }
1040 }
1041
1042 /*
1043  * Extern function.
1044  */
1045 static struct extern_func *
1046 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1047 {
1048         struct extern_func *elem;
1049
1050         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1051                 if (strcmp(elem->name, name) == 0)
1052                         return elem;
1053
1054         return NULL;
1055 }
1056
1057 int
1058 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1059                                       const char *name,
1060                                       const char *mailbox_struct_type_name,
1061                                       rte_swx_extern_func_t func)
1062 {
1063         struct extern_func *f;
1064         struct struct_type *mailbox_struct_type;
1065
1066         CHECK(p, EINVAL);
1067
1068         CHECK_NAME(name, EINVAL);
1069         CHECK(!extern_func_find(p, name), EEXIST);
1070
1071         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1072         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1073         CHECK(mailbox_struct_type, EINVAL);
1074
1075         CHECK(func, EINVAL);
1076
1077         /* Node allocation. */
1078         f = calloc(1, sizeof(struct extern_func));
1079         CHECK(func, ENOMEM);
1080
1081         /* Node initialization. */
1082         strcpy(f->name, name);
1083         f->mailbox_struct_type = mailbox_struct_type;
1084         f->func = func;
1085         f->struct_id = p->n_structs;
1086         f->id = p->n_extern_funcs;
1087
1088         /* Node add to tailq. */
1089         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1090         p->n_extern_funcs++;
1091         p->n_structs++;
1092
1093         return 0;
1094 }
1095
1096 static int
1097 extern_func_build(struct rte_swx_pipeline *p)
1098 {
1099         uint32_t i;
1100
1101         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1102                 struct thread *t = &p->threads[i];
1103                 struct extern_func *func;
1104
1105                 /* Memory allocation. */
1106                 t->extern_funcs = calloc(p->n_extern_funcs,
1107                                          sizeof(struct extern_func_runtime));
1108                 CHECK(t->extern_funcs, ENOMEM);
1109
1110                 /* Extern function. */
1111                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1112                         struct extern_func_runtime *r =
1113                                 &t->extern_funcs[func->id];
1114                         uint32_t mailbox_size =
1115                                 func->mailbox_struct_type->n_bits / 8;
1116
1117                         r->func = func->func;
1118
1119                         r->mailbox = calloc(1, mailbox_size);
1120                         CHECK(r->mailbox, ENOMEM);
1121
1122                         t->structs[func->struct_id] = r->mailbox;
1123                 }
1124         }
1125
1126         return 0;
1127 }
1128
1129 static void
1130 extern_func_build_free(struct rte_swx_pipeline *p)
1131 {
1132         uint32_t i;
1133
1134         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1135                 struct thread *t = &p->threads[i];
1136                 uint32_t j;
1137
1138                 if (!t->extern_funcs)
1139                         continue;
1140
1141                 for (j = 0; j < p->n_extern_funcs; j++) {
1142                         struct extern_func_runtime *r = &t->extern_funcs[j];
1143
1144                         free(r->mailbox);
1145                 }
1146
1147                 free(t->extern_funcs);
1148                 t->extern_funcs = NULL;
1149         }
1150 }
1151
1152 static void
1153 extern_func_free(struct rte_swx_pipeline *p)
1154 {
1155         extern_func_build_free(p);
1156
1157         for ( ; ; ) {
1158                 struct extern_func *elem;
1159
1160                 elem = TAILQ_FIRST(&p->extern_funcs);
1161                 if (!elem)
1162                         break;
1163
1164                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1165                 free(elem);
1166         }
1167 }
1168
1169 /*
1170  * Header.
1171  */
1172 static struct header *
1173 header_find(struct rte_swx_pipeline *p, const char *name)
1174 {
1175         struct header *elem;
1176
1177         TAILQ_FOREACH(elem, &p->headers, node)
1178                 if (strcmp(elem->name, name) == 0)
1179                         return elem;
1180
1181         return NULL;
1182 }
1183
1184 static struct field *
1185 header_field_parse(struct rte_swx_pipeline *p,
1186                    const char *name,
1187                    struct header **header)
1188 {
1189         struct header *h;
1190         struct field *f;
1191         char *header_name, *field_name;
1192
1193         if ((name[0] != 'h') || (name[1] != '.'))
1194                 return NULL;
1195
1196         header_name = strdup(&name[2]);
1197         if (!header_name)
1198                 return NULL;
1199
1200         field_name = strchr(header_name, '.');
1201         if (!field_name) {
1202                 free(header_name);
1203                 return NULL;
1204         }
1205
1206         *field_name = 0;
1207         field_name++;
1208
1209         h = header_find(p, header_name);
1210         if (!h) {
1211                 free(header_name);
1212                 return NULL;
1213         }
1214
1215         f = struct_type_field_find(h->st, field_name);
1216         if (!f) {
1217                 free(header_name);
1218                 return NULL;
1219         }
1220
1221         if (header)
1222                 *header = h;
1223
1224         free(header_name);
1225         return f;
1226 }
1227
1228 int
1229 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1230                                         const char *name,
1231                                         const char *struct_type_name)
1232 {
1233         struct struct_type *st;
1234         struct header *h;
1235         size_t n_headers_max;
1236
1237         CHECK(p, EINVAL);
1238         CHECK_NAME(name, EINVAL);
1239         CHECK_NAME(struct_type_name, EINVAL);
1240
1241         CHECK(!header_find(p, name), EEXIST);
1242
1243         st = struct_type_find(p, struct_type_name);
1244         CHECK(st, EINVAL);
1245
1246         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1247         CHECK(p->n_headers < n_headers_max, ENOSPC);
1248
1249         /* Node allocation. */
1250         h = calloc(1, sizeof(struct header));
1251         CHECK(h, ENOMEM);
1252
1253         /* Node initialization. */
1254         strcpy(h->name, name);
1255         h->st = st;
1256         h->struct_id = p->n_structs;
1257         h->id = p->n_headers;
1258
1259         /* Node add to tailq. */
1260         TAILQ_INSERT_TAIL(&p->headers, h, node);
1261         p->n_headers++;
1262         p->n_structs++;
1263
1264         return 0;
1265 }
1266
1267 static int
1268 header_build(struct rte_swx_pipeline *p)
1269 {
1270         struct header *h;
1271         uint32_t n_bytes = 0, i;
1272
1273         TAILQ_FOREACH(h, &p->headers, node) {
1274                 n_bytes += h->st->n_bits / 8;
1275         }
1276
1277         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1278                 struct thread *t = &p->threads[i];
1279                 uint32_t offset = 0;
1280
1281                 t->headers = calloc(p->n_headers,
1282                                     sizeof(struct header_runtime));
1283                 CHECK(t->headers, ENOMEM);
1284
1285                 t->headers_out = calloc(p->n_headers,
1286                                         sizeof(struct header_out_runtime));
1287                 CHECK(t->headers_out, ENOMEM);
1288
1289                 t->header_storage = calloc(1, n_bytes);
1290                 CHECK(t->header_storage, ENOMEM);
1291
1292                 t->header_out_storage = calloc(1, n_bytes);
1293                 CHECK(t->header_out_storage, ENOMEM);
1294
1295                 TAILQ_FOREACH(h, &p->headers, node) {
1296                         uint8_t *header_storage;
1297
1298                         header_storage = &t->header_storage[offset];
1299                         offset += h->st->n_bits / 8;
1300
1301                         t->headers[h->id].ptr0 = header_storage;
1302                         t->structs[h->struct_id] = header_storage;
1303                 }
1304         }
1305
1306         return 0;
1307 }
1308
1309 static void
1310 header_build_free(struct rte_swx_pipeline *p)
1311 {
1312         uint32_t i;
1313
1314         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1315                 struct thread *t = &p->threads[i];
1316
1317                 free(t->headers_out);
1318                 t->headers_out = NULL;
1319
1320                 free(t->headers);
1321                 t->headers = NULL;
1322
1323                 free(t->header_out_storage);
1324                 t->header_out_storage = NULL;
1325
1326                 free(t->header_storage);
1327                 t->header_storage = NULL;
1328         }
1329 }
1330
1331 static void
1332 header_free(struct rte_swx_pipeline *p)
1333 {
1334         header_build_free(p);
1335
1336         for ( ; ; ) {
1337                 struct header *elem;
1338
1339                 elem = TAILQ_FIRST(&p->headers);
1340                 if (!elem)
1341                         break;
1342
1343                 TAILQ_REMOVE(&p->headers, elem, node);
1344                 free(elem);
1345         }
1346 }
1347
1348 /*
1349  * Meta-data.
1350  */
1351 static struct field *
1352 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1353 {
1354         if (!p->metadata_st)
1355                 return NULL;
1356
1357         if (name[0] != 'm' || name[1] != '.')
1358                 return NULL;
1359
1360         return struct_type_field_find(p->metadata_st, &name[2]);
1361 }
1362
1363 int
1364 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1365                                           const char *struct_type_name)
1366 {
1367         struct struct_type *st = NULL;
1368
1369         CHECK(p, EINVAL);
1370
1371         CHECK_NAME(struct_type_name, EINVAL);
1372         st  = struct_type_find(p, struct_type_name);
1373         CHECK(st, EINVAL);
1374         CHECK(!p->metadata_st, EINVAL);
1375
1376         p->metadata_st = st;
1377         p->metadata_struct_id = p->n_structs;
1378
1379         p->n_structs++;
1380
1381         return 0;
1382 }
1383
1384 static int
1385 metadata_build(struct rte_swx_pipeline *p)
1386 {
1387         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1388         uint32_t i;
1389
1390         /* Thread-level initialization. */
1391         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1392                 struct thread *t = &p->threads[i];
1393                 uint8_t *metadata;
1394
1395                 metadata = calloc(1, n_bytes);
1396                 CHECK(metadata, ENOMEM);
1397
1398                 t->metadata = metadata;
1399                 t->structs[p->metadata_struct_id] = metadata;
1400         }
1401
1402         return 0;
1403 }
1404
1405 static void
1406 metadata_build_free(struct rte_swx_pipeline *p)
1407 {
1408         uint32_t i;
1409
1410         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1411                 struct thread *t = &p->threads[i];
1412
1413                 free(t->metadata);
1414                 t->metadata = NULL;
1415         }
1416 }
1417
1418 static void
1419 metadata_free(struct rte_swx_pipeline *p)
1420 {
1421         metadata_build_free(p);
1422 }
1423
1424 /*
1425  * Instruction.
1426  */
1427 static int
1428 instruction_config(struct rte_swx_pipeline *p __rte_unused,
1429                    struct action *a __rte_unused,
1430                    const char **instructions __rte_unused,
1431                    uint32_t n_instructions __rte_unused)
1432 {
1433         return 0;
1434 }
1435
1436 /*
1437  * Action.
1438  */
1439 static struct action *
1440 action_find(struct rte_swx_pipeline *p, const char *name)
1441 {
1442         struct action *elem;
1443
1444         if (!name)
1445                 return NULL;
1446
1447         TAILQ_FOREACH(elem, &p->actions, node)
1448                 if (strcmp(elem->name, name) == 0)
1449                         return elem;
1450
1451         return NULL;
1452 }
1453
1454 int
1455 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
1456                                const char *name,
1457                                const char *args_struct_type_name,
1458                                const char **instructions,
1459                                uint32_t n_instructions)
1460 {
1461         struct struct_type *args_struct_type;
1462         struct action *a;
1463         int err;
1464
1465         CHECK(p, EINVAL);
1466
1467         CHECK_NAME(name, EINVAL);
1468         CHECK(!action_find(p, name), EEXIST);
1469
1470         if (args_struct_type_name) {
1471                 CHECK_NAME(args_struct_type_name, EINVAL);
1472                 args_struct_type = struct_type_find(p, args_struct_type_name);
1473                 CHECK(args_struct_type, EINVAL);
1474         } else {
1475                 args_struct_type = NULL;
1476         }
1477
1478         /* Node allocation. */
1479         a = calloc(1, sizeof(struct action));
1480         CHECK(a, ENOMEM);
1481
1482         /* Node initialization. */
1483         strcpy(a->name, name);
1484         a->st = args_struct_type;
1485         a->id = p->n_actions;
1486
1487         /* Instruction translation. */
1488         err = instruction_config(p, a, instructions, n_instructions);
1489         if (err) {
1490                 free(a);
1491                 return err;
1492         }
1493
1494         /* Node add to tailq. */
1495         TAILQ_INSERT_TAIL(&p->actions, a, node);
1496         p->n_actions++;
1497
1498         return 0;
1499 }
1500
1501 static int
1502 action_build(struct rte_swx_pipeline *p)
1503 {
1504         struct action *action;
1505
1506         p->action_instructions = calloc(p->n_actions,
1507                                         sizeof(struct instruction *));
1508         CHECK(p->action_instructions, ENOMEM);
1509
1510         TAILQ_FOREACH(action, &p->actions, node)
1511                 p->action_instructions[action->id] = action->instructions;
1512
1513         return 0;
1514 }
1515
1516 static void
1517 action_build_free(struct rte_swx_pipeline *p)
1518 {
1519         free(p->action_instructions);
1520         p->action_instructions = NULL;
1521 }
1522
1523 static void
1524 action_free(struct rte_swx_pipeline *p)
1525 {
1526         action_build_free(p);
1527
1528         for ( ; ; ) {
1529                 struct action *action;
1530
1531                 action = TAILQ_FIRST(&p->actions);
1532                 if (!action)
1533                         break;
1534
1535                 TAILQ_REMOVE(&p->actions, action, node);
1536                 free(action->instructions);
1537                 free(action);
1538         }
1539 }
1540
1541 /*
1542  * Table.
1543  */
1544 static struct table_type *
1545 table_type_find(struct rte_swx_pipeline *p, const char *name)
1546 {
1547         struct table_type *elem;
1548
1549         TAILQ_FOREACH(elem, &p->table_types, node)
1550                 if (strcmp(elem->name, name) == 0)
1551                         return elem;
1552
1553         return NULL;
1554 }
1555
1556 static struct table_type *
1557 table_type_resolve(struct rte_swx_pipeline *p,
1558                    const char *recommended_type_name,
1559                    enum rte_swx_table_match_type match_type)
1560 {
1561         struct table_type *elem;
1562
1563         /* Only consider the recommended type if the match type is correct. */
1564         if (recommended_type_name)
1565                 TAILQ_FOREACH(elem, &p->table_types, node)
1566                         if (!strcmp(elem->name, recommended_type_name) &&
1567                             (elem->match_type == match_type))
1568                                 return elem;
1569
1570         /* Ignore the recommended type and get the first element with this match
1571          * type.
1572          */
1573         TAILQ_FOREACH(elem, &p->table_types, node)
1574                 if (elem->match_type == match_type)
1575                         return elem;
1576
1577         return NULL;
1578 }
1579
1580 static struct table *
1581 table_find(struct rte_swx_pipeline *p, const char *name)
1582 {
1583         struct table *elem;
1584
1585         TAILQ_FOREACH(elem, &p->tables, node)
1586                 if (strcmp(elem->name, name) == 0)
1587                         return elem;
1588
1589         return NULL;
1590 }
1591
1592 static struct table *
1593 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
1594 {
1595         struct table *table = NULL;
1596
1597         TAILQ_FOREACH(table, &p->tables, node)
1598                 if (table->id == id)
1599                         return table;
1600
1601         return NULL;
1602 }
1603
1604 int
1605 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
1606                                      const char *name,
1607                                      enum rte_swx_table_match_type match_type,
1608                                      struct rte_swx_table_ops *ops)
1609 {
1610         struct table_type *elem;
1611
1612         CHECK(p, EINVAL);
1613
1614         CHECK_NAME(name, EINVAL);
1615         CHECK(!table_type_find(p, name), EEXIST);
1616
1617         CHECK(ops, EINVAL);
1618         CHECK(ops->create, EINVAL);
1619         CHECK(ops->lkp, EINVAL);
1620         CHECK(ops->free, EINVAL);
1621
1622         /* Node allocation. */
1623         elem = calloc(1, sizeof(struct table_type));
1624         CHECK(elem, ENOMEM);
1625
1626         /* Node initialization. */
1627         strcpy(elem->name, name);
1628         elem->match_type = match_type;
1629         memcpy(&elem->ops, ops, sizeof(*ops));
1630
1631         /* Node add to tailq. */
1632         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
1633
1634         return 0;
1635 }
1636
1637 static enum rte_swx_table_match_type
1638 table_match_type_resolve(struct rte_swx_match_field_params *fields,
1639                          uint32_t n_fields)
1640 {
1641         uint32_t i;
1642
1643         for (i = 0; i < n_fields; i++)
1644                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
1645                         break;
1646
1647         if (i == n_fields)
1648                 return RTE_SWX_TABLE_MATCH_EXACT;
1649
1650         if ((i == n_fields - 1) &&
1651             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
1652                 return RTE_SWX_TABLE_MATCH_LPM;
1653
1654         return RTE_SWX_TABLE_MATCH_WILDCARD;
1655 }
1656
1657 int
1658 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
1659                               const char *name,
1660                               struct rte_swx_pipeline_table_params *params,
1661                               const char *recommended_table_type_name,
1662                               const char *args,
1663                               uint32_t size)
1664 {
1665         struct table_type *type;
1666         struct table *t;
1667         struct action *default_action;
1668         struct header *header = NULL;
1669         int is_header = 0;
1670         uint32_t offset_prev = 0, action_data_size_max = 0, i;
1671
1672         CHECK(p, EINVAL);
1673
1674         CHECK_NAME(name, EINVAL);
1675         CHECK(!table_find(p, name), EEXIST);
1676
1677         CHECK(params, EINVAL);
1678
1679         /* Match checks. */
1680         CHECK(!params->n_fields || params->fields, EINVAL);
1681         for (i = 0; i < params->n_fields; i++) {
1682                 struct rte_swx_match_field_params *field = &params->fields[i];
1683                 struct header *h;
1684                 struct field *hf, *mf;
1685                 uint32_t offset;
1686
1687                 CHECK_NAME(field->name, EINVAL);
1688
1689                 hf = header_field_parse(p, field->name, &h);
1690                 mf = metadata_field_parse(p, field->name);
1691                 CHECK(hf || mf, EINVAL);
1692
1693                 offset = hf ? hf->offset : mf->offset;
1694
1695                 if (i == 0) {
1696                         is_header = hf ? 1 : 0;
1697                         header = hf ? h : NULL;
1698                         offset_prev = offset;
1699
1700                         continue;
1701                 }
1702
1703                 CHECK((is_header && hf && (h->id == header->id)) ||
1704                       (!is_header && mf), EINVAL);
1705
1706                 CHECK(offset > offset_prev, EINVAL);
1707                 offset_prev = offset;
1708         }
1709
1710         /* Action checks. */
1711         CHECK(params->n_actions, EINVAL);
1712         CHECK(params->action_names, EINVAL);
1713         for (i = 0; i < params->n_actions; i++) {
1714                 const char *action_name = params->action_names[i];
1715                 struct action *a;
1716                 uint32_t action_data_size;
1717
1718                 CHECK(action_name, EINVAL);
1719
1720                 a = action_find(p, action_name);
1721                 CHECK(a, EINVAL);
1722
1723                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
1724                 if (action_data_size > action_data_size_max)
1725                         action_data_size_max = action_data_size;
1726         }
1727
1728         CHECK(params->default_action_name, EINVAL);
1729         for (i = 0; i < p->n_actions; i++)
1730                 if (!strcmp(params->action_names[i],
1731                             params->default_action_name))
1732                         break;
1733         CHECK(i < params->n_actions, EINVAL);
1734         default_action = action_find(p, params->default_action_name);
1735         CHECK((default_action->st && params->default_action_data) ||
1736               !params->default_action_data, EINVAL);
1737
1738         /* Table type checks. */
1739         if (params->n_fields) {
1740                 enum rte_swx_table_match_type match_type;
1741
1742                 match_type = table_match_type_resolve(params->fields,
1743                                                       params->n_fields);
1744                 type = table_type_resolve(p,
1745                                           recommended_table_type_name,
1746                                           match_type);
1747                 CHECK(type, EINVAL);
1748         } else {
1749                 type = NULL;
1750         }
1751
1752         /* Memory allocation. */
1753         t = calloc(1, sizeof(struct table));
1754         CHECK(t, ENOMEM);
1755
1756         t->fields = calloc(params->n_fields, sizeof(struct match_field));
1757         if (!t->fields) {
1758                 free(t);
1759                 CHECK(0, ENOMEM);
1760         }
1761
1762         t->actions = calloc(params->n_actions, sizeof(struct action *));
1763         if (!t->actions) {
1764                 free(t->fields);
1765                 free(t);
1766                 CHECK(0, ENOMEM);
1767         }
1768
1769         if (action_data_size_max) {
1770                 t->default_action_data = calloc(1, action_data_size_max);
1771                 if (!t->default_action_data) {
1772                         free(t->actions);
1773                         free(t->fields);
1774                         free(t);
1775                         CHECK(0, ENOMEM);
1776                 }
1777         }
1778
1779         /* Node initialization. */
1780         strcpy(t->name, name);
1781         if (args && args[0])
1782                 strcpy(t->args, args);
1783         t->type = type;
1784
1785         for (i = 0; i < params->n_fields; i++) {
1786                 struct rte_swx_match_field_params *field = &params->fields[i];
1787                 struct match_field *f = &t->fields[i];
1788
1789                 f->match_type = field->match_type;
1790                 f->field = is_header ?
1791                         header_field_parse(p, field->name, NULL) :
1792                         metadata_field_parse(p, field->name);
1793         }
1794         t->n_fields = params->n_fields;
1795         t->is_header = is_header;
1796         t->header = header;
1797
1798         for (i = 0; i < params->n_actions; i++)
1799                 t->actions[i] = action_find(p, params->action_names[i]);
1800         t->default_action = default_action;
1801         if (default_action->st)
1802                 memcpy(t->default_action_data,
1803                        params->default_action_data,
1804                        default_action->st->n_bits / 8);
1805         t->n_actions = params->n_actions;
1806         t->default_action_is_const = params->default_action_is_const;
1807         t->action_data_size_max = action_data_size_max;
1808
1809         t->size = size;
1810         t->id = p->n_tables;
1811
1812         /* Node add to tailq. */
1813         TAILQ_INSERT_TAIL(&p->tables, t, node);
1814         p->n_tables++;
1815
1816         return 0;
1817 }
1818
1819 static struct rte_swx_table_params *
1820 table_params_get(struct table *table)
1821 {
1822         struct rte_swx_table_params *params;
1823         struct field *first, *last;
1824         uint8_t *key_mask;
1825         uint32_t key_size, key_offset, action_data_size, i;
1826
1827         /* Memory allocation. */
1828         params = calloc(1, sizeof(struct rte_swx_table_params));
1829         if (!params)
1830                 return NULL;
1831
1832         /* Key offset and size. */
1833         first = table->fields[0].field;
1834         last = table->fields[table->n_fields - 1].field;
1835         key_offset = first->offset / 8;
1836         key_size = (last->offset + last->n_bits - first->offset) / 8;
1837
1838         /* Memory allocation. */
1839         key_mask = calloc(1, key_size);
1840         if (!key_mask) {
1841                 free(params);
1842                 return NULL;
1843         }
1844
1845         /* Key mask. */
1846         for (i = 0; i < table->n_fields; i++) {
1847                 struct field *f = table->fields[i].field;
1848                 uint32_t start = (f->offset - first->offset) / 8;
1849                 size_t size = f->n_bits / 8;
1850
1851                 memset(&key_mask[start], 0xFF, size);
1852         }
1853
1854         /* Action data size. */
1855         action_data_size = 0;
1856         for (i = 0; i < table->n_actions; i++) {
1857                 struct action *action = table->actions[i];
1858                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
1859
1860                 if (ads > action_data_size)
1861                         action_data_size = ads;
1862         }
1863
1864         /* Fill in. */
1865         params->match_type = table->type->match_type;
1866         params->key_size = key_size;
1867         params->key_offset = key_offset;
1868         params->key_mask0 = key_mask;
1869         params->action_data_size = action_data_size;
1870         params->n_keys_max = table->size;
1871
1872         return params;
1873 }
1874
1875 static void
1876 table_params_free(struct rte_swx_table_params *params)
1877 {
1878         if (!params)
1879                 return;
1880
1881         free(params->key_mask0);
1882         free(params);
1883 }
1884
1885 static int
1886 table_state_build(struct rte_swx_pipeline *p)
1887 {
1888         struct table *table;
1889
1890         p->table_state = calloc(p->n_tables,
1891                                 sizeof(struct rte_swx_table_state));
1892         CHECK(p->table_state, ENOMEM);
1893
1894         TAILQ_FOREACH(table, &p->tables, node) {
1895                 struct rte_swx_table_state *ts = &p->table_state[table->id];
1896
1897                 if (table->type) {
1898                         struct rte_swx_table_params *params;
1899
1900                         /* ts->obj. */
1901                         params = table_params_get(table);
1902                         CHECK(params, ENOMEM);
1903
1904                         ts->obj = table->type->ops.create(params,
1905                                 NULL,
1906                                 table->args,
1907                                 p->numa_node);
1908
1909                         table_params_free(params);
1910                         CHECK(ts->obj, ENODEV);
1911                 }
1912
1913                 /* ts->default_action_data. */
1914                 if (table->action_data_size_max) {
1915                         ts->default_action_data =
1916                                 malloc(table->action_data_size_max);
1917                         CHECK(ts->default_action_data, ENOMEM);
1918
1919                         memcpy(ts->default_action_data,
1920                                table->default_action_data,
1921                                table->action_data_size_max);
1922                 }
1923
1924                 /* ts->default_action_id. */
1925                 ts->default_action_id = table->default_action->id;
1926         }
1927
1928         return 0;
1929 }
1930
1931 static void
1932 table_state_build_free(struct rte_swx_pipeline *p)
1933 {
1934         uint32_t i;
1935
1936         if (!p->table_state)
1937                 return;
1938
1939         for (i = 0; i < p->n_tables; i++) {
1940                 struct rte_swx_table_state *ts = &p->table_state[i];
1941                 struct table *table = table_find_by_id(p, i);
1942
1943                 /* ts->obj. */
1944                 if (table->type && ts->obj)
1945                         table->type->ops.free(ts->obj);
1946
1947                 /* ts->default_action_data. */
1948                 free(ts->default_action_data);
1949         }
1950
1951         free(p->table_state);
1952         p->table_state = NULL;
1953 }
1954
1955 static void
1956 table_state_free(struct rte_swx_pipeline *p)
1957 {
1958         table_state_build_free(p);
1959 }
1960
1961 static int
1962 table_stub_lkp(void *table __rte_unused,
1963                void *mailbox __rte_unused,
1964                uint8_t **key __rte_unused,
1965                uint64_t *action_id __rte_unused,
1966                uint8_t **action_data __rte_unused,
1967                int *hit)
1968 {
1969         *hit = 0;
1970         return 1; /* DONE. */
1971 }
1972
1973 static int
1974 table_build(struct rte_swx_pipeline *p)
1975 {
1976         uint32_t i;
1977
1978         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1979                 struct thread *t = &p->threads[i];
1980                 struct table *table;
1981
1982                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
1983                 CHECK(t->tables, ENOMEM);
1984
1985                 TAILQ_FOREACH(table, &p->tables, node) {
1986                         struct table_runtime *r = &t->tables[table->id];
1987
1988                         if (table->type) {
1989                                 uint64_t size;
1990
1991                                 size = table->type->ops.mailbox_size_get();
1992
1993                                 /* r->func. */
1994                                 r->func = table->type->ops.lkp;
1995
1996                                 /* r->mailbox. */
1997                                 if (size) {
1998                                         r->mailbox = calloc(1, size);
1999                                         CHECK(r->mailbox, ENOMEM);
2000                                 }
2001
2002                                 /* r->key. */
2003                                 r->key = table->is_header ?
2004                                         &t->structs[table->header->struct_id] :
2005                                         &t->structs[p->metadata_struct_id];
2006                         } else {
2007                                 r->func = table_stub_lkp;
2008                         }
2009                 }
2010         }
2011
2012         return 0;
2013 }
2014
2015 static void
2016 table_build_free(struct rte_swx_pipeline *p)
2017 {
2018         uint32_t i;
2019
2020         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2021                 struct thread *t = &p->threads[i];
2022                 uint32_t j;
2023
2024                 if (!t->tables)
2025                         continue;
2026
2027                 for (j = 0; j < p->n_tables; j++) {
2028                         struct table_runtime *r = &t->tables[j];
2029
2030                         free(r->mailbox);
2031                 }
2032
2033                 free(t->tables);
2034                 t->tables = NULL;
2035         }
2036 }
2037
2038 static void
2039 table_free(struct rte_swx_pipeline *p)
2040 {
2041         table_build_free(p);
2042
2043         /* Tables. */
2044         for ( ; ; ) {
2045                 struct table *elem;
2046
2047                 elem = TAILQ_FIRST(&p->tables);
2048                 if (!elem)
2049                         break;
2050
2051                 TAILQ_REMOVE(&p->tables, elem, node);
2052                 free(elem->fields);
2053                 free(elem->actions);
2054                 free(elem->default_action_data);
2055                 free(elem);
2056         }
2057
2058         /* Table types. */
2059         for ( ; ; ) {
2060                 struct table_type *elem;
2061
2062                 elem = TAILQ_FIRST(&p->table_types);
2063                 if (!elem)
2064                         break;
2065
2066                 TAILQ_REMOVE(&p->table_types, elem, node);
2067                 free(elem);
2068         }
2069 }
2070
2071 /*
2072  * Pipeline.
2073  */
2074 int
2075 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
2076 {
2077         struct rte_swx_pipeline *pipeline;
2078
2079         /* Check input parameters. */
2080         CHECK(p, EINVAL);
2081
2082         /* Memory allocation. */
2083         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
2084         CHECK(pipeline, ENOMEM);
2085
2086         /* Initialization. */
2087         TAILQ_INIT(&pipeline->struct_types);
2088         TAILQ_INIT(&pipeline->port_in_types);
2089         TAILQ_INIT(&pipeline->ports_in);
2090         TAILQ_INIT(&pipeline->port_out_types);
2091         TAILQ_INIT(&pipeline->ports_out);
2092         TAILQ_INIT(&pipeline->extern_types);
2093         TAILQ_INIT(&pipeline->extern_objs);
2094         TAILQ_INIT(&pipeline->extern_funcs);
2095         TAILQ_INIT(&pipeline->headers);
2096         TAILQ_INIT(&pipeline->actions);
2097         TAILQ_INIT(&pipeline->table_types);
2098         TAILQ_INIT(&pipeline->tables);
2099
2100         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
2101         pipeline->numa_node = numa_node;
2102
2103         *p = pipeline;
2104         return 0;
2105 }
2106
2107 void
2108 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
2109 {
2110         if (!p)
2111                 return;
2112
2113         table_state_free(p);
2114         table_free(p);
2115         action_free(p);
2116         metadata_free(p);
2117         header_free(p);
2118         extern_func_free(p);
2119         extern_obj_free(p);
2120         port_out_free(p);
2121         port_in_free(p);
2122         struct_free(p);
2123
2124         free(p);
2125 }
2126
2127 int
2128 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
2129 {
2130         int status;
2131
2132         CHECK(p, EINVAL);
2133         CHECK(p->build_done == 0, EEXIST);
2134
2135         status = port_in_build(p);
2136         if (status)
2137                 goto error;
2138
2139         status = port_out_build(p);
2140         if (status)
2141                 goto error;
2142
2143         status = struct_build(p);
2144         if (status)
2145                 goto error;
2146
2147         status = extern_obj_build(p);
2148         if (status)
2149                 goto error;
2150
2151         status = extern_func_build(p);
2152         if (status)
2153                 goto error;
2154
2155         status = header_build(p);
2156         if (status)
2157                 goto error;
2158
2159         status = metadata_build(p);
2160         if (status)
2161                 goto error;
2162
2163         status = action_build(p);
2164         if (status)
2165                 goto error;
2166
2167         status = table_build(p);
2168         if (status)
2169                 goto error;
2170
2171         status = table_state_build(p);
2172         if (status)
2173                 goto error;
2174
2175         p->build_done = 1;
2176         return 0;
2177
2178 error:
2179         table_state_build_free(p);
2180         table_build_free(p);
2181         action_build_free(p);
2182         metadata_build_free(p);
2183         header_build_free(p);
2184         extern_func_build_free(p);
2185         extern_obj_build_free(p);
2186         port_out_build_free(p);
2187         port_in_build_free(p);
2188         struct_build_free(p);
2189
2190         return status;
2191 }
2192
2193 /*
2194  * Control.
2195  */
2196 int
2197 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
2198                                  struct rte_swx_table_state **table_state)
2199 {
2200         if (!p || !table_state || !p->build_done)
2201                 return -EINVAL;
2202
2203         *table_state = p->table_state;
2204         return 0;
2205 }
2206
2207 int
2208 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
2209                                  struct rte_swx_table_state *table_state)
2210 {
2211         if (!p || !table_state || !p->build_done)
2212                 return -EINVAL;
2213
2214         p->table_state = table_state;
2215         return 0;
2216 }