lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9
  10 #include <rte_common.h>
  11
  12 #include "rte_swx_pipeline.h"
  13
  14 #define CHECK(condition, err_code)                                             \
  15 do {                                                                           \
  16         if (!(condition))                                                      \
  17                 return -(err_code);                                            \
  18 } while (0)
  19
  20 #define CHECK_NAME(name, err_code)                                             \
  21         CHECK((name) && (name)[0], err_code)
  22
  23 /*
  24  * Struct.
  25  */
  26 struct field {
  27         char name[RTE_SWX_NAME_SIZE];
  28         uint32_t n_bits;
  29         uint32_t offset;
  30 };
  31
  32 struct struct_type {
  33         TAILQ_ENTRY(struct_type) node;
  34         char name[RTE_SWX_NAME_SIZE];
  35         struct field *fields;
  36         uint32_t n_fields;
  37         uint32_t n_bits;
  38 };
  39
  40 TAILQ_HEAD(struct_type_tailq, struct_type);
  41
  42 /*
  43  * Input port.
  44  */
  45 struct port_in_type {
  46         TAILQ_ENTRY(port_in_type) node;
  47         char name[RTE_SWX_NAME_SIZE];
  48         struct rte_swx_port_in_ops ops;
  49 };
  50
  51 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  52
  53 struct port_in {
  54         TAILQ_ENTRY(port_in) node;
  55         struct port_in_type *type;
  56         void *obj;
  57         uint32_t id;
  58 };
  59
  60 TAILQ_HEAD(port_in_tailq, port_in);
  61
  62 struct port_in_runtime {
  63         rte_swx_port_in_pkt_rx_t pkt_rx;
  64         void *obj;
  65 };
  66
  67 /*
  68  * Output port.
  69  */
  70 struct port_out_type {
  71         TAILQ_ENTRY(port_out_type) node;
  72         char name[RTE_SWX_NAME_SIZE];
  73         struct rte_swx_port_out_ops ops;
  74 };
  75
  76 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  77
  78 struct port_out {
  79         TAILQ_ENTRY(port_out) node;
  80         struct port_out_type *type;
  81         void *obj;
  82         uint32_t id;
  83 };
  84
  85 TAILQ_HEAD(port_out_tailq, port_out);
  86
  87 struct port_out_runtime {
  88         rte_swx_port_out_pkt_tx_t pkt_tx;
  89         rte_swx_port_out_flush_t flush;
  90         void *obj;
  91 };
  92
  93 /*
  94  * Extern object.
  95  */
  96 struct extern_type_member_func {
  97         TAILQ_ENTRY(extern_type_member_func) node;
  98         char name[RTE_SWX_NAME_SIZE];
  99         rte_swx_extern_type_member_func_t func;
 100         uint32_t id;
 101 };
 102
 103 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 104
 105 struct extern_type {
 106         TAILQ_ENTRY(extern_type) node;
 107         char name[RTE_SWX_NAME_SIZE];
 108         struct struct_type *mailbox_struct_type;
 109         rte_swx_extern_type_constructor_t constructor;
 110         rte_swx_extern_type_destructor_t destructor;
 111         struct extern_type_member_func_tailq funcs;
 112         uint32_t n_funcs;
 113 };
 114
 115 TAILQ_HEAD(extern_type_tailq, extern_type);
 116
 117 struct extern_obj {
 118         TAILQ_ENTRY(extern_obj) node;
 119         char name[RTE_SWX_NAME_SIZE];
 120         struct extern_type *type;
 121         void *obj;
 122         uint32_t struct_id;
 123         uint32_t id;
 124 };
 125
 126 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 127
 128 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 129 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 130 #endif
 131
 132 struct extern_obj_runtime {
 133         void *obj;
 134         uint8_t *mailbox;
 135         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 136 };
 137
 138 /*
 139  * Extern function.
 140  */
 141 struct extern_func {
 142         TAILQ_ENTRY(extern_func) node;
 143         char name[RTE_SWX_NAME_SIZE];
 144         struct struct_type *mailbox_struct_type;
 145         rte_swx_extern_func_t func;
 146         uint32_t struct_id;
 147         uint32_t id;
 148 };
 149
 150 TAILQ_HEAD(extern_func_tailq, extern_func);
 151
 152 struct extern_func_runtime {
 153         uint8_t *mailbox;
 154         rte_swx_extern_func_t func;
 155 };
 156
 157 /*
 158  * Header.
 159  */
 160 struct header {
 161         TAILQ_ENTRY(header) node;
 162         char name[RTE_SWX_NAME_SIZE];
 163         struct struct_type *st;
 164         uint32_t struct_id;
 165         uint32_t id;
 166 };
 167
 168 TAILQ_HEAD(header_tailq, header);
 169
 170 struct header_runtime {
 171         uint8_t *ptr0;
 172 };
 173
 174 struct header_out_runtime {
 175         uint8_t *ptr0;
 176         uint8_t *ptr;
 177         uint32_t n_bytes;
 178 };
 179
 180 /*
 181  * Pipeline.
 182  */
 183 struct thread {
 184         /* Structures. */
 185         uint8_t **structs;
 186
 187         /* Packet headers. */
 188         struct header_runtime *headers; /* Extracted or generated headers. */
 189         struct header_out_runtime *headers_out; /* Emitted headers. */
 190         uint8_t *header_storage;
 191         uint8_t *header_out_storage;
 192         uint64_t valid_headers;
 193         uint32_t n_headers_out;
 194
 195         /* Packet meta-data. */
 196         uint8_t *metadata;
 197
 198         /* Extern objects and functions. */
 199         struct extern_obj_runtime *extern_objs;
 200         struct extern_func_runtime *extern_funcs;
 201 };
 202
 203 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 204 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 205 #endif
 206
 207 struct rte_swx_pipeline {
 208         struct struct_type_tailq struct_types;
 209         struct port_in_type_tailq port_in_types;
 210         struct port_in_tailq ports_in;
 211         struct port_out_type_tailq port_out_types;
 212         struct port_out_tailq ports_out;
 213         struct extern_type_tailq extern_types;
 214         struct extern_obj_tailq extern_objs;
 215         struct extern_func_tailq extern_funcs;
 216         struct header_tailq headers;
 217         struct struct_type *metadata_st;
 218         uint32_t metadata_struct_id;
 219
 220         struct port_in_runtime *in;
 221         struct port_out_runtime *out;
 222         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 223
 224         uint32_t n_structs;
 225         uint32_t n_ports_in;
 226         uint32_t n_ports_out;
 227         uint32_t n_extern_objs;
 228         uint32_t n_extern_funcs;
 229         uint32_t n_headers;
 230         int build_done;
 231         int numa_node;
 232 };
 233
 234 /*
 235  * Struct.
 236  */
 237 static struct struct_type *
 238 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 239 {
 240         struct struct_type *elem;
 241
 242         TAILQ_FOREACH(elem, &p->struct_types, node)
 243                 if (strcmp(elem->name, name) == 0)
 244                         return elem;
 245
 246         return NULL;
 247 }
 248
 249 int
 250 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 251                                       const char *name,
 252                                       struct rte_swx_field_params *fields,
 253                                       uint32_t n_fields)
 254 {
 255         struct struct_type *st;
 256         uint32_t i;
 257
 258         CHECK(p, EINVAL);
 259         CHECK_NAME(name, EINVAL);
 260         CHECK(fields, EINVAL);
 261         CHECK(n_fields, EINVAL);
 262
 263         for (i = 0; i < n_fields; i++) {
 264                 struct rte_swx_field_params *f = &fields[i];
 265                 uint32_t j;
 266
 267                 CHECK_NAME(f->name, EINVAL);
 268                 CHECK(f->n_bits, EINVAL);
 269                 CHECK(f->n_bits <= 64, EINVAL);
 270                 CHECK((f->n_bits & 7) == 0, EINVAL);
 271
 272                 for (j = 0; j < i; j++) {
 273                         struct rte_swx_field_params *f_prev = &fields[j];
 274
 275                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 276                 }
 277         }
 278
 279         CHECK(!struct_type_find(p, name), EEXIST);
 280
 281         /* Node allocation. */
 282         st = calloc(1, sizeof(struct struct_type));
 283         CHECK(st, ENOMEM);
 284
 285         st->fields = calloc(n_fields, sizeof(struct field));
 286         if (!st->fields) {
 287                 free(st);
 288                 CHECK(0, ENOMEM);
 289         }
 290
 291         /* Node initialization. */
 292         strcpy(st->name, name);
 293         for (i = 0; i < n_fields; i++) {
 294                 struct field *dst = &st->fields[i];
 295                 struct rte_swx_field_params *src = &fields[i];
 296
 297                 strcpy(dst->name, src->name);
 298                 dst->n_bits = src->n_bits;
 299                 dst->offset = st->n_bits;
 300
 301                 st->n_bits += src->n_bits;
 302         }
 303         st->n_fields = n_fields;
 304
 305         /* Node add to tailq. */
 306         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 307
 308         return 0;
 309 }
 310
 311 static int
 312 struct_build(struct rte_swx_pipeline *p)
 313 {
 314         uint32_t i;
 315
 316         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 317                 struct thread *t = &p->threads[i];
 318
 319                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 320                 CHECK(t->structs, ENOMEM);
 321         }
 322
 323         return 0;
 324 }
 325
 326 static void
 327 struct_build_free(struct rte_swx_pipeline *p)
 328 {
 329         uint32_t i;
 330
 331         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 332                 struct thread *t = &p->threads[i];
 333
 334                 free(t->structs);
 335                 t->structs = NULL;
 336         }
 337 }
 338
 339 static void
 340 struct_free(struct rte_swx_pipeline *p)
 341 {
 342         struct_build_free(p);
 343
 344         /* Struct types. */
 345         for ( ; ; ) {
 346                 struct struct_type *elem;
 347
 348                 elem = TAILQ_FIRST(&p->struct_types);
 349                 if (!elem)
 350                         break;
 351
 352                 TAILQ_REMOVE(&p->struct_types, elem, node);
 353                 free(elem->fields);
 354                 free(elem);
 355         }
 356 }
 357
 358 /*
 359  * Input port.
 360  */
 361 static struct port_in_type *
 362 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 363 {
 364         struct port_in_type *elem;
 365
 366         if (!name)
 367                 return NULL;
 368
 369         TAILQ_FOREACH(elem, &p->port_in_types, node)
 370                 if (strcmp(elem->name, name) == 0)
 371                         return elem;
 372
 373         return NULL;
 374 }
 375
 376 int
 377 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 378                                        const char *name,
 379                                        struct rte_swx_port_in_ops *ops)
 380 {
 381         struct port_in_type *elem;
 382
 383         CHECK(p, EINVAL);
 384         CHECK_NAME(name, EINVAL);
 385         CHECK(ops, EINVAL);
 386         CHECK(ops->create, EINVAL);
 387         CHECK(ops->free, EINVAL);
 388         CHECK(ops->pkt_rx, EINVAL);
 389         CHECK(ops->stats_read, EINVAL);
 390
 391         CHECK(!port_in_type_find(p, name), EEXIST);
 392
 393         /* Node allocation. */
 394         elem = calloc(1, sizeof(struct port_in_type));
 395         CHECK(elem, ENOMEM);
 396
 397         /* Node initialization. */
 398         strcpy(elem->name, name);
 399         memcpy(&elem->ops, ops, sizeof(*ops));
 400
 401         /* Node add to tailq. */
 402         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 403
 404         return 0;
 405 }
 406
 407 static struct port_in *
 408 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 409 {
 410         struct port_in *port;
 411
 412         TAILQ_FOREACH(port, &p->ports_in, node)
 413                 if (port->id == port_id)
 414                         return port;
 415
 416         return NULL;
 417 }
 418
 419 int
 420 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 421                                 uint32_t port_id,
 422                                 const char *port_type_name,
 423                                 void *args)
 424 {
 425         struct port_in_type *type = NULL;
 426         struct port_in *port = NULL;
 427         void *obj = NULL;
 428
 429         CHECK(p, EINVAL);
 430
 431         CHECK(!port_in_find(p, port_id), EINVAL);
 432
 433         CHECK_NAME(port_type_name, EINVAL);
 434         type = port_in_type_find(p, port_type_name);
 435         CHECK(type, EINVAL);
 436
 437         obj = type->ops.create(args);
 438         CHECK(obj, ENODEV);
 439
 440         /* Node allocation. */
 441         port = calloc(1, sizeof(struct port_in));
 442         CHECK(port, ENOMEM);
 443
 444         /* Node initialization. */
 445         port->type = type;
 446         port->obj = obj;
 447         port->id = port_id;
 448
 449         /* Node add to tailq. */
 450         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 451         if (p->n_ports_in < port_id + 1)
 452                 p->n_ports_in = port_id + 1;
 453
 454         return 0;
 455 }
 456
 457 static int
 458 port_in_build(struct rte_swx_pipeline *p)
 459 {
 460         struct port_in *port;
 461         uint32_t i;
 462
 463         CHECK(p->n_ports_in, EINVAL);
 464         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 465
 466         for (i = 0; i < p->n_ports_in; i++)
 467                 CHECK(port_in_find(p, i), EINVAL);
 468
 469         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 470         CHECK(p->in, ENOMEM);
 471
 472         TAILQ_FOREACH(port, &p->ports_in, node) {
 473                 struct port_in_runtime *in = &p->in[port->id];
 474
 475                 in->pkt_rx = port->type->ops.pkt_rx;
 476                 in->obj = port->obj;
 477         }
 478
 479         return 0;
 480 }
 481
 482 static void
 483 port_in_build_free(struct rte_swx_pipeline *p)
 484 {
 485         free(p->in);
 486         p->in = NULL;
 487 }
 488
 489 static void
 490 port_in_free(struct rte_swx_pipeline *p)
 491 {
 492         port_in_build_free(p);
 493
 494         /* Input ports. */
 495         for ( ; ; ) {
 496                 struct port_in *port;
 497
 498                 port = TAILQ_FIRST(&p->ports_in);
 499                 if (!port)
 500                         break;
 501
 502                 TAILQ_REMOVE(&p->ports_in, port, node);
 503                 port->type->ops.free(port->obj);
 504                 free(port);
 505         }
 506
 507         /* Input port types. */
 508         for ( ; ; ) {
 509                 struct port_in_type *elem;
 510
 511                 elem = TAILQ_FIRST(&p->port_in_types);
 512                 if (!elem)
 513                         break;
 514
 515                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 516                 free(elem);
 517         }
 518 }
 519
 520 /*
 521  * Output port.
 522  */
 523 static struct port_out_type *
 524 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 525 {
 526         struct port_out_type *elem;
 527
 528         if (!name)
 529                 return NULL;
 530
 531         TAILQ_FOREACH(elem, &p->port_out_types, node)
 532                 if (!strcmp(elem->name, name))
 533                         return elem;
 534
 535         return NULL;
 536 }
 537
 538 int
 539 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 540                                         const char *name,
 541                                         struct rte_swx_port_out_ops *ops)
 542 {
 543         struct port_out_type *elem;
 544
 545         CHECK(p, EINVAL);
 546         CHECK_NAME(name, EINVAL);
 547         CHECK(ops, EINVAL);
 548         CHECK(ops->create, EINVAL);
 549         CHECK(ops->free, EINVAL);
 550         CHECK(ops->pkt_tx, EINVAL);
 551         CHECK(ops->stats_read, EINVAL);
 552
 553         CHECK(!port_out_type_find(p, name), EEXIST);
 554
 555         /* Node allocation. */
 556         elem = calloc(1, sizeof(struct port_out_type));
 557         CHECK(elem, ENOMEM);
 558
 559         /* Node initialization. */
 560         strcpy(elem->name, name);
 561         memcpy(&elem->ops, ops, sizeof(*ops));
 562
 563         /* Node add to tailq. */
 564         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 565
 566         return 0;
 567 }
 568
 569 static struct port_out *
 570 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 571 {
 572         struct port_out *port;
 573
 574         TAILQ_FOREACH(port, &p->ports_out, node)
 575                 if (port->id == port_id)
 576                         return port;
 577
 578         return NULL;
 579 }
 580
 581 int
 582 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 583                                  uint32_t port_id,
 584                                  const char *port_type_name,
 585                                  void *args)
 586 {
 587         struct port_out_type *type = NULL;
 588         struct port_out *port = NULL;
 589         void *obj = NULL;
 590
 591         CHECK(p, EINVAL);
 592
 593         CHECK(!port_out_find(p, port_id), EINVAL);
 594
 595         CHECK_NAME(port_type_name, EINVAL);
 596         type = port_out_type_find(p, port_type_name);
 597         CHECK(type, EINVAL);
 598
 599         obj = type->ops.create(args);
 600         CHECK(obj, ENODEV);
 601
 602         /* Node allocation. */
 603         port = calloc(1, sizeof(struct port_out));
 604         CHECK(port, ENOMEM);
 605
 606         /* Node initialization. */
 607         port->type = type;
 608         port->obj = obj;
 609         port->id = port_id;
 610
 611         /* Node add to tailq. */
 612         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 613         if (p->n_ports_out < port_id + 1)
 614                 p->n_ports_out = port_id + 1;
 615
 616         return 0;
 617 }
 618
 619 static int
 620 port_out_build(struct rte_swx_pipeline *p)
 621 {
 622         struct port_out *port;
 623         uint32_t i;
 624
 625         CHECK(p->n_ports_out, EINVAL);
 626
 627         for (i = 0; i < p->n_ports_out; i++)
 628                 CHECK(port_out_find(p, i), EINVAL);
 629
 630         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 631         CHECK(p->out, ENOMEM);
 632
 633         TAILQ_FOREACH(port, &p->ports_out, node) {
 634                 struct port_out_runtime *out = &p->out[port->id];
 635
 636                 out->pkt_tx = port->type->ops.pkt_tx;
 637                 out->flush = port->type->ops.flush;
 638                 out->obj = port->obj;
 639         }
 640
 641         return 0;
 642 }
 643
 644 static void
 645 port_out_build_free(struct rte_swx_pipeline *p)
 646 {
 647         free(p->out);
 648         p->out = NULL;
 649 }
 650
 651 static void
 652 port_out_free(struct rte_swx_pipeline *p)
 653 {
 654         port_out_build_free(p);
 655
 656         /* Output ports. */
 657         for ( ; ; ) {
 658                 struct port_out *port;
 659
 660                 port = TAILQ_FIRST(&p->ports_out);
 661                 if (!port)
 662                         break;
 663
 664                 TAILQ_REMOVE(&p->ports_out, port, node);
 665                 port->type->ops.free(port->obj);
 666                 free(port);
 667         }
 668
 669         /* Output port types. */
 670         for ( ; ; ) {
 671                 struct port_out_type *elem;
 672
 673                 elem = TAILQ_FIRST(&p->port_out_types);
 674                 if (!elem)
 675                         break;
 676
 677                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 678                 free(elem);
 679         }
 680 }
 681
 682 /*
 683  * Extern object.
 684  */
 685 static struct extern_type *
 686 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 687 {
 688         struct extern_type *elem;
 689
 690         TAILQ_FOREACH(elem, &p->extern_types, node)
 691                 if (strcmp(elem->name, name) == 0)
 692                         return elem;
 693
 694         return NULL;
 695 }
 696
 697 static struct extern_type_member_func *
 698 extern_type_member_func_find(struct extern_type *type, const char *name)
 699 {
 700         struct extern_type_member_func *elem;
 701
 702         TAILQ_FOREACH(elem, &type->funcs, node)
 703                 if (strcmp(elem->name, name) == 0)
 704                         return elem;
 705
 706         return NULL;
 707 }
 708
 709 static struct extern_obj *
 710 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 711 {
 712         struct extern_obj *elem;
 713
 714         TAILQ_FOREACH(elem, &p->extern_objs, node)
 715                 if (strcmp(elem->name, name) == 0)
 716                         return elem;
 717
 718         return NULL;
 719 }
 720
 721 int
 722 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 723         const char *name,
 724         const char *mailbox_struct_type_name,
 725         rte_swx_extern_type_constructor_t constructor,
 726         rte_swx_extern_type_destructor_t destructor)
 727 {
 728         struct extern_type *elem;
 729         struct struct_type *mailbox_struct_type;
 730
 731         CHECK(p, EINVAL);
 732
 733         CHECK_NAME(name, EINVAL);
 734         CHECK(!extern_type_find(p, name), EEXIST);
 735
 736         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 737         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 738         CHECK(mailbox_struct_type, EINVAL);
 739
 740         CHECK(constructor, EINVAL);
 741         CHECK(destructor, EINVAL);
 742
 743         /* Node allocation. */
 744         elem = calloc(1, sizeof(struct extern_type));
 745         CHECK(elem, ENOMEM);
 746
 747         /* Node initialization. */
 748         strcpy(elem->name, name);
 749         elem->mailbox_struct_type = mailbox_struct_type;
 750         elem->constructor = constructor;
 751         elem->destructor = destructor;
 752         TAILQ_INIT(&elem->funcs);
 753
 754         /* Node add to tailq. */
 755         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 756
 757         return 0;
 758 }
 759
 760 int
 761 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 762         const char *extern_type_name,
 763         const char *name,
 764         rte_swx_extern_type_member_func_t member_func)
 765 {
 766         struct extern_type *type;
 767         struct extern_type_member_func *type_member;
 768
 769         CHECK(p, EINVAL);
 770
 771         CHECK(extern_type_name, EINVAL);
 772         type = extern_type_find(p, extern_type_name);
 773         CHECK(type, EINVAL);
 774         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 775
 776         CHECK(name, EINVAL);
 777         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 778
 779         CHECK(member_func, EINVAL);
 780
 781         /* Node allocation. */
 782         type_member = calloc(1, sizeof(struct extern_type_member_func));
 783         CHECK(type_member, ENOMEM);
 784
 785         /* Node initialization. */
 786         strcpy(type_member->name, name);
 787         type_member->func = member_func;
 788         type_member->id = type->n_funcs;
 789
 790         /* Node add to tailq. */
 791         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
 792         type->n_funcs++;
 793
 794         return 0;
 795 }
 796
 797 int
 798 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
 799                                       const char *extern_type_name,
 800                                       const char *name,
 801                                       const char *args)
 802 {
 803         struct extern_type *type;
 804         struct extern_obj *obj;
 805         void *obj_handle;
 806
 807         CHECK(p, EINVAL);
 808
 809         CHECK_NAME(extern_type_name, EINVAL);
 810         type = extern_type_find(p, extern_type_name);
 811         CHECK(type, EINVAL);
 812
 813         CHECK_NAME(name, EINVAL);
 814         CHECK(!extern_obj_find(p, name), EEXIST);
 815
 816         /* Node allocation. */
 817         obj = calloc(1, sizeof(struct extern_obj));
 818         CHECK(obj, ENOMEM);
 819
 820         /* Object construction. */
 821         obj_handle = type->constructor(args);
 822         if (!obj_handle) {
 823                 free(obj);
 824                 CHECK(0, ENODEV);
 825         }
 826
 827         /* Node initialization. */
 828         strcpy(obj->name, name);
 829         obj->type = type;
 830         obj->obj = obj_handle;
 831         obj->struct_id = p->n_structs;
 832         obj->id = p->n_extern_objs;
 833
 834         /* Node add to tailq. */
 835         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
 836         p->n_extern_objs++;
 837         p->n_structs++;
 838
 839         return 0;
 840 }
 841
 842 static int
 843 extern_obj_build(struct rte_swx_pipeline *p)
 844 {
 845         uint32_t i;
 846
 847         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 848                 struct thread *t = &p->threads[i];
 849                 struct extern_obj *obj;
 850
 851                 t->extern_objs = calloc(p->n_extern_objs,
 852                                         sizeof(struct extern_obj_runtime));
 853                 CHECK(t->extern_objs, ENOMEM);
 854
 855                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
 856                         struct extern_obj_runtime *r =
 857                                 &t->extern_objs[obj->id];
 858                         struct extern_type_member_func *func;
 859                         uint32_t mailbox_size =
 860                                 obj->type->mailbox_struct_type->n_bits / 8;
 861
 862                         r->obj = obj->obj;
 863
 864                         r->mailbox = calloc(1, mailbox_size);
 865                         CHECK(r->mailbox, ENOMEM);
 866
 867                         TAILQ_FOREACH(func, &obj->type->funcs, node)
 868                                 r->funcs[func->id] = func->func;
 869
 870                         t->structs[obj->struct_id] = r->mailbox;
 871                 }
 872         }
 873
 874         return 0;
 875 }
 876
 877 static void
 878 extern_obj_build_free(struct rte_swx_pipeline *p)
 879 {
 880         uint32_t i;
 881
 882         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 883                 struct thread *t = &p->threads[i];
 884                 uint32_t j;
 885
 886                 if (!t->extern_objs)
 887                         continue;
 888
 889                 for (j = 0; j < p->n_extern_objs; j++) {
 890                         struct extern_obj_runtime *r = &t->extern_objs[j];
 891
 892                         free(r->mailbox);
 893                 }
 894
 895                 free(t->extern_objs);
 896                 t->extern_objs = NULL;
 897         }
 898 }
 899
 900 static void
 901 extern_obj_free(struct rte_swx_pipeline *p)
 902 {
 903         extern_obj_build_free(p);
 904
 905         /* Extern objects. */
 906         for ( ; ; ) {
 907                 struct extern_obj *elem;
 908
 909                 elem = TAILQ_FIRST(&p->extern_objs);
 910                 if (!elem)
 911                         break;
 912
 913                 TAILQ_REMOVE(&p->extern_objs, elem, node);
 914                 if (elem->obj)
 915                         elem->type->destructor(elem->obj);
 916                 free(elem);
 917         }
 918
 919         /* Extern types. */
 920         for ( ; ; ) {
 921                 struct extern_type *elem;
 922
 923                 elem = TAILQ_FIRST(&p->extern_types);
 924                 if (!elem)
 925                         break;
 926
 927                 TAILQ_REMOVE(&p->extern_types, elem, node);
 928
 929                 for ( ; ; ) {
 930                         struct extern_type_member_func *func;
 931
 932                         func = TAILQ_FIRST(&elem->funcs);
 933                         if (!func)
 934                                 break;
 935
 936                         TAILQ_REMOVE(&elem->funcs, func, node);
 937                         free(func);
 938                 }
 939
 940                 free(elem);
 941         }
 942 }
 943
 944 /*
 945  * Extern function.
 946  */
 947 static struct extern_func *
 948 extern_func_find(struct rte_swx_pipeline *p, const char *name)
 949 {
 950         struct extern_func *elem;
 951
 952         TAILQ_FOREACH(elem, &p->extern_funcs, node)
 953                 if (strcmp(elem->name, name) == 0)
 954                         return elem;
 955
 956         return NULL;
 957 }
 958
 959 int
 960 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
 961                                       const char *name,
 962                                       const char *mailbox_struct_type_name,
 963                                       rte_swx_extern_func_t func)
 964 {
 965         struct extern_func *f;
 966         struct struct_type *mailbox_struct_type;
 967
 968         CHECK(p, EINVAL);
 969
 970         CHECK_NAME(name, EINVAL);
 971         CHECK(!extern_func_find(p, name), EEXIST);
 972
 973         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 974         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 975         CHECK(mailbox_struct_type, EINVAL);
 976
 977         CHECK(func, EINVAL);
 978
 979         /* Node allocation. */
 980         f = calloc(1, sizeof(struct extern_func));
 981         CHECK(func, ENOMEM);
 982
 983         /* Node initialization. */
 984         strcpy(f->name, name);
 985         f->mailbox_struct_type = mailbox_struct_type;
 986         f->func = func;
 987         f->struct_id = p->n_structs;
 988         f->id = p->n_extern_funcs;
 989
 990         /* Node add to tailq. */
 991         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
 992         p->n_extern_funcs++;
 993         p->n_structs++;
 994
 995         return 0;
 996 }
 997
 998 static int
 999 extern_func_build(struct rte_swx_pipeline *p)
1000 {
1001         uint32_t i;
1002
1003         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1004                 struct thread *t = &p->threads[i];
1005                 struct extern_func *func;
1006
1007                 /* Memory allocation. */
1008                 t->extern_funcs = calloc(p->n_extern_funcs,
1009                                          sizeof(struct extern_func_runtime));
1010                 CHECK(t->extern_funcs, ENOMEM);
1011
1012                 /* Extern function. */
1013                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1014                         struct extern_func_runtime *r =
1015                                 &t->extern_funcs[func->id];
1016                         uint32_t mailbox_size =
1017                                 func->mailbox_struct_type->n_bits / 8;
1018
1019                         r->func = func->func;
1020
1021                         r->mailbox = calloc(1, mailbox_size);
1022                         CHECK(r->mailbox, ENOMEM);
1023
1024                         t->structs[func->struct_id] = r->mailbox;
1025                 }
1026         }
1027
1028         return 0;
1029 }
1030
1031 static void
1032 extern_func_build_free(struct rte_swx_pipeline *p)
1033 {
1034         uint32_t i;
1035
1036         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1037                 struct thread *t = &p->threads[i];
1038                 uint32_t j;
1039
1040                 if (!t->extern_funcs)
1041                         continue;
1042
1043                 for (j = 0; j < p->n_extern_funcs; j++) {
1044                         struct extern_func_runtime *r = &t->extern_funcs[j];
1045
1046                         free(r->mailbox);
1047                 }
1048
1049                 free(t->extern_funcs);
1050                 t->extern_funcs = NULL;
1051         }
1052 }
1053
1054 static void
1055 extern_func_free(struct rte_swx_pipeline *p)
1056 {
1057         extern_func_build_free(p);
1058
1059         for ( ; ; ) {
1060                 struct extern_func *elem;
1061
1062                 elem = TAILQ_FIRST(&p->extern_funcs);
1063                 if (!elem)
1064                         break;
1065
1066                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1067                 free(elem);
1068         }
1069 }
1070
1071 /*
1072  * Header.
1073  */
1074 static struct header *
1075 header_find(struct rte_swx_pipeline *p, const char *name)
1076 {
1077         struct header *elem;
1078
1079         TAILQ_FOREACH(elem, &p->headers, node)
1080                 if (strcmp(elem->name, name) == 0)
1081                         return elem;
1082
1083         return NULL;
1084 }
1085
1086 int
1087 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1088                                         const char *name,
1089                                         const char *struct_type_name)
1090 {
1091         struct struct_type *st;
1092         struct header *h;
1093         size_t n_headers_max;
1094
1095         CHECK(p, EINVAL);
1096         CHECK_NAME(name, EINVAL);
1097         CHECK_NAME(struct_type_name, EINVAL);
1098
1099         CHECK(!header_find(p, name), EEXIST);
1100
1101         st = struct_type_find(p, struct_type_name);
1102         CHECK(st, EINVAL);
1103
1104         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1105         CHECK(p->n_headers < n_headers_max, ENOSPC);
1106
1107         /* Node allocation. */
1108         h = calloc(1, sizeof(struct header));
1109         CHECK(h, ENOMEM);
1110
1111         /* Node initialization. */
1112         strcpy(h->name, name);
1113         h->st = st;
1114         h->struct_id = p->n_structs;
1115         h->id = p->n_headers;
1116
1117         /* Node add to tailq. */
1118         TAILQ_INSERT_TAIL(&p->headers, h, node);
1119         p->n_headers++;
1120         p->n_structs++;
1121
1122         return 0;
1123 }
1124
1125 static int
1126 header_build(struct rte_swx_pipeline *p)
1127 {
1128         struct header *h;
1129         uint32_t n_bytes = 0, i;
1130
1131         TAILQ_FOREACH(h, &p->headers, node) {
1132                 n_bytes += h->st->n_bits / 8;
1133         }
1134
1135         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1136                 struct thread *t = &p->threads[i];
1137                 uint32_t offset = 0;
1138
1139                 t->headers = calloc(p->n_headers,
1140                                     sizeof(struct header_runtime));
1141                 CHECK(t->headers, ENOMEM);
1142
1143                 t->headers_out = calloc(p->n_headers,
1144                                         sizeof(struct header_out_runtime));
1145                 CHECK(t->headers_out, ENOMEM);
1146
1147                 t->header_storage = calloc(1, n_bytes);
1148                 CHECK(t->header_storage, ENOMEM);
1149
1150                 t->header_out_storage = calloc(1, n_bytes);
1151                 CHECK(t->header_out_storage, ENOMEM);
1152
1153                 TAILQ_FOREACH(h, &p->headers, node) {
1154                         uint8_t *header_storage;
1155
1156                         header_storage = &t->header_storage[offset];
1157                         offset += h->st->n_bits / 8;
1158
1159                         t->headers[h->id].ptr0 = header_storage;
1160                         t->structs[h->struct_id] = header_storage;
1161                 }
1162         }
1163
1164         return 0;
1165 }
1166
1167 static void
1168 header_build_free(struct rte_swx_pipeline *p)
1169 {
1170         uint32_t i;
1171
1172         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1173                 struct thread *t = &p->threads[i];
1174
1175                 free(t->headers_out);
1176                 t->headers_out = NULL;
1177
1178                 free(t->headers);
1179                 t->headers = NULL;
1180
1181                 free(t->header_out_storage);
1182                 t->header_out_storage = NULL;
1183
1184                 free(t->header_storage);
1185                 t->header_storage = NULL;
1186         }
1187 }
1188
1189 static void
1190 header_free(struct rte_swx_pipeline *p)
1191 {
1192         header_build_free(p);
1193
1194         for ( ; ; ) {
1195                 struct header *elem;
1196
1197                 elem = TAILQ_FIRST(&p->headers);
1198                 if (!elem)
1199                         break;
1200
1201                 TAILQ_REMOVE(&p->headers, elem, node);
1202                 free(elem);
1203         }
1204 }
1205
1206 /*
1207  * Meta-data.
1208  */
1209 int
1210 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1211                                           const char *struct_type_name)
1212 {
1213         struct struct_type *st = NULL;
1214
1215         CHECK(p, EINVAL);
1216
1217         CHECK_NAME(struct_type_name, EINVAL);
1218         st  = struct_type_find(p, struct_type_name);
1219         CHECK(st, EINVAL);
1220         CHECK(!p->metadata_st, EINVAL);
1221
1222         p->metadata_st = st;
1223         p->metadata_struct_id = p->n_structs;
1224
1225         p->n_structs++;
1226
1227         return 0;
1228 }
1229
1230 static int
1231 metadata_build(struct rte_swx_pipeline *p)
1232 {
1233         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1234         uint32_t i;
1235
1236         /* Thread-level initialization. */
1237         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1238                 struct thread *t = &p->threads[i];
1239                 uint8_t *metadata;
1240
1241                 metadata = calloc(1, n_bytes);
1242                 CHECK(metadata, ENOMEM);
1243
1244                 t->metadata = metadata;
1245                 t->structs[p->metadata_struct_id] = metadata;
1246         }
1247
1248         return 0;
1249 }
1250
1251 static void
1252 metadata_build_free(struct rte_swx_pipeline *p)
1253 {
1254         uint32_t i;
1255
1256         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1257                 struct thread *t = &p->threads[i];
1258
1259                 free(t->metadata);
1260                 t->metadata = NULL;
1261         }
1262 }
1263
1264 static void
1265 metadata_free(struct rte_swx_pipeline *p)
1266 {
1267         metadata_build_free(p);
1268 }
1269
1270 /*
1271  * Pipeline.
1272  */
1273 int
1274 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
1275 {
1276         struct rte_swx_pipeline *pipeline;
1277
1278         /* Check input parameters. */
1279         CHECK(p, EINVAL);
1280
1281         /* Memory allocation. */
1282         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
1283         CHECK(pipeline, ENOMEM);
1284
1285         /* Initialization. */
1286         TAILQ_INIT(&pipeline->struct_types);
1287         TAILQ_INIT(&pipeline->port_in_types);
1288         TAILQ_INIT(&pipeline->ports_in);
1289         TAILQ_INIT(&pipeline->port_out_types);
1290         TAILQ_INIT(&pipeline->ports_out);
1291         TAILQ_INIT(&pipeline->extern_types);
1292         TAILQ_INIT(&pipeline->extern_objs);
1293         TAILQ_INIT(&pipeline->extern_funcs);
1294         TAILQ_INIT(&pipeline->headers);
1295
1296         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
1297         pipeline->numa_node = numa_node;
1298
1299         *p = pipeline;
1300         return 0;
1301 }
1302
1303 void
1304 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
1305 {
1306         if (!p)
1307                 return;
1308
1309         metadata_free(p);
1310         header_free(p);
1311         extern_func_free(p);
1312         extern_obj_free(p);
1313         port_out_free(p);
1314         port_in_free(p);
1315         struct_free(p);
1316
1317         free(p);
1318 }
1319
1320 int
1321 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
1322 {
1323         int status;
1324
1325         CHECK(p, EINVAL);
1326         CHECK(p->build_done == 0, EEXIST);
1327
1328         status = port_in_build(p);
1329         if (status)
1330                 goto error;
1331
1332         status = port_out_build(p);
1333         if (status)
1334                 goto error;
1335
1336         status = struct_build(p);
1337         if (status)
1338                 goto error;
1339
1340         status = extern_obj_build(p);
1341         if (status)
1342                 goto error;
1343
1344         status = extern_func_build(p);
1345         if (status)
1346                 goto error;
1347
1348         status = header_build(p);
1349         if (status)
1350                 goto error;
1351
1352         status = metadata_build(p);
1353         if (status)
1354                 goto error;
1355
1356         p->build_done = 1;
1357         return 0;
1358
1359 error:
1360         metadata_build_free(p);
1361         header_build_free(p);
1362         extern_func_build_free(p);
1363         extern_obj_build_free(p);
1364         port_out_build_free(p);
1365         port_in_build_free(p);
1366         struct_build_free(p);
1367
1368         return status;
1369 }