lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9
  10 #include <rte_common.h>
  11 #include <rte_prefetch.h>
  12
  13 #include "rte_swx_pipeline.h"
  14 #include "rte_swx_ctl.h"
  15
  16 #define CHECK(condition, err_code)                                             \
  17 do {                                                                           \
  18         if (!(condition))                                                      \
  19                 return -(err_code);                                            \
  20 } while (0)
  21
  22 #define CHECK_NAME(name, err_code)                                             \
  23         CHECK((name) && (name)[0], err_code)
  24
  25 #ifndef TRACE_LEVEL
  26 #define TRACE_LEVEL 0
  27 #endif
  28
  29 #if TRACE_LEVEL
  30 #define TRACE(...) printf(__VA_ARGS__)
  31 #else
  32 #define TRACE(...)
  33 #endif
  34
  35 /*
  36  * Struct.
  37  */
  38 struct field {
  39         char name[RTE_SWX_NAME_SIZE];
  40         uint32_t n_bits;
  41         uint32_t offset;
  42 };
  43
  44 struct struct_type {
  45         TAILQ_ENTRY(struct_type) node;
  46         char name[RTE_SWX_NAME_SIZE];
  47         struct field *fields;
  48         uint32_t n_fields;
  49         uint32_t n_bits;
  50 };
  51
  52 TAILQ_HEAD(struct_type_tailq, struct_type);
  53
  54 /*
  55  * Input port.
  56  */
  57 struct port_in_type {
  58         TAILQ_ENTRY(port_in_type) node;
  59         char name[RTE_SWX_NAME_SIZE];
  60         struct rte_swx_port_in_ops ops;
  61 };
  62
  63 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  64
  65 struct port_in {
  66         TAILQ_ENTRY(port_in) node;
  67         struct port_in_type *type;
  68         void *obj;
  69         uint32_t id;
  70 };
  71
  72 TAILQ_HEAD(port_in_tailq, port_in);
  73
  74 struct port_in_runtime {
  75         rte_swx_port_in_pkt_rx_t pkt_rx;
  76         void *obj;
  77 };
  78
  79 /*
  80  * Output port.
  81  */
  82 struct port_out_type {
  83         TAILQ_ENTRY(port_out_type) node;
  84         char name[RTE_SWX_NAME_SIZE];
  85         struct rte_swx_port_out_ops ops;
  86 };
  87
  88 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  89
  90 struct port_out {
  91         TAILQ_ENTRY(port_out) node;
  92         struct port_out_type *type;
  93         void *obj;
  94         uint32_t id;
  95 };
  96
  97 TAILQ_HEAD(port_out_tailq, port_out);
  98
  99 struct port_out_runtime {
 100         rte_swx_port_out_pkt_tx_t pkt_tx;
 101         rte_swx_port_out_flush_t flush;
 102         void *obj;
 103 };
 104
 105 /*
 106  * Extern object.
 107  */
 108 struct extern_type_member_func {
 109         TAILQ_ENTRY(extern_type_member_func) node;
 110         char name[RTE_SWX_NAME_SIZE];
 111         rte_swx_extern_type_member_func_t func;
 112         uint32_t id;
 113 };
 114
 115 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 116
 117 struct extern_type {
 118         TAILQ_ENTRY(extern_type) node;
 119         char name[RTE_SWX_NAME_SIZE];
 120         struct struct_type *mailbox_struct_type;
 121         rte_swx_extern_type_constructor_t constructor;
 122         rte_swx_extern_type_destructor_t destructor;
 123         struct extern_type_member_func_tailq funcs;
 124         uint32_t n_funcs;
 125 };
 126
 127 TAILQ_HEAD(extern_type_tailq, extern_type);
 128
 129 struct extern_obj {
 130         TAILQ_ENTRY(extern_obj) node;
 131         char name[RTE_SWX_NAME_SIZE];
 132         struct extern_type *type;
 133         void *obj;
 134         uint32_t struct_id;
 135         uint32_t id;
 136 };
 137
 138 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 139
 140 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 141 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 142 #endif
 143
 144 struct extern_obj_runtime {
 145         void *obj;
 146         uint8_t *mailbox;
 147         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 148 };
 149
 150 /*
 151  * Extern function.
 152  */
 153 struct extern_func {
 154         TAILQ_ENTRY(extern_func) node;
 155         char name[RTE_SWX_NAME_SIZE];
 156         struct struct_type *mailbox_struct_type;
 157         rte_swx_extern_func_t func;
 158         uint32_t struct_id;
 159         uint32_t id;
 160 };
 161
 162 TAILQ_HEAD(extern_func_tailq, extern_func);
 163
 164 struct extern_func_runtime {
 165         uint8_t *mailbox;
 166         rte_swx_extern_func_t func;
 167 };
 168
 169 /*
 170  * Header.
 171  */
 172 struct header {
 173         TAILQ_ENTRY(header) node;
 174         char name[RTE_SWX_NAME_SIZE];
 175         struct struct_type *st;
 176         uint32_t struct_id;
 177         uint32_t id;
 178 };
 179
 180 TAILQ_HEAD(header_tailq, header);
 181
 182 struct header_runtime {
 183         uint8_t *ptr0;
 184 };
 185
 186 struct header_out_runtime {
 187         uint8_t *ptr0;
 188         uint8_t *ptr;
 189         uint32_t n_bytes;
 190 };
 191
 192 /*
 193  * Instruction.
 194  */
 195
 196 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 197  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 198  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 199  * when transferred to packet meta-data and in NBO when transferred to packet
 200  * headers.
 201  */
 202
 203 /* Notation conventions:
 204  *    -Header field: H = h.header.field (dst/src)
 205  *    -Meta-data field: M = m.field (dst/src)
 206  *    -Extern object mailbox field: E = e.field (dst/src)
 207  *    -Extern function mailbox field: F = f.field (dst/src)
 208  *    -Table action data field: T = t.field (src only)
 209  *    -Immediate value: I = 32-bit unsigned value (src only)
 210  */
 211
 212 enum instruction_type {
 213         /* rx m.port_in */
 214         INSTR_RX,
 215
 216         /* tx m.port_out */
 217         INSTR_TX,
 218
 219         /* extract h.header */
 220         INSTR_HDR_EXTRACT,
 221         INSTR_HDR_EXTRACT2,
 222         INSTR_HDR_EXTRACT3,
 223         INSTR_HDR_EXTRACT4,
 224         INSTR_HDR_EXTRACT5,
 225         INSTR_HDR_EXTRACT6,
 226         INSTR_HDR_EXTRACT7,
 227         INSTR_HDR_EXTRACT8,
 228
 229         /* emit h.header */
 230         INSTR_HDR_EMIT,
 231         INSTR_HDR_EMIT_TX,
 232         INSTR_HDR_EMIT2_TX,
 233         INSTR_HDR_EMIT3_TX,
 234         INSTR_HDR_EMIT4_TX,
 235         INSTR_HDR_EMIT5_TX,
 236         INSTR_HDR_EMIT6_TX,
 237         INSTR_HDR_EMIT7_TX,
 238         INSTR_HDR_EMIT8_TX,
 239
 240         /* validate h.header */
 241         INSTR_HDR_VALIDATE,
 242
 243         /* invalidate h.header */
 244         INSTR_HDR_INVALIDATE,
 245 };
 246
 247 struct instr_io {
 248         struct {
 249                 uint8_t offset;
 250                 uint8_t n_bits;
 251                 uint8_t pad[2];
 252         } io;
 253
 254         struct {
 255                 uint8_t header_id[8];
 256                 uint8_t struct_id[8];
 257                 uint8_t n_bytes[8];
 258         } hdr;
 259 };
 260
 261 struct instr_hdr_validity {
 262         uint8_t header_id;
 263 };
 264
 265 struct instruction {
 266         enum instruction_type type;
 267         union {
 268                 struct instr_io io;
 269                 struct instr_hdr_validity valid;
 270         };
 271 };
 272
 273 struct instruction_data {
 274         char label[RTE_SWX_NAME_SIZE];
 275         char jmp_label[RTE_SWX_NAME_SIZE];
 276         uint32_t n_users; /* user = jmp instruction to this instruction. */
 277         int invalid;
 278 };
 279
 280 /*
 281  * Action.
 282  */
 283 struct action {
 284         TAILQ_ENTRY(action) node;
 285         char name[RTE_SWX_NAME_SIZE];
 286         struct struct_type *st;
 287         struct instruction *instructions;
 288         uint32_t n_instructions;
 289         uint32_t id;
 290 };
 291
 292 TAILQ_HEAD(action_tailq, action);
 293
 294 /*
 295  * Table.
 296  */
 297 struct table_type {
 298         TAILQ_ENTRY(table_type) node;
 299         char name[RTE_SWX_NAME_SIZE];
 300         enum rte_swx_table_match_type match_type;
 301         struct rte_swx_table_ops ops;
 302 };
 303
 304 TAILQ_HEAD(table_type_tailq, table_type);
 305
 306 struct match_field {
 307         enum rte_swx_table_match_type match_type;
 308         struct field *field;
 309 };
 310
 311 struct table {
 312         TAILQ_ENTRY(table) node;
 313         char name[RTE_SWX_NAME_SIZE];
 314         char args[RTE_SWX_NAME_SIZE];
 315         struct table_type *type; /* NULL when n_fields == 0. */
 316
 317         /* Match. */
 318         struct match_field *fields;
 319         uint32_t n_fields;
 320         int is_header; /* Only valid when n_fields > 0. */
 321         struct header *header; /* Only valid when n_fields > 0. */
 322
 323         /* Action. */
 324         struct action **actions;
 325         struct action *default_action;
 326         uint8_t *default_action_data;
 327         uint32_t n_actions;
 328         int default_action_is_const;
 329         uint32_t action_data_size_max;
 330
 331         uint32_t size;
 332         uint32_t id;
 333 };
 334
 335 TAILQ_HEAD(table_tailq, table);
 336
 337 struct table_runtime {
 338         rte_swx_table_lookup_t func;
 339         void *mailbox;
 340         uint8_t **key;
 341 };
 342
 343 /*
 344  * Pipeline.
 345  */
 346 struct thread {
 347         /* Packet. */
 348         struct rte_swx_pkt pkt;
 349         uint8_t *ptr;
 350
 351         /* Structures. */
 352         uint8_t **structs;
 353
 354         /* Packet headers. */
 355         struct header_runtime *headers; /* Extracted or generated headers. */
 356         struct header_out_runtime *headers_out; /* Emitted headers. */
 357         uint8_t *header_storage;
 358         uint8_t *header_out_storage;
 359         uint64_t valid_headers;
 360         uint32_t n_headers_out;
 361
 362         /* Packet meta-data. */
 363         uint8_t *metadata;
 364
 365         /* Tables. */
 366         struct table_runtime *tables;
 367         struct rte_swx_table_state *table_state;
 368         uint64_t action_id;
 369         int hit; /* 0 = Miss, 1 = Hit. */
 370
 371         /* Extern objects and functions. */
 372         struct extern_obj_runtime *extern_objs;
 373         struct extern_func_runtime *extern_funcs;
 374
 375         /* Instructions. */
 376         struct instruction *ip;
 377         struct instruction *ret;
 378 };
 379
 380 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 381 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 382 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 383
 384 #define METADATA_READ(thread, offset, n_bits)                                  \
 385 ({                                                                             \
 386         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 387         uint64_t m64 = *m64_ptr;                                               \
 388         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 389         (m64 & m64_mask);                                                      \
 390 })
 391
 392 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 393 {                                                                              \
 394         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 395         uint64_t m64 = *m64_ptr;                                               \
 396         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 397                                                                                \
 398         uint64_t m_new = value;                                                \
 399                                                                                \
 400         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 401 }
 402
 403 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 404 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 405 #endif
 406
 407 struct rte_swx_pipeline {
 408         struct struct_type_tailq struct_types;
 409         struct port_in_type_tailq port_in_types;
 410         struct port_in_tailq ports_in;
 411         struct port_out_type_tailq port_out_types;
 412         struct port_out_tailq ports_out;
 413         struct extern_type_tailq extern_types;
 414         struct extern_obj_tailq extern_objs;
 415         struct extern_func_tailq extern_funcs;
 416         struct header_tailq headers;
 417         struct struct_type *metadata_st;
 418         uint32_t metadata_struct_id;
 419         struct action_tailq actions;
 420         struct table_type_tailq table_types;
 421         struct table_tailq tables;
 422
 423         struct port_in_runtime *in;
 424         struct port_out_runtime *out;
 425         struct instruction **action_instructions;
 426         struct rte_swx_table_state *table_state;
 427         struct instruction *instructions;
 428         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 429
 430         uint32_t n_structs;
 431         uint32_t n_ports_in;
 432         uint32_t n_ports_out;
 433         uint32_t n_extern_objs;
 434         uint32_t n_extern_funcs;
 435         uint32_t n_actions;
 436         uint32_t n_tables;
 437         uint32_t n_headers;
 438         uint32_t thread_id;
 439         uint32_t port_id;
 440         uint32_t n_instructions;
 441         int build_done;
 442         int numa_node;
 443 };
 444
 445 /*
 446  * Struct.
 447  */
 448 static struct struct_type *
 449 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 450 {
 451         struct struct_type *elem;
 452
 453         TAILQ_FOREACH(elem, &p->struct_types, node)
 454                 if (strcmp(elem->name, name) == 0)
 455                         return elem;
 456
 457         return NULL;
 458 }
 459
 460 static struct field *
 461 struct_type_field_find(struct struct_type *st, const char *name)
 462 {
 463         uint32_t i;
 464
 465         for (i = 0; i < st->n_fields; i++) {
 466                 struct field *f = &st->fields[i];
 467
 468                 if (strcmp(f->name, name) == 0)
 469                         return f;
 470         }
 471
 472         return NULL;
 473 }
 474
 475 int
 476 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 477                                       const char *name,
 478                                       struct rte_swx_field_params *fields,
 479                                       uint32_t n_fields)
 480 {
 481         struct struct_type *st;
 482         uint32_t i;
 483
 484         CHECK(p, EINVAL);
 485         CHECK_NAME(name, EINVAL);
 486         CHECK(fields, EINVAL);
 487         CHECK(n_fields, EINVAL);
 488
 489         for (i = 0; i < n_fields; i++) {
 490                 struct rte_swx_field_params *f = &fields[i];
 491                 uint32_t j;
 492
 493                 CHECK_NAME(f->name, EINVAL);
 494                 CHECK(f->n_bits, EINVAL);
 495                 CHECK(f->n_bits <= 64, EINVAL);
 496                 CHECK((f->n_bits & 7) == 0, EINVAL);
 497
 498                 for (j = 0; j < i; j++) {
 499                         struct rte_swx_field_params *f_prev = &fields[j];
 500
 501                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 502                 }
 503         }
 504
 505         CHECK(!struct_type_find(p, name), EEXIST);
 506
 507         /* Node allocation. */
 508         st = calloc(1, sizeof(struct struct_type));
 509         CHECK(st, ENOMEM);
 510
 511         st->fields = calloc(n_fields, sizeof(struct field));
 512         if (!st->fields) {
 513                 free(st);
 514                 CHECK(0, ENOMEM);
 515         }
 516
 517         /* Node initialization. */
 518         strcpy(st->name, name);
 519         for (i = 0; i < n_fields; i++) {
 520                 struct field *dst = &st->fields[i];
 521                 struct rte_swx_field_params *src = &fields[i];
 522
 523                 strcpy(dst->name, src->name);
 524                 dst->n_bits = src->n_bits;
 525                 dst->offset = st->n_bits;
 526
 527                 st->n_bits += src->n_bits;
 528         }
 529         st->n_fields = n_fields;
 530
 531         /* Node add to tailq. */
 532         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 533
 534         return 0;
 535 }
 536
 537 static int
 538 struct_build(struct rte_swx_pipeline *p)
 539 {
 540         uint32_t i;
 541
 542         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 543                 struct thread *t = &p->threads[i];
 544
 545                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 546                 CHECK(t->structs, ENOMEM);
 547         }
 548
 549         return 0;
 550 }
 551
 552 static void
 553 struct_build_free(struct rte_swx_pipeline *p)
 554 {
 555         uint32_t i;
 556
 557         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 558                 struct thread *t = &p->threads[i];
 559
 560                 free(t->structs);
 561                 t->structs = NULL;
 562         }
 563 }
 564
 565 static void
 566 struct_free(struct rte_swx_pipeline *p)
 567 {
 568         struct_build_free(p);
 569
 570         /* Struct types. */
 571         for ( ; ; ) {
 572                 struct struct_type *elem;
 573
 574                 elem = TAILQ_FIRST(&p->struct_types);
 575                 if (!elem)
 576                         break;
 577
 578                 TAILQ_REMOVE(&p->struct_types, elem, node);
 579                 free(elem->fields);
 580                 free(elem);
 581         }
 582 }
 583
 584 /*
 585  * Input port.
 586  */
 587 static struct port_in_type *
 588 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 589 {
 590         struct port_in_type *elem;
 591
 592         if (!name)
 593                 return NULL;
 594
 595         TAILQ_FOREACH(elem, &p->port_in_types, node)
 596                 if (strcmp(elem->name, name) == 0)
 597                         return elem;
 598
 599         return NULL;
 600 }
 601
 602 int
 603 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 604                                        const char *name,
 605                                        struct rte_swx_port_in_ops *ops)
 606 {
 607         struct port_in_type *elem;
 608
 609         CHECK(p, EINVAL);
 610         CHECK_NAME(name, EINVAL);
 611         CHECK(ops, EINVAL);
 612         CHECK(ops->create, EINVAL);
 613         CHECK(ops->free, EINVAL);
 614         CHECK(ops->pkt_rx, EINVAL);
 615         CHECK(ops->stats_read, EINVAL);
 616
 617         CHECK(!port_in_type_find(p, name), EEXIST);
 618
 619         /* Node allocation. */
 620         elem = calloc(1, sizeof(struct port_in_type));
 621         CHECK(elem, ENOMEM);
 622
 623         /* Node initialization. */
 624         strcpy(elem->name, name);
 625         memcpy(&elem->ops, ops, sizeof(*ops));
 626
 627         /* Node add to tailq. */
 628         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 629
 630         return 0;
 631 }
 632
 633 static struct port_in *
 634 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 635 {
 636         struct port_in *port;
 637
 638         TAILQ_FOREACH(port, &p->ports_in, node)
 639                 if (port->id == port_id)
 640                         return port;
 641
 642         return NULL;
 643 }
 644
 645 int
 646 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 647                                 uint32_t port_id,
 648                                 const char *port_type_name,
 649                                 void *args)
 650 {
 651         struct port_in_type *type = NULL;
 652         struct port_in *port = NULL;
 653         void *obj = NULL;
 654
 655         CHECK(p, EINVAL);
 656
 657         CHECK(!port_in_find(p, port_id), EINVAL);
 658
 659         CHECK_NAME(port_type_name, EINVAL);
 660         type = port_in_type_find(p, port_type_name);
 661         CHECK(type, EINVAL);
 662
 663         obj = type->ops.create(args);
 664         CHECK(obj, ENODEV);
 665
 666         /* Node allocation. */
 667         port = calloc(1, sizeof(struct port_in));
 668         CHECK(port, ENOMEM);
 669
 670         /* Node initialization. */
 671         port->type = type;
 672         port->obj = obj;
 673         port->id = port_id;
 674
 675         /* Node add to tailq. */
 676         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 677         if (p->n_ports_in < port_id + 1)
 678                 p->n_ports_in = port_id + 1;
 679
 680         return 0;
 681 }
 682
 683 static int
 684 port_in_build(struct rte_swx_pipeline *p)
 685 {
 686         struct port_in *port;
 687         uint32_t i;
 688
 689         CHECK(p->n_ports_in, EINVAL);
 690         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 691
 692         for (i = 0; i < p->n_ports_in; i++)
 693                 CHECK(port_in_find(p, i), EINVAL);
 694
 695         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 696         CHECK(p->in, ENOMEM);
 697
 698         TAILQ_FOREACH(port, &p->ports_in, node) {
 699                 struct port_in_runtime *in = &p->in[port->id];
 700
 701                 in->pkt_rx = port->type->ops.pkt_rx;
 702                 in->obj = port->obj;
 703         }
 704
 705         return 0;
 706 }
 707
 708 static void
 709 port_in_build_free(struct rte_swx_pipeline *p)
 710 {
 711         free(p->in);
 712         p->in = NULL;
 713 }
 714
 715 static void
 716 port_in_free(struct rte_swx_pipeline *p)
 717 {
 718         port_in_build_free(p);
 719
 720         /* Input ports. */
 721         for ( ; ; ) {
 722                 struct port_in *port;
 723
 724                 port = TAILQ_FIRST(&p->ports_in);
 725                 if (!port)
 726                         break;
 727
 728                 TAILQ_REMOVE(&p->ports_in, port, node);
 729                 port->type->ops.free(port->obj);
 730                 free(port);
 731         }
 732
 733         /* Input port types. */
 734         for ( ; ; ) {
 735                 struct port_in_type *elem;
 736
 737                 elem = TAILQ_FIRST(&p->port_in_types);
 738                 if (!elem)
 739                         break;
 740
 741                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 742                 free(elem);
 743         }
 744 }
 745
 746 /*
 747  * Output port.
 748  */
 749 static struct port_out_type *
 750 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 751 {
 752         struct port_out_type *elem;
 753
 754         if (!name)
 755                 return NULL;
 756
 757         TAILQ_FOREACH(elem, &p->port_out_types, node)
 758                 if (!strcmp(elem->name, name))
 759                         return elem;
 760
 761         return NULL;
 762 }
 763
 764 int
 765 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 766                                         const char *name,
 767                                         struct rte_swx_port_out_ops *ops)
 768 {
 769         struct port_out_type *elem;
 770
 771         CHECK(p, EINVAL);
 772         CHECK_NAME(name, EINVAL);
 773         CHECK(ops, EINVAL);
 774         CHECK(ops->create, EINVAL);
 775         CHECK(ops->free, EINVAL);
 776         CHECK(ops->pkt_tx, EINVAL);
 777         CHECK(ops->stats_read, EINVAL);
 778
 779         CHECK(!port_out_type_find(p, name), EEXIST);
 780
 781         /* Node allocation. */
 782         elem = calloc(1, sizeof(struct port_out_type));
 783         CHECK(elem, ENOMEM);
 784
 785         /* Node initialization. */
 786         strcpy(elem->name, name);
 787         memcpy(&elem->ops, ops, sizeof(*ops));
 788
 789         /* Node add to tailq. */
 790         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 791
 792         return 0;
 793 }
 794
 795 static struct port_out *
 796 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 797 {
 798         struct port_out *port;
 799
 800         TAILQ_FOREACH(port, &p->ports_out, node)
 801                 if (port->id == port_id)
 802                         return port;
 803
 804         return NULL;
 805 }
 806
 807 int
 808 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 809                                  uint32_t port_id,
 810                                  const char *port_type_name,
 811                                  void *args)
 812 {
 813         struct port_out_type *type = NULL;
 814         struct port_out *port = NULL;
 815         void *obj = NULL;
 816
 817         CHECK(p, EINVAL);
 818
 819         CHECK(!port_out_find(p, port_id), EINVAL);
 820
 821         CHECK_NAME(port_type_name, EINVAL);
 822         type = port_out_type_find(p, port_type_name);
 823         CHECK(type, EINVAL);
 824
 825         obj = type->ops.create(args);
 826         CHECK(obj, ENODEV);
 827
 828         /* Node allocation. */
 829         port = calloc(1, sizeof(struct port_out));
 830         CHECK(port, ENOMEM);
 831
 832         /* Node initialization. */
 833         port->type = type;
 834         port->obj = obj;
 835         port->id = port_id;
 836
 837         /* Node add to tailq. */
 838         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 839         if (p->n_ports_out < port_id + 1)
 840                 p->n_ports_out = port_id + 1;
 841
 842         return 0;
 843 }
 844
 845 static int
 846 port_out_build(struct rte_swx_pipeline *p)
 847 {
 848         struct port_out *port;
 849         uint32_t i;
 850
 851         CHECK(p->n_ports_out, EINVAL);
 852
 853         for (i = 0; i < p->n_ports_out; i++)
 854                 CHECK(port_out_find(p, i), EINVAL);
 855
 856         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 857         CHECK(p->out, ENOMEM);
 858
 859         TAILQ_FOREACH(port, &p->ports_out, node) {
 860                 struct port_out_runtime *out = &p->out[port->id];
 861
 862                 out->pkt_tx = port->type->ops.pkt_tx;
 863                 out->flush = port->type->ops.flush;
 864                 out->obj = port->obj;
 865         }
 866
 867         return 0;
 868 }
 869
 870 static void
 871 port_out_build_free(struct rte_swx_pipeline *p)
 872 {
 873         free(p->out);
 874         p->out = NULL;
 875 }
 876
 877 static void
 878 port_out_free(struct rte_swx_pipeline *p)
 879 {
 880         port_out_build_free(p);
 881
 882         /* Output ports. */
 883         for ( ; ; ) {
 884                 struct port_out *port;
 885
 886                 port = TAILQ_FIRST(&p->ports_out);
 887                 if (!port)
 888                         break;
 889
 890                 TAILQ_REMOVE(&p->ports_out, port, node);
 891                 port->type->ops.free(port->obj);
 892                 free(port);
 893         }
 894
 895         /* Output port types. */
 896         for ( ; ; ) {
 897                 struct port_out_type *elem;
 898
 899                 elem = TAILQ_FIRST(&p->port_out_types);
 900                 if (!elem)
 901                         break;
 902
 903                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 904                 free(elem);
 905         }
 906 }
 907
 908 /*
 909  * Extern object.
 910  */
 911 static struct extern_type *
 912 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 913 {
 914         struct extern_type *elem;
 915
 916         TAILQ_FOREACH(elem, &p->extern_types, node)
 917                 if (strcmp(elem->name, name) == 0)
 918                         return elem;
 919
 920         return NULL;
 921 }
 922
 923 static struct extern_type_member_func *
 924 extern_type_member_func_find(struct extern_type *type, const char *name)
 925 {
 926         struct extern_type_member_func *elem;
 927
 928         TAILQ_FOREACH(elem, &type->funcs, node)
 929                 if (strcmp(elem->name, name) == 0)
 930                         return elem;
 931
 932         return NULL;
 933 }
 934
 935 static struct extern_obj *
 936 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 937 {
 938         struct extern_obj *elem;
 939
 940         TAILQ_FOREACH(elem, &p->extern_objs, node)
 941                 if (strcmp(elem->name, name) == 0)
 942                         return elem;
 943
 944         return NULL;
 945 }
 946
 947 int
 948 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 949         const char *name,
 950         const char *mailbox_struct_type_name,
 951         rte_swx_extern_type_constructor_t constructor,
 952         rte_swx_extern_type_destructor_t destructor)
 953 {
 954         struct extern_type *elem;
 955         struct struct_type *mailbox_struct_type;
 956
 957         CHECK(p, EINVAL);
 958
 959         CHECK_NAME(name, EINVAL);
 960         CHECK(!extern_type_find(p, name), EEXIST);
 961
 962         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 963         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 964         CHECK(mailbox_struct_type, EINVAL);
 965
 966         CHECK(constructor, EINVAL);
 967         CHECK(destructor, EINVAL);
 968
 969         /* Node allocation. */
 970         elem = calloc(1, sizeof(struct extern_type));
 971         CHECK(elem, ENOMEM);
 972
 973         /* Node initialization. */
 974         strcpy(elem->name, name);
 975         elem->mailbox_struct_type = mailbox_struct_type;
 976         elem->constructor = constructor;
 977         elem->destructor = destructor;
 978         TAILQ_INIT(&elem->funcs);
 979
 980         /* Node add to tailq. */
 981         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 982
 983         return 0;
 984 }
 985
 986 int
 987 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 988         const char *extern_type_name,
 989         const char *name,
 990         rte_swx_extern_type_member_func_t member_func)
 991 {
 992         struct extern_type *type;
 993         struct extern_type_member_func *type_member;
 994
 995         CHECK(p, EINVAL);
 996
 997         CHECK(extern_type_name, EINVAL);
 998         type = extern_type_find(p, extern_type_name);
 999         CHECK(type, EINVAL);
1000         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1001
1002         CHECK(name, EINVAL);
1003         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1004
1005         CHECK(member_func, EINVAL);
1006
1007         /* Node allocation. */
1008         type_member = calloc(1, sizeof(struct extern_type_member_func));
1009         CHECK(type_member, ENOMEM);
1010
1011         /* Node initialization. */
1012         strcpy(type_member->name, name);
1013         type_member->func = member_func;
1014         type_member->id = type->n_funcs;
1015
1016         /* Node add to tailq. */
1017         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1018         type->n_funcs++;
1019
1020         return 0;
1021 }
1022
1023 int
1024 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1025                                       const char *extern_type_name,
1026                                       const char *name,
1027                                       const char *args)
1028 {
1029         struct extern_type *type;
1030         struct extern_obj *obj;
1031         void *obj_handle;
1032
1033         CHECK(p, EINVAL);
1034
1035         CHECK_NAME(extern_type_name, EINVAL);
1036         type = extern_type_find(p, extern_type_name);
1037         CHECK(type, EINVAL);
1038
1039         CHECK_NAME(name, EINVAL);
1040         CHECK(!extern_obj_find(p, name), EEXIST);
1041
1042         /* Node allocation. */
1043         obj = calloc(1, sizeof(struct extern_obj));
1044         CHECK(obj, ENOMEM);
1045
1046         /* Object construction. */
1047         obj_handle = type->constructor(args);
1048         if (!obj_handle) {
1049                 free(obj);
1050                 CHECK(0, ENODEV);
1051         }
1052
1053         /* Node initialization. */
1054         strcpy(obj->name, name);
1055         obj->type = type;
1056         obj->obj = obj_handle;
1057         obj->struct_id = p->n_structs;
1058         obj->id = p->n_extern_objs;
1059
1060         /* Node add to tailq. */
1061         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1062         p->n_extern_objs++;
1063         p->n_structs++;
1064
1065         return 0;
1066 }
1067
1068 static int
1069 extern_obj_build(struct rte_swx_pipeline *p)
1070 {
1071         uint32_t i;
1072
1073         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1074                 struct thread *t = &p->threads[i];
1075                 struct extern_obj *obj;
1076
1077                 t->extern_objs = calloc(p->n_extern_objs,
1078                                         sizeof(struct extern_obj_runtime));
1079                 CHECK(t->extern_objs, ENOMEM);
1080
1081                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1082                         struct extern_obj_runtime *r =
1083                                 &t->extern_objs[obj->id];
1084                         struct extern_type_member_func *func;
1085                         uint32_t mailbox_size =
1086                                 obj->type->mailbox_struct_type->n_bits / 8;
1087
1088                         r->obj = obj->obj;
1089
1090                         r->mailbox = calloc(1, mailbox_size);
1091                         CHECK(r->mailbox, ENOMEM);
1092
1093                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1094                                 r->funcs[func->id] = func->func;
1095
1096                         t->structs[obj->struct_id] = r->mailbox;
1097                 }
1098         }
1099
1100         return 0;
1101 }
1102
1103 static void
1104 extern_obj_build_free(struct rte_swx_pipeline *p)
1105 {
1106         uint32_t i;
1107
1108         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1109                 struct thread *t = &p->threads[i];
1110                 uint32_t j;
1111
1112                 if (!t->extern_objs)
1113                         continue;
1114
1115                 for (j = 0; j < p->n_extern_objs; j++) {
1116                         struct extern_obj_runtime *r = &t->extern_objs[j];
1117
1118                         free(r->mailbox);
1119                 }
1120
1121                 free(t->extern_objs);
1122                 t->extern_objs = NULL;
1123         }
1124 }
1125
1126 static void
1127 extern_obj_free(struct rte_swx_pipeline *p)
1128 {
1129         extern_obj_build_free(p);
1130
1131         /* Extern objects. */
1132         for ( ; ; ) {
1133                 struct extern_obj *elem;
1134
1135                 elem = TAILQ_FIRST(&p->extern_objs);
1136                 if (!elem)
1137                         break;
1138
1139                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1140                 if (elem->obj)
1141                         elem->type->destructor(elem->obj);
1142                 free(elem);
1143         }
1144
1145         /* Extern types. */
1146         for ( ; ; ) {
1147                 struct extern_type *elem;
1148
1149                 elem = TAILQ_FIRST(&p->extern_types);
1150                 if (!elem)
1151                         break;
1152
1153                 TAILQ_REMOVE(&p->extern_types, elem, node);
1154
1155                 for ( ; ; ) {
1156                         struct extern_type_member_func *func;
1157
1158                         func = TAILQ_FIRST(&elem->funcs);
1159                         if (!func)
1160                                 break;
1161
1162                         TAILQ_REMOVE(&elem->funcs, func, node);
1163                         free(func);
1164                 }
1165
1166                 free(elem);
1167         }
1168 }
1169
1170 /*
1171  * Extern function.
1172  */
1173 static struct extern_func *
1174 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1175 {
1176         struct extern_func *elem;
1177
1178         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1179                 if (strcmp(elem->name, name) == 0)
1180                         return elem;
1181
1182         return NULL;
1183 }
1184
1185 int
1186 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1187                                       const char *name,
1188                                       const char *mailbox_struct_type_name,
1189                                       rte_swx_extern_func_t func)
1190 {
1191         struct extern_func *f;
1192         struct struct_type *mailbox_struct_type;
1193
1194         CHECK(p, EINVAL);
1195
1196         CHECK_NAME(name, EINVAL);
1197         CHECK(!extern_func_find(p, name), EEXIST);
1198
1199         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1200         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1201         CHECK(mailbox_struct_type, EINVAL);
1202
1203         CHECK(func, EINVAL);
1204
1205         /* Node allocation. */
1206         f = calloc(1, sizeof(struct extern_func));
1207         CHECK(func, ENOMEM);
1208
1209         /* Node initialization. */
1210         strcpy(f->name, name);
1211         f->mailbox_struct_type = mailbox_struct_type;
1212         f->func = func;
1213         f->struct_id = p->n_structs;
1214         f->id = p->n_extern_funcs;
1215
1216         /* Node add to tailq. */
1217         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1218         p->n_extern_funcs++;
1219         p->n_structs++;
1220
1221         return 0;
1222 }
1223
1224 static int
1225 extern_func_build(struct rte_swx_pipeline *p)
1226 {
1227         uint32_t i;
1228
1229         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1230                 struct thread *t = &p->threads[i];
1231                 struct extern_func *func;
1232
1233                 /* Memory allocation. */
1234                 t->extern_funcs = calloc(p->n_extern_funcs,
1235                                          sizeof(struct extern_func_runtime));
1236                 CHECK(t->extern_funcs, ENOMEM);
1237
1238                 /* Extern function. */
1239                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1240                         struct extern_func_runtime *r =
1241                                 &t->extern_funcs[func->id];
1242                         uint32_t mailbox_size =
1243                                 func->mailbox_struct_type->n_bits / 8;
1244
1245                         r->func = func->func;
1246
1247                         r->mailbox = calloc(1, mailbox_size);
1248                         CHECK(r->mailbox, ENOMEM);
1249
1250                         t->structs[func->struct_id] = r->mailbox;
1251                 }
1252         }
1253
1254         return 0;
1255 }
1256
1257 static void
1258 extern_func_build_free(struct rte_swx_pipeline *p)
1259 {
1260         uint32_t i;
1261
1262         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1263                 struct thread *t = &p->threads[i];
1264                 uint32_t j;
1265
1266                 if (!t->extern_funcs)
1267                         continue;
1268
1269                 for (j = 0; j < p->n_extern_funcs; j++) {
1270                         struct extern_func_runtime *r = &t->extern_funcs[j];
1271
1272                         free(r->mailbox);
1273                 }
1274
1275                 free(t->extern_funcs);
1276                 t->extern_funcs = NULL;
1277         }
1278 }
1279
1280 static void
1281 extern_func_free(struct rte_swx_pipeline *p)
1282 {
1283         extern_func_build_free(p);
1284
1285         for ( ; ; ) {
1286                 struct extern_func *elem;
1287
1288                 elem = TAILQ_FIRST(&p->extern_funcs);
1289                 if (!elem)
1290                         break;
1291
1292                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1293                 free(elem);
1294         }
1295 }
1296
1297 /*
1298  * Header.
1299  */
1300 static struct header *
1301 header_find(struct rte_swx_pipeline *p, const char *name)
1302 {
1303         struct header *elem;
1304
1305         TAILQ_FOREACH(elem, &p->headers, node)
1306                 if (strcmp(elem->name, name) == 0)
1307                         return elem;
1308
1309         return NULL;
1310 }
1311
1312 static struct header *
1313 header_parse(struct rte_swx_pipeline *p,
1314              const char *name)
1315 {
1316         if (name[0] != 'h' || name[1] != '.')
1317                 return NULL;
1318
1319         return header_find(p, &name[2]);
1320 }
1321
1322 static struct field *
1323 header_field_parse(struct rte_swx_pipeline *p,
1324                    const char *name,
1325                    struct header **header)
1326 {
1327         struct header *h;
1328         struct field *f;
1329         char *header_name, *field_name;
1330
1331         if ((name[0] != 'h') || (name[1] != '.'))
1332                 return NULL;
1333
1334         header_name = strdup(&name[2]);
1335         if (!header_name)
1336                 return NULL;
1337
1338         field_name = strchr(header_name, '.');
1339         if (!field_name) {
1340                 free(header_name);
1341                 return NULL;
1342         }
1343
1344         *field_name = 0;
1345         field_name++;
1346
1347         h = header_find(p, header_name);
1348         if (!h) {
1349                 free(header_name);
1350                 return NULL;
1351         }
1352
1353         f = struct_type_field_find(h->st, field_name);
1354         if (!f) {
1355                 free(header_name);
1356                 return NULL;
1357         }
1358
1359         if (header)
1360                 *header = h;
1361
1362         free(header_name);
1363         return f;
1364 }
1365
1366 int
1367 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1368                                         const char *name,
1369                                         const char *struct_type_name)
1370 {
1371         struct struct_type *st;
1372         struct header *h;
1373         size_t n_headers_max;
1374
1375         CHECK(p, EINVAL);
1376         CHECK_NAME(name, EINVAL);
1377         CHECK_NAME(struct_type_name, EINVAL);
1378
1379         CHECK(!header_find(p, name), EEXIST);
1380
1381         st = struct_type_find(p, struct_type_name);
1382         CHECK(st, EINVAL);
1383
1384         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1385         CHECK(p->n_headers < n_headers_max, ENOSPC);
1386
1387         /* Node allocation. */
1388         h = calloc(1, sizeof(struct header));
1389         CHECK(h, ENOMEM);
1390
1391         /* Node initialization. */
1392         strcpy(h->name, name);
1393         h->st = st;
1394         h->struct_id = p->n_structs;
1395         h->id = p->n_headers;
1396
1397         /* Node add to tailq. */
1398         TAILQ_INSERT_TAIL(&p->headers, h, node);
1399         p->n_headers++;
1400         p->n_structs++;
1401
1402         return 0;
1403 }
1404
1405 static int
1406 header_build(struct rte_swx_pipeline *p)
1407 {
1408         struct header *h;
1409         uint32_t n_bytes = 0, i;
1410
1411         TAILQ_FOREACH(h, &p->headers, node) {
1412                 n_bytes += h->st->n_bits / 8;
1413         }
1414
1415         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1416                 struct thread *t = &p->threads[i];
1417                 uint32_t offset = 0;
1418
1419                 t->headers = calloc(p->n_headers,
1420                                     sizeof(struct header_runtime));
1421                 CHECK(t->headers, ENOMEM);
1422
1423                 t->headers_out = calloc(p->n_headers,
1424                                         sizeof(struct header_out_runtime));
1425                 CHECK(t->headers_out, ENOMEM);
1426
1427                 t->header_storage = calloc(1, n_bytes);
1428                 CHECK(t->header_storage, ENOMEM);
1429
1430                 t->header_out_storage = calloc(1, n_bytes);
1431                 CHECK(t->header_out_storage, ENOMEM);
1432
1433                 TAILQ_FOREACH(h, &p->headers, node) {
1434                         uint8_t *header_storage;
1435
1436                         header_storage = &t->header_storage[offset];
1437                         offset += h->st->n_bits / 8;
1438
1439                         t->headers[h->id].ptr0 = header_storage;
1440                         t->structs[h->struct_id] = header_storage;
1441                 }
1442         }
1443
1444         return 0;
1445 }
1446
1447 static void
1448 header_build_free(struct rte_swx_pipeline *p)
1449 {
1450         uint32_t i;
1451
1452         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1453                 struct thread *t = &p->threads[i];
1454
1455                 free(t->headers_out);
1456                 t->headers_out = NULL;
1457
1458                 free(t->headers);
1459                 t->headers = NULL;
1460
1461                 free(t->header_out_storage);
1462                 t->header_out_storage = NULL;
1463
1464                 free(t->header_storage);
1465                 t->header_storage = NULL;
1466         }
1467 }
1468
1469 static void
1470 header_free(struct rte_swx_pipeline *p)
1471 {
1472         header_build_free(p);
1473
1474         for ( ; ; ) {
1475                 struct header *elem;
1476
1477                 elem = TAILQ_FIRST(&p->headers);
1478                 if (!elem)
1479                         break;
1480
1481                 TAILQ_REMOVE(&p->headers, elem, node);
1482                 free(elem);
1483         }
1484 }
1485
1486 /*
1487  * Meta-data.
1488  */
1489 static struct field *
1490 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1491 {
1492         if (!p->metadata_st)
1493                 return NULL;
1494
1495         if (name[0] != 'm' || name[1] != '.')
1496                 return NULL;
1497
1498         return struct_type_field_find(p->metadata_st, &name[2]);
1499 }
1500
1501 int
1502 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1503                                           const char *struct_type_name)
1504 {
1505         struct struct_type *st = NULL;
1506
1507         CHECK(p, EINVAL);
1508
1509         CHECK_NAME(struct_type_name, EINVAL);
1510         st  = struct_type_find(p, struct_type_name);
1511         CHECK(st, EINVAL);
1512         CHECK(!p->metadata_st, EINVAL);
1513
1514         p->metadata_st = st;
1515         p->metadata_struct_id = p->n_structs;
1516
1517         p->n_structs++;
1518
1519         return 0;
1520 }
1521
1522 static int
1523 metadata_build(struct rte_swx_pipeline *p)
1524 {
1525         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1526         uint32_t i;
1527
1528         /* Thread-level initialization. */
1529         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1530                 struct thread *t = &p->threads[i];
1531                 uint8_t *metadata;
1532
1533                 metadata = calloc(1, n_bytes);
1534                 CHECK(metadata, ENOMEM);
1535
1536                 t->metadata = metadata;
1537                 t->structs[p->metadata_struct_id] = metadata;
1538         }
1539
1540         return 0;
1541 }
1542
1543 static void
1544 metadata_build_free(struct rte_swx_pipeline *p)
1545 {
1546         uint32_t i;
1547
1548         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1549                 struct thread *t = &p->threads[i];
1550
1551                 free(t->metadata);
1552                 t->metadata = NULL;
1553         }
1554 }
1555
1556 static void
1557 metadata_free(struct rte_swx_pipeline *p)
1558 {
1559         metadata_build_free(p);
1560 }
1561
1562 /*
1563  * Instruction.
1564  */
1565 static inline void
1566 pipeline_port_inc(struct rte_swx_pipeline *p)
1567 {
1568         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1569 }
1570
1571 static inline void
1572 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1573 {
1574         t->ip = p->instructions;
1575 }
1576
1577 static inline void
1578 thread_ip_inc(struct rte_swx_pipeline *p);
1579
1580 static inline void
1581 thread_ip_inc(struct rte_swx_pipeline *p)
1582 {
1583         struct thread *t = &p->threads[p->thread_id];
1584
1585         t->ip++;
1586 }
1587
1588 static inline void
1589 thread_ip_inc_cond(struct thread *t, int cond)
1590 {
1591         t->ip += cond;
1592 }
1593
1594 static inline void
1595 thread_yield(struct rte_swx_pipeline *p)
1596 {
1597         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1598 }
1599
1600 /*
1601  * rx.
1602  */
1603 static int
1604 instr_rx_translate(struct rte_swx_pipeline *p,
1605                    struct action *action,
1606                    char **tokens,
1607                    int n_tokens,
1608                    struct instruction *instr,
1609                    struct instruction_data *data __rte_unused)
1610 {
1611         struct field *f;
1612
1613         CHECK(!action, EINVAL);
1614         CHECK(n_tokens == 2, EINVAL);
1615
1616         f = metadata_field_parse(p, tokens[1]);
1617         CHECK(f, EINVAL);
1618
1619         instr->type = INSTR_RX;
1620         instr->io.io.offset = f->offset / 8;
1621         instr->io.io.n_bits = f->n_bits;
1622         return 0;
1623 }
1624
1625 static inline void
1626 instr_rx_exec(struct rte_swx_pipeline *p);
1627
1628 static inline void
1629 instr_rx_exec(struct rte_swx_pipeline *p)
1630 {
1631         struct thread *t = &p->threads[p->thread_id];
1632         struct instruction *ip = t->ip;
1633         struct port_in_runtime *port = &p->in[p->port_id];
1634         struct rte_swx_pkt *pkt = &t->pkt;
1635         int pkt_received;
1636
1637         /* Packet. */
1638         pkt_received = port->pkt_rx(port->obj, pkt);
1639         t->ptr = &pkt->pkt[pkt->offset];
1640         rte_prefetch0(t->ptr);
1641
1642         TRACE("[Thread %2u] rx %s from port %u\n",
1643               p->thread_id,
1644               pkt_received ? "1 pkt" : "0 pkts",
1645               p->port_id);
1646
1647         /* Headers. */
1648         t->valid_headers = 0;
1649         t->n_headers_out = 0;
1650
1651         /* Meta-data. */
1652         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1653
1654         /* Tables. */
1655         t->table_state = p->table_state;
1656
1657         /* Thread. */
1658         pipeline_port_inc(p);
1659         thread_ip_inc_cond(t, pkt_received);
1660         thread_yield(p);
1661 }
1662
1663 /*
1664  * tx.
1665  */
1666 static int
1667 instr_tx_translate(struct rte_swx_pipeline *p,
1668                    struct action *action __rte_unused,
1669                    char **tokens,
1670                    int n_tokens,
1671                    struct instruction *instr,
1672                    struct instruction_data *data __rte_unused)
1673 {
1674         struct field *f;
1675
1676         CHECK(n_tokens == 2, EINVAL);
1677
1678         f = metadata_field_parse(p, tokens[1]);
1679         CHECK(f, EINVAL);
1680
1681         instr->type = INSTR_TX;
1682         instr->io.io.offset = f->offset / 8;
1683         instr->io.io.n_bits = f->n_bits;
1684         return 0;
1685 }
1686
1687 static inline void
1688 emit_handler(struct thread *t)
1689 {
1690         struct header_out_runtime *h0 = &t->headers_out[0];
1691         struct header_out_runtime *h1 = &t->headers_out[1];
1692         uint32_t offset = 0, i;
1693
1694         /* No header change or header decapsulation. */
1695         if ((t->n_headers_out == 1) &&
1696             (h0->ptr + h0->n_bytes == t->ptr)) {
1697                 TRACE("Emit handler: no header change or header decap.\n");
1698
1699                 t->pkt.offset -= h0->n_bytes;
1700                 t->pkt.length += h0->n_bytes;
1701
1702                 return;
1703         }
1704
1705         /* Header encapsulation (optionally, with prior header decasulation). */
1706         if ((t->n_headers_out == 2) &&
1707             (h1->ptr + h1->n_bytes == t->ptr) &&
1708             (h0->ptr == h0->ptr0)) {
1709                 uint32_t offset;
1710
1711                 TRACE("Emit handler: header encapsulation.\n");
1712
1713                 offset = h0->n_bytes + h1->n_bytes;
1714                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1715                 t->pkt.offset -= offset;
1716                 t->pkt.length += offset;
1717
1718                 return;
1719         }
1720
1721         /* Header insertion. */
1722         /* TBD */
1723
1724         /* Header extraction. */
1725         /* TBD */
1726
1727         /* For any other case. */
1728         TRACE("Emit handler: complex case.\n");
1729
1730         for (i = 0; i < t->n_headers_out; i++) {
1731                 struct header_out_runtime *h = &t->headers_out[i];
1732
1733                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1734                 offset += h->n_bytes;
1735         }
1736
1737         if (offset) {
1738                 memcpy(t->ptr - offset, t->header_out_storage, offset);
1739                 t->pkt.offset -= offset;
1740                 t->pkt.length += offset;
1741         }
1742 }
1743
1744 static inline void
1745 instr_tx_exec(struct rte_swx_pipeline *p);
1746
1747 static inline void
1748 instr_tx_exec(struct rte_swx_pipeline *p)
1749 {
1750         struct thread *t = &p->threads[p->thread_id];
1751         struct instruction *ip = t->ip;
1752         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1753         struct port_out_runtime *port = &p->out[port_id];
1754         struct rte_swx_pkt *pkt = &t->pkt;
1755
1756         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1757               p->thread_id,
1758               (uint32_t)port_id);
1759
1760         /* Headers. */
1761         emit_handler(t);
1762
1763         /* Packet. */
1764         port->pkt_tx(port->obj, pkt);
1765
1766         /* Thread. */
1767         thread_ip_reset(p, t);
1768         instr_rx_exec(p);
1769 }
1770
1771 /*
1772  * extract.
1773  */
1774 static int
1775 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1776                             struct action *action,
1777                             char **tokens,
1778                             int n_tokens,
1779                             struct instruction *instr,
1780                             struct instruction_data *data __rte_unused)
1781 {
1782         struct header *h;
1783
1784         CHECK(!action, EINVAL);
1785         CHECK(n_tokens == 2, EINVAL);
1786
1787         h = header_parse(p, tokens[1]);
1788         CHECK(h, EINVAL);
1789
1790         instr->type = INSTR_HDR_EXTRACT;
1791         instr->io.hdr.header_id[0] = h->id;
1792         instr->io.hdr.struct_id[0] = h->struct_id;
1793         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1794         return 0;
1795 }
1796
1797 static inline void
1798 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
1799
1800 static inline void
1801 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
1802 {
1803         struct thread *t = &p->threads[p->thread_id];
1804         struct instruction *ip = t->ip;
1805         uint64_t valid_headers = t->valid_headers;
1806         uint8_t *ptr = t->ptr;
1807         uint32_t offset = t->pkt.offset;
1808         uint32_t length = t->pkt.length;
1809         uint32_t i;
1810
1811         for (i = 0; i < n_extract; i++) {
1812                 uint32_t header_id = ip->io.hdr.header_id[i];
1813                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1814                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1815
1816                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1817                       p->thread_id,
1818                       header_id,
1819                       n_bytes);
1820
1821                 /* Headers. */
1822                 t->structs[struct_id] = ptr;
1823                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1824
1825                 /* Packet. */
1826                 offset += n_bytes;
1827                 length -= n_bytes;
1828                 ptr += n_bytes;
1829         }
1830
1831         /* Headers. */
1832         t->valid_headers = valid_headers;
1833
1834         /* Packet. */
1835         t->pkt.offset = offset;
1836         t->pkt.length = length;
1837         t->ptr = ptr;
1838 }
1839
1840 static inline void
1841 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1842 {
1843         __instr_hdr_extract_exec(p, 1);
1844
1845         /* Thread. */
1846         thread_ip_inc(p);
1847 }
1848
1849 static inline void
1850 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1851 {
1852         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
1853               p->thread_id);
1854
1855         __instr_hdr_extract_exec(p, 2);
1856
1857         /* Thread. */
1858         thread_ip_inc(p);
1859 }
1860
1861 static inline void
1862 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1863 {
1864         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
1865               p->thread_id);
1866
1867         __instr_hdr_extract_exec(p, 3);
1868
1869         /* Thread. */
1870         thread_ip_inc(p);
1871 }
1872
1873 static inline void
1874 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1875 {
1876         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
1877               p->thread_id);
1878
1879         __instr_hdr_extract_exec(p, 4);
1880
1881         /* Thread. */
1882         thread_ip_inc(p);
1883 }
1884
1885 static inline void
1886 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1887 {
1888         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
1889               p->thread_id);
1890
1891         __instr_hdr_extract_exec(p, 5);
1892
1893         /* Thread. */
1894         thread_ip_inc(p);
1895 }
1896
1897 static inline void
1898 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1899 {
1900         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
1901               p->thread_id);
1902
1903         __instr_hdr_extract_exec(p, 6);
1904
1905         /* Thread. */
1906         thread_ip_inc(p);
1907 }
1908
1909 static inline void
1910 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1911 {
1912         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
1913               p->thread_id);
1914
1915         __instr_hdr_extract_exec(p, 7);
1916
1917         /* Thread. */
1918         thread_ip_inc(p);
1919 }
1920
1921 static inline void
1922 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1923 {
1924         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
1925               p->thread_id);
1926
1927         __instr_hdr_extract_exec(p, 8);
1928
1929         /* Thread. */
1930         thread_ip_inc(p);
1931 }
1932
1933 /*
1934  * emit.
1935  */
1936 static int
1937 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1938                          struct action *action __rte_unused,
1939                          char **tokens,
1940                          int n_tokens,
1941                          struct instruction *instr,
1942                          struct instruction_data *data __rte_unused)
1943 {
1944         struct header *h;
1945
1946         CHECK(n_tokens == 2, EINVAL);
1947
1948         h = header_parse(p, tokens[1]);
1949         CHECK(h, EINVAL);
1950
1951         instr->type = INSTR_HDR_EMIT;
1952         instr->io.hdr.header_id[0] = h->id;
1953         instr->io.hdr.struct_id[0] = h->struct_id;
1954         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1955         return 0;
1956 }
1957
1958 static inline void
1959 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
1960
1961 static inline void
1962 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
1963 {
1964         struct thread *t = &p->threads[p->thread_id];
1965         struct instruction *ip = t->ip;
1966         uint32_t n_headers_out = t->n_headers_out;
1967         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
1968         uint8_t *ho_ptr = NULL;
1969         uint32_t ho_nbytes = 0, i;
1970
1971         for (i = 0; i < n_emit; i++) {
1972                 uint32_t header_id = ip->io.hdr.header_id[i];
1973                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1974                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1975
1976                 struct header_runtime *hi = &t->headers[header_id];
1977                 uint8_t *hi_ptr = t->structs[struct_id];
1978
1979                 TRACE("[Thread %2u]: emit header %u\n",
1980                       p->thread_id,
1981                       header_id);
1982
1983                 /* Headers. */
1984                 if (!i) {
1985                         if (!t->n_headers_out) {
1986                                 ho = &t->headers_out[0];
1987
1988                                 ho->ptr0 = hi->ptr0;
1989                                 ho->ptr = hi_ptr;
1990
1991                                 ho_ptr = hi_ptr;
1992                                 ho_nbytes = n_bytes;
1993
1994                                 n_headers_out = 1;
1995
1996                                 continue;
1997                         } else {
1998                                 ho_ptr = ho->ptr;
1999                                 ho_nbytes = ho->n_bytes;
2000                         }
2001                 }
2002
2003                 if (ho_ptr + ho_nbytes == hi_ptr) {
2004                         ho_nbytes += n_bytes;
2005                 } else {
2006                         ho->n_bytes = ho_nbytes;
2007
2008                         ho++;
2009                         ho->ptr0 = hi->ptr0;
2010                         ho->ptr = hi_ptr;
2011
2012                         ho_ptr = hi_ptr;
2013                         ho_nbytes = n_bytes;
2014
2015                         n_headers_out++;
2016                 }
2017         }
2018
2019         ho->n_bytes = ho_nbytes;
2020         t->n_headers_out = n_headers_out;
2021 }
2022
2023 static inline void
2024 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2025 {
2026         __instr_hdr_emit_exec(p, 1);
2027
2028         /* Thread. */
2029         thread_ip_inc(p);
2030 }
2031
2032 static inline void
2033 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2034 {
2035         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2036               p->thread_id);
2037
2038         __instr_hdr_emit_exec(p, 1);
2039         instr_tx_exec(p);
2040 }
2041
2042 static inline void
2043 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2044 {
2045         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2046               p->thread_id);
2047
2048         __instr_hdr_emit_exec(p, 2);
2049         instr_tx_exec(p);
2050 }
2051
2052 static inline void
2053 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2054 {
2055         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2056               p->thread_id);
2057
2058         __instr_hdr_emit_exec(p, 3);
2059         instr_tx_exec(p);
2060 }
2061
2062 static inline void
2063 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2064 {
2065         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2066               p->thread_id);
2067
2068         __instr_hdr_emit_exec(p, 4);
2069         instr_tx_exec(p);
2070 }
2071
2072 static inline void
2073 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2074 {
2075         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2076               p->thread_id);
2077
2078         __instr_hdr_emit_exec(p, 5);
2079         instr_tx_exec(p);
2080 }
2081
2082 static inline void
2083 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2084 {
2085         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2086               p->thread_id);
2087
2088         __instr_hdr_emit_exec(p, 6);
2089         instr_tx_exec(p);
2090 }
2091
2092 static inline void
2093 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2094 {
2095         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2096               p->thread_id);
2097
2098         __instr_hdr_emit_exec(p, 7);
2099         instr_tx_exec(p);
2100 }
2101
2102 static inline void
2103 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2104 {
2105         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2106               p->thread_id);
2107
2108         __instr_hdr_emit_exec(p, 8);
2109         instr_tx_exec(p);
2110 }
2111
2112 /*
2113  * validate.
2114  */
2115 static int
2116 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2117                              struct action *action __rte_unused,
2118                              char **tokens,
2119                              int n_tokens,
2120                              struct instruction *instr,
2121                              struct instruction_data *data __rte_unused)
2122 {
2123         struct header *h;
2124
2125         CHECK(n_tokens == 2, EINVAL);
2126
2127         h = header_parse(p, tokens[1]);
2128         CHECK(h, EINVAL);
2129
2130         instr->type = INSTR_HDR_VALIDATE;
2131         instr->valid.header_id = h->id;
2132         return 0;
2133 }
2134
2135 static inline void
2136 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2137 {
2138         struct thread *t = &p->threads[p->thread_id];
2139         struct instruction *ip = t->ip;
2140         uint32_t header_id = ip->valid.header_id;
2141
2142         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2143
2144         /* Headers. */
2145         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2146
2147         /* Thread. */
2148         thread_ip_inc(p);
2149 }
2150
2151 /*
2152  * invalidate.
2153  */
2154 static int
2155 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2156                                struct action *action __rte_unused,
2157                                char **tokens,
2158                                int n_tokens,
2159                                struct instruction *instr,
2160                                struct instruction_data *data __rte_unused)
2161 {
2162         struct header *h;
2163
2164         CHECK(n_tokens == 2, EINVAL);
2165
2166         h = header_parse(p, tokens[1]);
2167         CHECK(h, EINVAL);
2168
2169         instr->type = INSTR_HDR_INVALIDATE;
2170         instr->valid.header_id = h->id;
2171         return 0;
2172 }
2173
2174 static inline void
2175 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2176 {
2177         struct thread *t = &p->threads[p->thread_id];
2178         struct instruction *ip = t->ip;
2179         uint32_t header_id = ip->valid.header_id;
2180
2181         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2182
2183         /* Headers. */
2184         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2185
2186         /* Thread. */
2187         thread_ip_inc(p);
2188 }
2189
2190 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
2191
2192 static int
2193 instr_translate(struct rte_swx_pipeline *p,
2194                 struct action *action,
2195                 char *string,
2196                 struct instruction *instr,
2197                 struct instruction_data *data)
2198 {
2199         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
2200         int n_tokens = 0, tpos = 0;
2201
2202         /* Parse the instruction string into tokens. */
2203         for ( ; ; ) {
2204                 char *token;
2205
2206                 token = strtok_r(string, " \t\v", &string);
2207                 if (!token)
2208                         break;
2209
2210                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
2211
2212                 tokens[n_tokens] = token;
2213                 n_tokens++;
2214         }
2215
2216         CHECK(n_tokens, EINVAL);
2217
2218         /* Handle the optional instruction label. */
2219         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
2220                 strcpy(data->label, tokens[0]);
2221
2222                 tpos += 2;
2223                 CHECK(n_tokens - tpos, EINVAL);
2224         }
2225
2226         /* Identify the instruction type. */
2227         if (!strcmp(tokens[tpos], "rx"))
2228                 return instr_rx_translate(p,
2229                                           action,
2230                                           &tokens[tpos],
2231                                           n_tokens - tpos,
2232                                           instr,
2233                                           data);
2234
2235         if (!strcmp(tokens[tpos], "tx"))
2236                 return instr_tx_translate(p,
2237                                           action,
2238                                           &tokens[tpos],
2239                                           n_tokens - tpos,
2240                                           instr,
2241                                           data);
2242
2243         if (!strcmp(tokens[tpos], "extract"))
2244                 return instr_hdr_extract_translate(p,
2245                                                    action,
2246                                                    &tokens[tpos],
2247                                                    n_tokens - tpos,
2248                                                    instr,
2249                                                    data);
2250
2251         if (!strcmp(tokens[tpos], "emit"))
2252                 return instr_hdr_emit_translate(p,
2253                                                 action,
2254                                                 &tokens[tpos],
2255                                                 n_tokens - tpos,
2256                                                 instr,
2257                                                 data);
2258
2259         if (!strcmp(tokens[tpos], "validate"))
2260                 return instr_hdr_validate_translate(p,
2261                                                     action,
2262                                                     &tokens[tpos],
2263                                                     n_tokens - tpos,
2264                                                     instr,
2265                                                     data);
2266
2267         if (!strcmp(tokens[tpos], "invalidate"))
2268                 return instr_hdr_invalidate_translate(p,
2269                                                       action,
2270                                                       &tokens[tpos],
2271                                                       n_tokens - tpos,
2272                                                       instr,
2273                                                       data);
2274
2275         CHECK(0, EINVAL);
2276 }
2277
2278 static uint32_t
2279 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
2280 {
2281         uint32_t count = 0, i;
2282
2283         if (!label[0])
2284                 return 0;
2285
2286         for (i = 0; i < n; i++)
2287                 if (!strcmp(label, data[i].jmp_label))
2288                         count++;
2289
2290         return count;
2291 }
2292
2293 static int
2294 instr_label_check(struct instruction_data *instruction_data,
2295                   uint32_t n_instructions)
2296 {
2297         uint32_t i;
2298
2299         /* Check that all instruction labels are unique. */
2300         for (i = 0; i < n_instructions; i++) {
2301                 struct instruction_data *data = &instruction_data[i];
2302                 char *label = data->label;
2303                 uint32_t j;
2304
2305                 if (!label[0])
2306                         continue;
2307
2308                 for (j = i + 1; j < n_instructions; j++)
2309                         CHECK(strcmp(label, data[j].label), EINVAL);
2310         }
2311
2312         /* Get users for each instruction label. */
2313         for (i = 0; i < n_instructions; i++) {
2314                 struct instruction_data *data = &instruction_data[i];
2315                 char *label = data->label;
2316
2317                 data->n_users = label_is_used(instruction_data,
2318                                               n_instructions,
2319                                               label);
2320         }
2321
2322         return 0;
2323 }
2324
2325 static int
2326 instruction_config(struct rte_swx_pipeline *p,
2327                    struct action *a,
2328                    const char **instructions,
2329                    uint32_t n_instructions)
2330 {
2331         struct instruction *instr = NULL;
2332         struct instruction_data *data = NULL;
2333         char *string = NULL;
2334         int err = 0;
2335         uint32_t i;
2336
2337         CHECK(n_instructions, EINVAL);
2338         CHECK(instructions, EINVAL);
2339         for (i = 0; i < n_instructions; i++)
2340                 CHECK(instructions[i], EINVAL);
2341
2342         /* Memory allocation. */
2343         instr = calloc(n_instructions, sizeof(struct instruction));
2344         if (!instr) {
2345                 err = ENOMEM;
2346                 goto error;
2347         }
2348
2349         data = calloc(n_instructions, sizeof(struct instruction_data));
2350         if (!data) {
2351                 err = ENOMEM;
2352                 goto error;
2353         }
2354
2355         for (i = 0; i < n_instructions; i++) {
2356                 string = strdup(instructions[i]);
2357                 if (!string) {
2358                         err = ENOMEM;
2359                         goto error;
2360                 }
2361
2362                 err = instr_translate(p, a, string, &instr[i], &data[i]);
2363                 if (err)
2364                         goto error;
2365
2366                 free(string);
2367         }
2368
2369         err = instr_label_check(data, n_instructions);
2370         if (err)
2371                 goto error;
2372
2373         free(data);
2374
2375         if (a) {
2376                 a->instructions = instr;
2377                 a->n_instructions = n_instructions;
2378         } else {
2379                 p->instructions = instr;
2380                 p->n_instructions = n_instructions;
2381         }
2382
2383         return 0;
2384
2385 error:
2386         free(string);
2387         free(data);
2388         free(instr);
2389         return err;
2390 }
2391
2392 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
2393
2394 static instr_exec_t instruction_table[] = {
2395         [INSTR_RX] = instr_rx_exec,
2396         [INSTR_TX] = instr_tx_exec,
2397
2398         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
2399         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
2400         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
2401         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
2402         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
2403         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
2404         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
2405         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
2406
2407         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
2408         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
2409         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
2410         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
2411         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
2412         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
2413         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
2414         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
2415         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
2416
2417         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
2418         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
2419 };
2420
2421 static inline void
2422 instr_exec(struct rte_swx_pipeline *p)
2423 {
2424         struct thread *t = &p->threads[p->thread_id];
2425         struct instruction *ip = t->ip;
2426         instr_exec_t instr = instruction_table[ip->type];
2427
2428         instr(p);
2429 }
2430
2431 /*
2432  * Action.
2433  */
2434 static struct action *
2435 action_find(struct rte_swx_pipeline *p, const char *name)
2436 {
2437         struct action *elem;
2438
2439         if (!name)
2440                 return NULL;
2441
2442         TAILQ_FOREACH(elem, &p->actions, node)
2443                 if (strcmp(elem->name, name) == 0)
2444                         return elem;
2445
2446         return NULL;
2447 }
2448
2449 int
2450 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
2451                                const char *name,
2452                                const char *args_struct_type_name,
2453                                const char **instructions,
2454                                uint32_t n_instructions)
2455 {
2456         struct struct_type *args_struct_type;
2457         struct action *a;
2458         int err;
2459
2460         CHECK(p, EINVAL);
2461
2462         CHECK_NAME(name, EINVAL);
2463         CHECK(!action_find(p, name), EEXIST);
2464
2465         if (args_struct_type_name) {
2466                 CHECK_NAME(args_struct_type_name, EINVAL);
2467                 args_struct_type = struct_type_find(p, args_struct_type_name);
2468                 CHECK(args_struct_type, EINVAL);
2469         } else {
2470                 args_struct_type = NULL;
2471         }
2472
2473         /* Node allocation. */
2474         a = calloc(1, sizeof(struct action));
2475         CHECK(a, ENOMEM);
2476
2477         /* Node initialization. */
2478         strcpy(a->name, name);
2479         a->st = args_struct_type;
2480         a->id = p->n_actions;
2481
2482         /* Instruction translation. */
2483         err = instruction_config(p, a, instructions, n_instructions);
2484         if (err) {
2485                 free(a);
2486                 return err;
2487         }
2488
2489         /* Node add to tailq. */
2490         TAILQ_INSERT_TAIL(&p->actions, a, node);
2491         p->n_actions++;
2492
2493         return 0;
2494 }
2495
2496 static int
2497 action_build(struct rte_swx_pipeline *p)
2498 {
2499         struct action *action;
2500
2501         p->action_instructions = calloc(p->n_actions,
2502                                         sizeof(struct instruction *));
2503         CHECK(p->action_instructions, ENOMEM);
2504
2505         TAILQ_FOREACH(action, &p->actions, node)
2506                 p->action_instructions[action->id] = action->instructions;
2507
2508         return 0;
2509 }
2510
2511 static void
2512 action_build_free(struct rte_swx_pipeline *p)
2513 {
2514         free(p->action_instructions);
2515         p->action_instructions = NULL;
2516 }
2517
2518 static void
2519 action_free(struct rte_swx_pipeline *p)
2520 {
2521         action_build_free(p);
2522
2523         for ( ; ; ) {
2524                 struct action *action;
2525
2526                 action = TAILQ_FIRST(&p->actions);
2527                 if (!action)
2528                         break;
2529
2530                 TAILQ_REMOVE(&p->actions, action, node);
2531                 free(action->instructions);
2532                 free(action);
2533         }
2534 }
2535
2536 /*
2537  * Table.
2538  */
2539 static struct table_type *
2540 table_type_find(struct rte_swx_pipeline *p, const char *name)
2541 {
2542         struct table_type *elem;
2543
2544         TAILQ_FOREACH(elem, &p->table_types, node)
2545                 if (strcmp(elem->name, name) == 0)
2546                         return elem;
2547
2548         return NULL;
2549 }
2550
2551 static struct table_type *
2552 table_type_resolve(struct rte_swx_pipeline *p,
2553                    const char *recommended_type_name,
2554                    enum rte_swx_table_match_type match_type)
2555 {
2556         struct table_type *elem;
2557
2558         /* Only consider the recommended type if the match type is correct. */
2559         if (recommended_type_name)
2560                 TAILQ_FOREACH(elem, &p->table_types, node)
2561                         if (!strcmp(elem->name, recommended_type_name) &&
2562                             (elem->match_type == match_type))
2563                                 return elem;
2564
2565         /* Ignore the recommended type and get the first element with this match
2566          * type.
2567          */
2568         TAILQ_FOREACH(elem, &p->table_types, node)
2569                 if (elem->match_type == match_type)
2570                         return elem;
2571
2572         return NULL;
2573 }
2574
2575 static struct table *
2576 table_find(struct rte_swx_pipeline *p, const char *name)
2577 {
2578         struct table *elem;
2579
2580         TAILQ_FOREACH(elem, &p->tables, node)
2581                 if (strcmp(elem->name, name) == 0)
2582                         return elem;
2583
2584         return NULL;
2585 }
2586
2587 static struct table *
2588 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
2589 {
2590         struct table *table = NULL;
2591
2592         TAILQ_FOREACH(table, &p->tables, node)
2593                 if (table->id == id)
2594                         return table;
2595
2596         return NULL;
2597 }
2598
2599 int
2600 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
2601                                      const char *name,
2602                                      enum rte_swx_table_match_type match_type,
2603                                      struct rte_swx_table_ops *ops)
2604 {
2605         struct table_type *elem;
2606
2607         CHECK(p, EINVAL);
2608
2609         CHECK_NAME(name, EINVAL);
2610         CHECK(!table_type_find(p, name), EEXIST);
2611
2612         CHECK(ops, EINVAL);
2613         CHECK(ops->create, EINVAL);
2614         CHECK(ops->lkp, EINVAL);
2615         CHECK(ops->free, EINVAL);
2616
2617         /* Node allocation. */
2618         elem = calloc(1, sizeof(struct table_type));
2619         CHECK(elem, ENOMEM);
2620
2621         /* Node initialization. */
2622         strcpy(elem->name, name);
2623         elem->match_type = match_type;
2624         memcpy(&elem->ops, ops, sizeof(*ops));
2625
2626         /* Node add to tailq. */
2627         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
2628
2629         return 0;
2630 }
2631
2632 static enum rte_swx_table_match_type
2633 table_match_type_resolve(struct rte_swx_match_field_params *fields,
2634                          uint32_t n_fields)
2635 {
2636         uint32_t i;
2637
2638         for (i = 0; i < n_fields; i++)
2639                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
2640                         break;
2641
2642         if (i == n_fields)
2643                 return RTE_SWX_TABLE_MATCH_EXACT;
2644
2645         if ((i == n_fields - 1) &&
2646             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
2647                 return RTE_SWX_TABLE_MATCH_LPM;
2648
2649         return RTE_SWX_TABLE_MATCH_WILDCARD;
2650 }
2651
2652 int
2653 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
2654                               const char *name,
2655                               struct rte_swx_pipeline_table_params *params,
2656                               const char *recommended_table_type_name,
2657                               const char *args,
2658                               uint32_t size)
2659 {
2660         struct table_type *type;
2661         struct table *t;
2662         struct action *default_action;
2663         struct header *header = NULL;
2664         int is_header = 0;
2665         uint32_t offset_prev = 0, action_data_size_max = 0, i;
2666
2667         CHECK(p, EINVAL);
2668
2669         CHECK_NAME(name, EINVAL);
2670         CHECK(!table_find(p, name), EEXIST);
2671
2672         CHECK(params, EINVAL);
2673
2674         /* Match checks. */
2675         CHECK(!params->n_fields || params->fields, EINVAL);
2676         for (i = 0; i < params->n_fields; i++) {
2677                 struct rte_swx_match_field_params *field = &params->fields[i];
2678                 struct header *h;
2679                 struct field *hf, *mf;
2680                 uint32_t offset;
2681
2682                 CHECK_NAME(field->name, EINVAL);
2683
2684                 hf = header_field_parse(p, field->name, &h);
2685                 mf = metadata_field_parse(p, field->name);
2686                 CHECK(hf || mf, EINVAL);
2687
2688                 offset = hf ? hf->offset : mf->offset;
2689
2690                 if (i == 0) {
2691                         is_header = hf ? 1 : 0;
2692                         header = hf ? h : NULL;
2693                         offset_prev = offset;
2694
2695                         continue;
2696                 }
2697
2698                 CHECK((is_header && hf && (h->id == header->id)) ||
2699                       (!is_header && mf), EINVAL);
2700
2701                 CHECK(offset > offset_prev, EINVAL);
2702                 offset_prev = offset;
2703         }
2704
2705         /* Action checks. */
2706         CHECK(params->n_actions, EINVAL);
2707         CHECK(params->action_names, EINVAL);
2708         for (i = 0; i < params->n_actions; i++) {
2709                 const char *action_name = params->action_names[i];
2710                 struct action *a;
2711                 uint32_t action_data_size;
2712
2713                 CHECK(action_name, EINVAL);
2714
2715                 a = action_find(p, action_name);
2716                 CHECK(a, EINVAL);
2717
2718                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
2719                 if (action_data_size > action_data_size_max)
2720                         action_data_size_max = action_data_size;
2721         }
2722
2723         CHECK(params->default_action_name, EINVAL);
2724         for (i = 0; i < p->n_actions; i++)
2725                 if (!strcmp(params->action_names[i],
2726                             params->default_action_name))
2727                         break;
2728         CHECK(i < params->n_actions, EINVAL);
2729         default_action = action_find(p, params->default_action_name);
2730         CHECK((default_action->st && params->default_action_data) ||
2731               !params->default_action_data, EINVAL);
2732
2733         /* Table type checks. */
2734         if (params->n_fields) {
2735                 enum rte_swx_table_match_type match_type;
2736
2737                 match_type = table_match_type_resolve(params->fields,
2738                                                       params->n_fields);
2739                 type = table_type_resolve(p,
2740                                           recommended_table_type_name,
2741                                           match_type);
2742                 CHECK(type, EINVAL);
2743         } else {
2744                 type = NULL;
2745         }
2746
2747         /* Memory allocation. */
2748         t = calloc(1, sizeof(struct table));
2749         CHECK(t, ENOMEM);
2750
2751         t->fields = calloc(params->n_fields, sizeof(struct match_field));
2752         if (!t->fields) {
2753                 free(t);
2754                 CHECK(0, ENOMEM);
2755         }
2756
2757         t->actions = calloc(params->n_actions, sizeof(struct action *));
2758         if (!t->actions) {
2759                 free(t->fields);
2760                 free(t);
2761                 CHECK(0, ENOMEM);
2762         }
2763
2764         if (action_data_size_max) {
2765                 t->default_action_data = calloc(1, action_data_size_max);
2766                 if (!t->default_action_data) {
2767                         free(t->actions);
2768                         free(t->fields);
2769                         free(t);
2770                         CHECK(0, ENOMEM);
2771                 }
2772         }
2773
2774         /* Node initialization. */
2775         strcpy(t->name, name);
2776         if (args && args[0])
2777                 strcpy(t->args, args);
2778         t->type = type;
2779
2780         for (i = 0; i < params->n_fields; i++) {
2781                 struct rte_swx_match_field_params *field = &params->fields[i];
2782                 struct match_field *f = &t->fields[i];
2783
2784                 f->match_type = field->match_type;
2785                 f->field = is_header ?
2786                         header_field_parse(p, field->name, NULL) :
2787                         metadata_field_parse(p, field->name);
2788         }
2789         t->n_fields = params->n_fields;
2790         t->is_header = is_header;
2791         t->header = header;
2792
2793         for (i = 0; i < params->n_actions; i++)
2794                 t->actions[i] = action_find(p, params->action_names[i]);
2795         t->default_action = default_action;
2796         if (default_action->st)
2797                 memcpy(t->default_action_data,
2798                        params->default_action_data,
2799                        default_action->st->n_bits / 8);
2800         t->n_actions = params->n_actions;
2801         t->default_action_is_const = params->default_action_is_const;
2802         t->action_data_size_max = action_data_size_max;
2803
2804         t->size = size;
2805         t->id = p->n_tables;
2806
2807         /* Node add to tailq. */
2808         TAILQ_INSERT_TAIL(&p->tables, t, node);
2809         p->n_tables++;
2810
2811         return 0;
2812 }
2813
2814 static struct rte_swx_table_params *
2815 table_params_get(struct table *table)
2816 {
2817         struct rte_swx_table_params *params;
2818         struct field *first, *last;
2819         uint8_t *key_mask;
2820         uint32_t key_size, key_offset, action_data_size, i;
2821
2822         /* Memory allocation. */
2823         params = calloc(1, sizeof(struct rte_swx_table_params));
2824         if (!params)
2825                 return NULL;
2826
2827         /* Key offset and size. */
2828         first = table->fields[0].field;
2829         last = table->fields[table->n_fields - 1].field;
2830         key_offset = first->offset / 8;
2831         key_size = (last->offset + last->n_bits - first->offset) / 8;
2832
2833         /* Memory allocation. */
2834         key_mask = calloc(1, key_size);
2835         if (!key_mask) {
2836                 free(params);
2837                 return NULL;
2838         }
2839
2840         /* Key mask. */
2841         for (i = 0; i < table->n_fields; i++) {
2842                 struct field *f = table->fields[i].field;
2843                 uint32_t start = (f->offset - first->offset) / 8;
2844                 size_t size = f->n_bits / 8;
2845
2846                 memset(&key_mask[start], 0xFF, size);
2847         }
2848
2849         /* Action data size. */
2850         action_data_size = 0;
2851         for (i = 0; i < table->n_actions; i++) {
2852                 struct action *action = table->actions[i];
2853                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
2854
2855                 if (ads > action_data_size)
2856                         action_data_size = ads;
2857         }
2858
2859         /* Fill in. */
2860         params->match_type = table->type->match_type;
2861         params->key_size = key_size;
2862         params->key_offset = key_offset;
2863         params->key_mask0 = key_mask;
2864         params->action_data_size = action_data_size;
2865         params->n_keys_max = table->size;
2866
2867         return params;
2868 }
2869
2870 static void
2871 table_params_free(struct rte_swx_table_params *params)
2872 {
2873         if (!params)
2874                 return;
2875
2876         free(params->key_mask0);
2877         free(params);
2878 }
2879
2880 static int
2881 table_state_build(struct rte_swx_pipeline *p)
2882 {
2883         struct table *table;
2884
2885         p->table_state = calloc(p->n_tables,
2886                                 sizeof(struct rte_swx_table_state));
2887         CHECK(p->table_state, ENOMEM);
2888
2889         TAILQ_FOREACH(table, &p->tables, node) {
2890                 struct rte_swx_table_state *ts = &p->table_state[table->id];
2891
2892                 if (table->type) {
2893                         struct rte_swx_table_params *params;
2894
2895                         /* ts->obj. */
2896                         params = table_params_get(table);
2897                         CHECK(params, ENOMEM);
2898
2899                         ts->obj = table->type->ops.create(params,
2900                                 NULL,
2901                                 table->args,
2902                                 p->numa_node);
2903
2904                         table_params_free(params);
2905                         CHECK(ts->obj, ENODEV);
2906                 }
2907
2908                 /* ts->default_action_data. */
2909                 if (table->action_data_size_max) {
2910                         ts->default_action_data =
2911                                 malloc(table->action_data_size_max);
2912                         CHECK(ts->default_action_data, ENOMEM);
2913
2914                         memcpy(ts->default_action_data,
2915                                table->default_action_data,
2916                                table->action_data_size_max);
2917                 }
2918
2919                 /* ts->default_action_id. */
2920                 ts->default_action_id = table->default_action->id;
2921         }
2922
2923         return 0;
2924 }
2925
2926 static void
2927 table_state_build_free(struct rte_swx_pipeline *p)
2928 {
2929         uint32_t i;
2930
2931         if (!p->table_state)
2932                 return;
2933
2934         for (i = 0; i < p->n_tables; i++) {
2935                 struct rte_swx_table_state *ts = &p->table_state[i];
2936                 struct table *table = table_find_by_id(p, i);
2937
2938                 /* ts->obj. */
2939                 if (table->type && ts->obj)
2940                         table->type->ops.free(ts->obj);
2941
2942                 /* ts->default_action_data. */
2943                 free(ts->default_action_data);
2944         }
2945
2946         free(p->table_state);
2947         p->table_state = NULL;
2948 }
2949
2950 static void
2951 table_state_free(struct rte_swx_pipeline *p)
2952 {
2953         table_state_build_free(p);
2954 }
2955
2956 static int
2957 table_stub_lkp(void *table __rte_unused,
2958                void *mailbox __rte_unused,
2959                uint8_t **key __rte_unused,
2960                uint64_t *action_id __rte_unused,
2961                uint8_t **action_data __rte_unused,
2962                int *hit)
2963 {
2964         *hit = 0;
2965         return 1; /* DONE. */
2966 }
2967
2968 static int
2969 table_build(struct rte_swx_pipeline *p)
2970 {
2971         uint32_t i;
2972
2973         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2974                 struct thread *t = &p->threads[i];
2975                 struct table *table;
2976
2977                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
2978                 CHECK(t->tables, ENOMEM);
2979
2980                 TAILQ_FOREACH(table, &p->tables, node) {
2981                         struct table_runtime *r = &t->tables[table->id];
2982
2983                         if (table->type) {
2984                                 uint64_t size;
2985
2986                                 size = table->type->ops.mailbox_size_get();
2987
2988                                 /* r->func. */
2989                                 r->func = table->type->ops.lkp;
2990
2991                                 /* r->mailbox. */
2992                                 if (size) {
2993                                         r->mailbox = calloc(1, size);
2994                                         CHECK(r->mailbox, ENOMEM);
2995                                 }
2996
2997                                 /* r->key. */
2998                                 r->key = table->is_header ?
2999                                         &t->structs[table->header->struct_id] :
3000                                         &t->structs[p->metadata_struct_id];
3001                         } else {
3002                                 r->func = table_stub_lkp;
3003                         }
3004                 }
3005         }
3006
3007         return 0;
3008 }
3009
3010 static void
3011 table_build_free(struct rte_swx_pipeline *p)
3012 {
3013         uint32_t i;
3014
3015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3016                 struct thread *t = &p->threads[i];
3017                 uint32_t j;
3018
3019                 if (!t->tables)
3020                         continue;
3021
3022                 for (j = 0; j < p->n_tables; j++) {
3023                         struct table_runtime *r = &t->tables[j];
3024
3025                         free(r->mailbox);
3026                 }
3027
3028                 free(t->tables);
3029                 t->tables = NULL;
3030         }
3031 }
3032
3033 static void
3034 table_free(struct rte_swx_pipeline *p)
3035 {
3036         table_build_free(p);
3037
3038         /* Tables. */
3039         for ( ; ; ) {
3040                 struct table *elem;
3041
3042                 elem = TAILQ_FIRST(&p->tables);
3043                 if (!elem)
3044                         break;
3045
3046                 TAILQ_REMOVE(&p->tables, elem, node);
3047                 free(elem->fields);
3048                 free(elem->actions);
3049                 free(elem->default_action_data);
3050                 free(elem);
3051         }
3052
3053         /* Table types. */
3054         for ( ; ; ) {
3055                 struct table_type *elem;
3056
3057                 elem = TAILQ_FIRST(&p->table_types);
3058                 if (!elem)
3059                         break;
3060
3061                 TAILQ_REMOVE(&p->table_types, elem, node);
3062                 free(elem);
3063         }
3064 }
3065
3066 /*
3067  * Pipeline.
3068  */
3069 int
3070 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
3071 {
3072         struct rte_swx_pipeline *pipeline;
3073
3074         /* Check input parameters. */
3075         CHECK(p, EINVAL);
3076
3077         /* Memory allocation. */
3078         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
3079         CHECK(pipeline, ENOMEM);
3080
3081         /* Initialization. */
3082         TAILQ_INIT(&pipeline->struct_types);
3083         TAILQ_INIT(&pipeline->port_in_types);
3084         TAILQ_INIT(&pipeline->ports_in);
3085         TAILQ_INIT(&pipeline->port_out_types);
3086         TAILQ_INIT(&pipeline->ports_out);
3087         TAILQ_INIT(&pipeline->extern_types);
3088         TAILQ_INIT(&pipeline->extern_objs);
3089         TAILQ_INIT(&pipeline->extern_funcs);
3090         TAILQ_INIT(&pipeline->headers);
3091         TAILQ_INIT(&pipeline->actions);
3092         TAILQ_INIT(&pipeline->table_types);
3093         TAILQ_INIT(&pipeline->tables);
3094
3095         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
3096         pipeline->numa_node = numa_node;
3097
3098         *p = pipeline;
3099         return 0;
3100 }
3101
3102 void
3103 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
3104 {
3105         if (!p)
3106                 return;
3107
3108         free(p->instructions);
3109
3110         table_state_free(p);
3111         table_free(p);
3112         action_free(p);
3113         metadata_free(p);
3114         header_free(p);
3115         extern_func_free(p);
3116         extern_obj_free(p);
3117         port_out_free(p);
3118         port_in_free(p);
3119         struct_free(p);
3120
3121         free(p);
3122 }
3123
3124 int
3125 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
3126                                      const char **instructions,
3127                                      uint32_t n_instructions)
3128 {
3129         int err;
3130         uint32_t i;
3131
3132         err = instruction_config(p, NULL, instructions, n_instructions);
3133         if (err)
3134                 return err;
3135
3136         /* Thread instruction pointer reset. */
3137         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3138                 struct thread *t = &p->threads[i];
3139
3140                 thread_ip_reset(p, t);
3141         }
3142
3143         return 0;
3144 }
3145
3146 int
3147 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
3148 {
3149         int status;
3150
3151         CHECK(p, EINVAL);
3152         CHECK(p->build_done == 0, EEXIST);
3153
3154         status = port_in_build(p);
3155         if (status)
3156                 goto error;
3157
3158         status = port_out_build(p);
3159         if (status)
3160                 goto error;
3161
3162         status = struct_build(p);
3163         if (status)
3164                 goto error;
3165
3166         status = extern_obj_build(p);
3167         if (status)
3168                 goto error;
3169
3170         status = extern_func_build(p);
3171         if (status)
3172                 goto error;
3173
3174         status = header_build(p);
3175         if (status)
3176                 goto error;
3177
3178         status = metadata_build(p);
3179         if (status)
3180                 goto error;
3181
3182         status = action_build(p);
3183         if (status)
3184                 goto error;
3185
3186         status = table_build(p);
3187         if (status)
3188                 goto error;
3189
3190         status = table_state_build(p);
3191         if (status)
3192                 goto error;
3193
3194         p->build_done = 1;
3195         return 0;
3196
3197 error:
3198         table_state_build_free(p);
3199         table_build_free(p);
3200         action_build_free(p);
3201         metadata_build_free(p);
3202         header_build_free(p);
3203         extern_func_build_free(p);
3204         extern_obj_build_free(p);
3205         port_out_build_free(p);
3206         port_in_build_free(p);
3207         struct_build_free(p);
3208
3209         return status;
3210 }
3211
3212 void
3213 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
3214 {
3215         uint32_t i;
3216
3217         for (i = 0; i < n_instructions; i++)
3218                 instr_exec(p);
3219 }
3220
3221 /*
3222  * Control.
3223  */
3224 int
3225 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
3226                                  struct rte_swx_table_state **table_state)
3227 {
3228         if (!p || !table_state || !p->build_done)
3229                 return -EINVAL;
3230
3231         *table_state = p->table_state;
3232         return 0;
3233 }
3234
3235 int
3236 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
3237                                  struct rte_swx_table_state *table_state)
3238 {
3239         if (!p || !table_state || !p->build_done)
3240                 return -EINVAL;
3241
3242         p->table_state = table_state;
3243         return 0;
3244 }