lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9
  10 #include <rte_common.h>
  11 #include <rte_prefetch.h>
  12
  13 #include "rte_swx_pipeline.h"
  14 #include "rte_swx_ctl.h"
  15
  16 #define CHECK(condition, err_code)                                             \
  17 do {                                                                           \
  18         if (!(condition))                                                      \
  19                 return -(err_code);                                            \
  20 } while (0)
  21
  22 #define CHECK_NAME(name, err_code)                                             \
  23         CHECK((name) && (name)[0], err_code)
  24
  25 #ifndef TRACE_LEVEL
  26 #define TRACE_LEVEL 0
  27 #endif
  28
  29 #if TRACE_LEVEL
  30 #define TRACE(...) printf(__VA_ARGS__)
  31 #else
  32 #define TRACE(...)
  33 #endif
  34
  35 /*
  36  * Struct.
  37  */
  38 struct field {
  39         char name[RTE_SWX_NAME_SIZE];
  40         uint32_t n_bits;
  41         uint32_t offset;
  42 };
  43
  44 struct struct_type {
  45         TAILQ_ENTRY(struct_type) node;
  46         char name[RTE_SWX_NAME_SIZE];
  47         struct field *fields;
  48         uint32_t n_fields;
  49         uint32_t n_bits;
  50 };
  51
  52 TAILQ_HEAD(struct_type_tailq, struct_type);
  53
  54 /*
  55  * Input port.
  56  */
  57 struct port_in_type {
  58         TAILQ_ENTRY(port_in_type) node;
  59         char name[RTE_SWX_NAME_SIZE];
  60         struct rte_swx_port_in_ops ops;
  61 };
  62
  63 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  64
  65 struct port_in {
  66         TAILQ_ENTRY(port_in) node;
  67         struct port_in_type *type;
  68         void *obj;
  69         uint32_t id;
  70 };
  71
  72 TAILQ_HEAD(port_in_tailq, port_in);
  73
  74 struct port_in_runtime {
  75         rte_swx_port_in_pkt_rx_t pkt_rx;
  76         void *obj;
  77 };
  78
  79 /*
  80  * Output port.
  81  */
  82 struct port_out_type {
  83         TAILQ_ENTRY(port_out_type) node;
  84         char name[RTE_SWX_NAME_SIZE];
  85         struct rte_swx_port_out_ops ops;
  86 };
  87
  88 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  89
  90 struct port_out {
  91         TAILQ_ENTRY(port_out) node;
  92         struct port_out_type *type;
  93         void *obj;
  94         uint32_t id;
  95 };
  96
  97 TAILQ_HEAD(port_out_tailq, port_out);
  98
  99 struct port_out_runtime {
 100         rte_swx_port_out_pkt_tx_t pkt_tx;
 101         rte_swx_port_out_flush_t flush;
 102         void *obj;
 103 };
 104
 105 /*
 106  * Extern object.
 107  */
 108 struct extern_type_member_func {
 109         TAILQ_ENTRY(extern_type_member_func) node;
 110         char name[RTE_SWX_NAME_SIZE];
 111         rte_swx_extern_type_member_func_t func;
 112         uint32_t id;
 113 };
 114
 115 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 116
 117 struct extern_type {
 118         TAILQ_ENTRY(extern_type) node;
 119         char name[RTE_SWX_NAME_SIZE];
 120         struct struct_type *mailbox_struct_type;
 121         rte_swx_extern_type_constructor_t constructor;
 122         rte_swx_extern_type_destructor_t destructor;
 123         struct extern_type_member_func_tailq funcs;
 124         uint32_t n_funcs;
 125 };
 126
 127 TAILQ_HEAD(extern_type_tailq, extern_type);
 128
 129 struct extern_obj {
 130         TAILQ_ENTRY(extern_obj) node;
 131         char name[RTE_SWX_NAME_SIZE];
 132         struct extern_type *type;
 133         void *obj;
 134         uint32_t struct_id;
 135         uint32_t id;
 136 };
 137
 138 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 139
 140 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 141 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 142 #endif
 143
 144 struct extern_obj_runtime {
 145         void *obj;
 146         uint8_t *mailbox;
 147         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 148 };
 149
 150 /*
 151  * Extern function.
 152  */
 153 struct extern_func {
 154         TAILQ_ENTRY(extern_func) node;
 155         char name[RTE_SWX_NAME_SIZE];
 156         struct struct_type *mailbox_struct_type;
 157         rte_swx_extern_func_t func;
 158         uint32_t struct_id;
 159         uint32_t id;
 160 };
 161
 162 TAILQ_HEAD(extern_func_tailq, extern_func);
 163
 164 struct extern_func_runtime {
 165         uint8_t *mailbox;
 166         rte_swx_extern_func_t func;
 167 };
 168
 169 /*
 170  * Header.
 171  */
 172 struct header {
 173         TAILQ_ENTRY(header) node;
 174         char name[RTE_SWX_NAME_SIZE];
 175         struct struct_type *st;
 176         uint32_t struct_id;
 177         uint32_t id;
 178 };
 179
 180 TAILQ_HEAD(header_tailq, header);
 181
 182 struct header_runtime {
 183         uint8_t *ptr0;
 184 };
 185
 186 struct header_out_runtime {
 187         uint8_t *ptr0;
 188         uint8_t *ptr;
 189         uint32_t n_bytes;
 190 };
 191
 192 /*
 193  * Instruction.
 194  */
 195
 196 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 197  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 198  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 199  * when transferred to packet meta-data and in NBO when transferred to packet
 200  * headers.
 201  */
 202
 203 /* Notation conventions:
 204  *    -Header field: H = h.header.field (dst/src)
 205  *    -Meta-data field: M = m.field (dst/src)
 206  *    -Extern object mailbox field: E = e.field (dst/src)
 207  *    -Extern function mailbox field: F = f.field (dst/src)
 208  *    -Table action data field: T = t.field (src only)
 209  *    -Immediate value: I = 32-bit unsigned value (src only)
 210  */
 211
 212 enum instruction_type {
 213         /* rx m.port_in */
 214         INSTR_RX,
 215
 216         /* tx m.port_out */
 217         INSTR_TX,
 218
 219         /* extract h.header */
 220         INSTR_HDR_EXTRACT,
 221         INSTR_HDR_EXTRACT2,
 222         INSTR_HDR_EXTRACT3,
 223         INSTR_HDR_EXTRACT4,
 224         INSTR_HDR_EXTRACT5,
 225         INSTR_HDR_EXTRACT6,
 226         INSTR_HDR_EXTRACT7,
 227         INSTR_HDR_EXTRACT8,
 228
 229         /* emit h.header */
 230         INSTR_HDR_EMIT,
 231         INSTR_HDR_EMIT_TX,
 232         INSTR_HDR_EMIT2_TX,
 233         INSTR_HDR_EMIT3_TX,
 234         INSTR_HDR_EMIT4_TX,
 235         INSTR_HDR_EMIT5_TX,
 236         INSTR_HDR_EMIT6_TX,
 237         INSTR_HDR_EMIT7_TX,
 238         INSTR_HDR_EMIT8_TX,
 239 };
 240
 241 struct instr_io {
 242         struct {
 243                 uint8_t offset;
 244                 uint8_t n_bits;
 245                 uint8_t pad[2];
 246         } io;
 247
 248         struct {
 249                 uint8_t header_id[8];
 250                 uint8_t struct_id[8];
 251                 uint8_t n_bytes[8];
 252         } hdr;
 253 };
 254
 255 struct instruction {
 256         enum instruction_type type;
 257         union {
 258                 struct instr_io io;
 259         };
 260 };
 261
 262 struct instruction_data {
 263         char label[RTE_SWX_NAME_SIZE];
 264         char jmp_label[RTE_SWX_NAME_SIZE];
 265         uint32_t n_users; /* user = jmp instruction to this instruction. */
 266         int invalid;
 267 };
 268
 269 /*
 270  * Action.
 271  */
 272 struct action {
 273         TAILQ_ENTRY(action) node;
 274         char name[RTE_SWX_NAME_SIZE];
 275         struct struct_type *st;
 276         struct instruction *instructions;
 277         uint32_t n_instructions;
 278         uint32_t id;
 279 };
 280
 281 TAILQ_HEAD(action_tailq, action);
 282
 283 /*
 284  * Table.
 285  */
 286 struct table_type {
 287         TAILQ_ENTRY(table_type) node;
 288         char name[RTE_SWX_NAME_SIZE];
 289         enum rte_swx_table_match_type match_type;
 290         struct rte_swx_table_ops ops;
 291 };
 292
 293 TAILQ_HEAD(table_type_tailq, table_type);
 294
 295 struct match_field {
 296         enum rte_swx_table_match_type match_type;
 297         struct field *field;
 298 };
 299
 300 struct table {
 301         TAILQ_ENTRY(table) node;
 302         char name[RTE_SWX_NAME_SIZE];
 303         char args[RTE_SWX_NAME_SIZE];
 304         struct table_type *type; /* NULL when n_fields == 0. */
 305
 306         /* Match. */
 307         struct match_field *fields;
 308         uint32_t n_fields;
 309         int is_header; /* Only valid when n_fields > 0. */
 310         struct header *header; /* Only valid when n_fields > 0. */
 311
 312         /* Action. */
 313         struct action **actions;
 314         struct action *default_action;
 315         uint8_t *default_action_data;
 316         uint32_t n_actions;
 317         int default_action_is_const;
 318         uint32_t action_data_size_max;
 319
 320         uint32_t size;
 321         uint32_t id;
 322 };
 323
 324 TAILQ_HEAD(table_tailq, table);
 325
 326 struct table_runtime {
 327         rte_swx_table_lookup_t func;
 328         void *mailbox;
 329         uint8_t **key;
 330 };
 331
 332 /*
 333  * Pipeline.
 334  */
 335 struct thread {
 336         /* Packet. */
 337         struct rte_swx_pkt pkt;
 338         uint8_t *ptr;
 339
 340         /* Structures. */
 341         uint8_t **structs;
 342
 343         /* Packet headers. */
 344         struct header_runtime *headers; /* Extracted or generated headers. */
 345         struct header_out_runtime *headers_out; /* Emitted headers. */
 346         uint8_t *header_storage;
 347         uint8_t *header_out_storage;
 348         uint64_t valid_headers;
 349         uint32_t n_headers_out;
 350
 351         /* Packet meta-data. */
 352         uint8_t *metadata;
 353
 354         /* Tables. */
 355         struct table_runtime *tables;
 356         struct rte_swx_table_state *table_state;
 357         uint64_t action_id;
 358         int hit; /* 0 = Miss, 1 = Hit. */
 359
 360         /* Extern objects and functions. */
 361         struct extern_obj_runtime *extern_objs;
 362         struct extern_func_runtime *extern_funcs;
 363
 364         /* Instructions. */
 365         struct instruction *ip;
 366         struct instruction *ret;
 367 };
 368
 369 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 370 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 371 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 372
 373 #define METADATA_READ(thread, offset, n_bits)                                  \
 374 ({                                                                             \
 375         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 376         uint64_t m64 = *m64_ptr;                                               \
 377         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 378         (m64 & m64_mask);                                                      \
 379 })
 380
 381 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 382 {                                                                              \
 383         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 384         uint64_t m64 = *m64_ptr;                                               \
 385         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 386                                                                                \
 387         uint64_t m_new = value;                                                \
 388                                                                                \
 389         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 390 }
 391
 392 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 393 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 394 #endif
 395
 396 struct rte_swx_pipeline {
 397         struct struct_type_tailq struct_types;
 398         struct port_in_type_tailq port_in_types;
 399         struct port_in_tailq ports_in;
 400         struct port_out_type_tailq port_out_types;
 401         struct port_out_tailq ports_out;
 402         struct extern_type_tailq extern_types;
 403         struct extern_obj_tailq extern_objs;
 404         struct extern_func_tailq extern_funcs;
 405         struct header_tailq headers;
 406         struct struct_type *metadata_st;
 407         uint32_t metadata_struct_id;
 408         struct action_tailq actions;
 409         struct table_type_tailq table_types;
 410         struct table_tailq tables;
 411
 412         struct port_in_runtime *in;
 413         struct port_out_runtime *out;
 414         struct instruction **action_instructions;
 415         struct rte_swx_table_state *table_state;
 416         struct instruction *instructions;
 417         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 418
 419         uint32_t n_structs;
 420         uint32_t n_ports_in;
 421         uint32_t n_ports_out;
 422         uint32_t n_extern_objs;
 423         uint32_t n_extern_funcs;
 424         uint32_t n_actions;
 425         uint32_t n_tables;
 426         uint32_t n_headers;
 427         uint32_t thread_id;
 428         uint32_t port_id;
 429         uint32_t n_instructions;
 430         int build_done;
 431         int numa_node;
 432 };
 433
 434 /*
 435  * Struct.
 436  */
 437 static struct struct_type *
 438 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 439 {
 440         struct struct_type *elem;
 441
 442         TAILQ_FOREACH(elem, &p->struct_types, node)
 443                 if (strcmp(elem->name, name) == 0)
 444                         return elem;
 445
 446         return NULL;
 447 }
 448
 449 static struct field *
 450 struct_type_field_find(struct struct_type *st, const char *name)
 451 {
 452         uint32_t i;
 453
 454         for (i = 0; i < st->n_fields; i++) {
 455                 struct field *f = &st->fields[i];
 456
 457                 if (strcmp(f->name, name) == 0)
 458                         return f;
 459         }
 460
 461         return NULL;
 462 }
 463
 464 int
 465 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 466                                       const char *name,
 467                                       struct rte_swx_field_params *fields,
 468                                       uint32_t n_fields)
 469 {
 470         struct struct_type *st;
 471         uint32_t i;
 472
 473         CHECK(p, EINVAL);
 474         CHECK_NAME(name, EINVAL);
 475         CHECK(fields, EINVAL);
 476         CHECK(n_fields, EINVAL);
 477
 478         for (i = 0; i < n_fields; i++) {
 479                 struct rte_swx_field_params *f = &fields[i];
 480                 uint32_t j;
 481
 482                 CHECK_NAME(f->name, EINVAL);
 483                 CHECK(f->n_bits, EINVAL);
 484                 CHECK(f->n_bits <= 64, EINVAL);
 485                 CHECK((f->n_bits & 7) == 0, EINVAL);
 486
 487                 for (j = 0; j < i; j++) {
 488                         struct rte_swx_field_params *f_prev = &fields[j];
 489
 490                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 491                 }
 492         }
 493
 494         CHECK(!struct_type_find(p, name), EEXIST);
 495
 496         /* Node allocation. */
 497         st = calloc(1, sizeof(struct struct_type));
 498         CHECK(st, ENOMEM);
 499
 500         st->fields = calloc(n_fields, sizeof(struct field));
 501         if (!st->fields) {
 502                 free(st);
 503                 CHECK(0, ENOMEM);
 504         }
 505
 506         /* Node initialization. */
 507         strcpy(st->name, name);
 508         for (i = 0; i < n_fields; i++) {
 509                 struct field *dst = &st->fields[i];
 510                 struct rte_swx_field_params *src = &fields[i];
 511
 512                 strcpy(dst->name, src->name);
 513                 dst->n_bits = src->n_bits;
 514                 dst->offset = st->n_bits;
 515
 516                 st->n_bits += src->n_bits;
 517         }
 518         st->n_fields = n_fields;
 519
 520         /* Node add to tailq. */
 521         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 522
 523         return 0;
 524 }
 525
 526 static int
 527 struct_build(struct rte_swx_pipeline *p)
 528 {
 529         uint32_t i;
 530
 531         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 532                 struct thread *t = &p->threads[i];
 533
 534                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 535                 CHECK(t->structs, ENOMEM);
 536         }
 537
 538         return 0;
 539 }
 540
 541 static void
 542 struct_build_free(struct rte_swx_pipeline *p)
 543 {
 544         uint32_t i;
 545
 546         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 547                 struct thread *t = &p->threads[i];
 548
 549                 free(t->structs);
 550                 t->structs = NULL;
 551         }
 552 }
 553
 554 static void
 555 struct_free(struct rte_swx_pipeline *p)
 556 {
 557         struct_build_free(p);
 558
 559         /* Struct types. */
 560         for ( ; ; ) {
 561                 struct struct_type *elem;
 562
 563                 elem = TAILQ_FIRST(&p->struct_types);
 564                 if (!elem)
 565                         break;
 566
 567                 TAILQ_REMOVE(&p->struct_types, elem, node);
 568                 free(elem->fields);
 569                 free(elem);
 570         }
 571 }
 572
 573 /*
 574  * Input port.
 575  */
 576 static struct port_in_type *
 577 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 578 {
 579         struct port_in_type *elem;
 580
 581         if (!name)
 582                 return NULL;
 583
 584         TAILQ_FOREACH(elem, &p->port_in_types, node)
 585                 if (strcmp(elem->name, name) == 0)
 586                         return elem;
 587
 588         return NULL;
 589 }
 590
 591 int
 592 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 593                                        const char *name,
 594                                        struct rte_swx_port_in_ops *ops)
 595 {
 596         struct port_in_type *elem;
 597
 598         CHECK(p, EINVAL);
 599         CHECK_NAME(name, EINVAL);
 600         CHECK(ops, EINVAL);
 601         CHECK(ops->create, EINVAL);
 602         CHECK(ops->free, EINVAL);
 603         CHECK(ops->pkt_rx, EINVAL);
 604         CHECK(ops->stats_read, EINVAL);
 605
 606         CHECK(!port_in_type_find(p, name), EEXIST);
 607
 608         /* Node allocation. */
 609         elem = calloc(1, sizeof(struct port_in_type));
 610         CHECK(elem, ENOMEM);
 611
 612         /* Node initialization. */
 613         strcpy(elem->name, name);
 614         memcpy(&elem->ops, ops, sizeof(*ops));
 615
 616         /* Node add to tailq. */
 617         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 618
 619         return 0;
 620 }
 621
 622 static struct port_in *
 623 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 624 {
 625         struct port_in *port;
 626
 627         TAILQ_FOREACH(port, &p->ports_in, node)
 628                 if (port->id == port_id)
 629                         return port;
 630
 631         return NULL;
 632 }
 633
 634 int
 635 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 636                                 uint32_t port_id,
 637                                 const char *port_type_name,
 638                                 void *args)
 639 {
 640         struct port_in_type *type = NULL;
 641         struct port_in *port = NULL;
 642         void *obj = NULL;
 643
 644         CHECK(p, EINVAL);
 645
 646         CHECK(!port_in_find(p, port_id), EINVAL);
 647
 648         CHECK_NAME(port_type_name, EINVAL);
 649         type = port_in_type_find(p, port_type_name);
 650         CHECK(type, EINVAL);
 651
 652         obj = type->ops.create(args);
 653         CHECK(obj, ENODEV);
 654
 655         /* Node allocation. */
 656         port = calloc(1, sizeof(struct port_in));
 657         CHECK(port, ENOMEM);
 658
 659         /* Node initialization. */
 660         port->type = type;
 661         port->obj = obj;
 662         port->id = port_id;
 663
 664         /* Node add to tailq. */
 665         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 666         if (p->n_ports_in < port_id + 1)
 667                 p->n_ports_in = port_id + 1;
 668
 669         return 0;
 670 }
 671
 672 static int
 673 port_in_build(struct rte_swx_pipeline *p)
 674 {
 675         struct port_in *port;
 676         uint32_t i;
 677
 678         CHECK(p->n_ports_in, EINVAL);
 679         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 680
 681         for (i = 0; i < p->n_ports_in; i++)
 682                 CHECK(port_in_find(p, i), EINVAL);
 683
 684         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 685         CHECK(p->in, ENOMEM);
 686
 687         TAILQ_FOREACH(port, &p->ports_in, node) {
 688                 struct port_in_runtime *in = &p->in[port->id];
 689
 690                 in->pkt_rx = port->type->ops.pkt_rx;
 691                 in->obj = port->obj;
 692         }
 693
 694         return 0;
 695 }
 696
 697 static void
 698 port_in_build_free(struct rte_swx_pipeline *p)
 699 {
 700         free(p->in);
 701         p->in = NULL;
 702 }
 703
 704 static void
 705 port_in_free(struct rte_swx_pipeline *p)
 706 {
 707         port_in_build_free(p);
 708
 709         /* Input ports. */
 710         for ( ; ; ) {
 711                 struct port_in *port;
 712
 713                 port = TAILQ_FIRST(&p->ports_in);
 714                 if (!port)
 715                         break;
 716
 717                 TAILQ_REMOVE(&p->ports_in, port, node);
 718                 port->type->ops.free(port->obj);
 719                 free(port);
 720         }
 721
 722         /* Input port types. */
 723         for ( ; ; ) {
 724                 struct port_in_type *elem;
 725
 726                 elem = TAILQ_FIRST(&p->port_in_types);
 727                 if (!elem)
 728                         break;
 729
 730                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 731                 free(elem);
 732         }
 733 }
 734
 735 /*
 736  * Output port.
 737  */
 738 static struct port_out_type *
 739 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 740 {
 741         struct port_out_type *elem;
 742
 743         if (!name)
 744                 return NULL;
 745
 746         TAILQ_FOREACH(elem, &p->port_out_types, node)
 747                 if (!strcmp(elem->name, name))
 748                         return elem;
 749
 750         return NULL;
 751 }
 752
 753 int
 754 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 755                                         const char *name,
 756                                         struct rte_swx_port_out_ops *ops)
 757 {
 758         struct port_out_type *elem;
 759
 760         CHECK(p, EINVAL);
 761         CHECK_NAME(name, EINVAL);
 762         CHECK(ops, EINVAL);
 763         CHECK(ops->create, EINVAL);
 764         CHECK(ops->free, EINVAL);
 765         CHECK(ops->pkt_tx, EINVAL);
 766         CHECK(ops->stats_read, EINVAL);
 767
 768         CHECK(!port_out_type_find(p, name), EEXIST);
 769
 770         /* Node allocation. */
 771         elem = calloc(1, sizeof(struct port_out_type));
 772         CHECK(elem, ENOMEM);
 773
 774         /* Node initialization. */
 775         strcpy(elem->name, name);
 776         memcpy(&elem->ops, ops, sizeof(*ops));
 777
 778         /* Node add to tailq. */
 779         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 780
 781         return 0;
 782 }
 783
 784 static struct port_out *
 785 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 786 {
 787         struct port_out *port;
 788
 789         TAILQ_FOREACH(port, &p->ports_out, node)
 790                 if (port->id == port_id)
 791                         return port;
 792
 793         return NULL;
 794 }
 795
 796 int
 797 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 798                                  uint32_t port_id,
 799                                  const char *port_type_name,
 800                                  void *args)
 801 {
 802         struct port_out_type *type = NULL;
 803         struct port_out *port = NULL;
 804         void *obj = NULL;
 805
 806         CHECK(p, EINVAL);
 807
 808         CHECK(!port_out_find(p, port_id), EINVAL);
 809
 810         CHECK_NAME(port_type_name, EINVAL);
 811         type = port_out_type_find(p, port_type_name);
 812         CHECK(type, EINVAL);
 813
 814         obj = type->ops.create(args);
 815         CHECK(obj, ENODEV);
 816
 817         /* Node allocation. */
 818         port = calloc(1, sizeof(struct port_out));
 819         CHECK(port, ENOMEM);
 820
 821         /* Node initialization. */
 822         port->type = type;
 823         port->obj = obj;
 824         port->id = port_id;
 825
 826         /* Node add to tailq. */
 827         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 828         if (p->n_ports_out < port_id + 1)
 829                 p->n_ports_out = port_id + 1;
 830
 831         return 0;
 832 }
 833
 834 static int
 835 port_out_build(struct rte_swx_pipeline *p)
 836 {
 837         struct port_out *port;
 838         uint32_t i;
 839
 840         CHECK(p->n_ports_out, EINVAL);
 841
 842         for (i = 0; i < p->n_ports_out; i++)
 843                 CHECK(port_out_find(p, i), EINVAL);
 844
 845         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 846         CHECK(p->out, ENOMEM);
 847
 848         TAILQ_FOREACH(port, &p->ports_out, node) {
 849                 struct port_out_runtime *out = &p->out[port->id];
 850
 851                 out->pkt_tx = port->type->ops.pkt_tx;
 852                 out->flush = port->type->ops.flush;
 853                 out->obj = port->obj;
 854         }
 855
 856         return 0;
 857 }
 858
 859 static void
 860 port_out_build_free(struct rte_swx_pipeline *p)
 861 {
 862         free(p->out);
 863         p->out = NULL;
 864 }
 865
 866 static void
 867 port_out_free(struct rte_swx_pipeline *p)
 868 {
 869         port_out_build_free(p);
 870
 871         /* Output ports. */
 872         for ( ; ; ) {
 873                 struct port_out *port;
 874
 875                 port = TAILQ_FIRST(&p->ports_out);
 876                 if (!port)
 877                         break;
 878
 879                 TAILQ_REMOVE(&p->ports_out, port, node);
 880                 port->type->ops.free(port->obj);
 881                 free(port);
 882         }
 883
 884         /* Output port types. */
 885         for ( ; ; ) {
 886                 struct port_out_type *elem;
 887
 888                 elem = TAILQ_FIRST(&p->port_out_types);
 889                 if (!elem)
 890                         break;
 891
 892                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 893                 free(elem);
 894         }
 895 }
 896
 897 /*
 898  * Extern object.
 899  */
 900 static struct extern_type *
 901 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 902 {
 903         struct extern_type *elem;
 904
 905         TAILQ_FOREACH(elem, &p->extern_types, node)
 906                 if (strcmp(elem->name, name) == 0)
 907                         return elem;
 908
 909         return NULL;
 910 }
 911
 912 static struct extern_type_member_func *
 913 extern_type_member_func_find(struct extern_type *type, const char *name)
 914 {
 915         struct extern_type_member_func *elem;
 916
 917         TAILQ_FOREACH(elem, &type->funcs, node)
 918                 if (strcmp(elem->name, name) == 0)
 919                         return elem;
 920
 921         return NULL;
 922 }
 923
 924 static struct extern_obj *
 925 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
 926 {
 927         struct extern_obj *elem;
 928
 929         TAILQ_FOREACH(elem, &p->extern_objs, node)
 930                 if (strcmp(elem->name, name) == 0)
 931                         return elem;
 932
 933         return NULL;
 934 }
 935
 936 int
 937 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
 938         const char *name,
 939         const char *mailbox_struct_type_name,
 940         rte_swx_extern_type_constructor_t constructor,
 941         rte_swx_extern_type_destructor_t destructor)
 942 {
 943         struct extern_type *elem;
 944         struct struct_type *mailbox_struct_type;
 945
 946         CHECK(p, EINVAL);
 947
 948         CHECK_NAME(name, EINVAL);
 949         CHECK(!extern_type_find(p, name), EEXIST);
 950
 951         CHECK_NAME(mailbox_struct_type_name, EINVAL);
 952         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
 953         CHECK(mailbox_struct_type, EINVAL);
 954
 955         CHECK(constructor, EINVAL);
 956         CHECK(destructor, EINVAL);
 957
 958         /* Node allocation. */
 959         elem = calloc(1, sizeof(struct extern_type));
 960         CHECK(elem, ENOMEM);
 961
 962         /* Node initialization. */
 963         strcpy(elem->name, name);
 964         elem->mailbox_struct_type = mailbox_struct_type;
 965         elem->constructor = constructor;
 966         elem->destructor = destructor;
 967         TAILQ_INIT(&elem->funcs);
 968
 969         /* Node add to tailq. */
 970         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
 971
 972         return 0;
 973 }
 974
 975 int
 976 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
 977         const char *extern_type_name,
 978         const char *name,
 979         rte_swx_extern_type_member_func_t member_func)
 980 {
 981         struct extern_type *type;
 982         struct extern_type_member_func *type_member;
 983
 984         CHECK(p, EINVAL);
 985
 986         CHECK(extern_type_name, EINVAL);
 987         type = extern_type_find(p, extern_type_name);
 988         CHECK(type, EINVAL);
 989         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
 990
 991         CHECK(name, EINVAL);
 992         CHECK(!extern_type_member_func_find(type, name), EEXIST);
 993
 994         CHECK(member_func, EINVAL);
 995
 996         /* Node allocation. */
 997         type_member = calloc(1, sizeof(struct extern_type_member_func));
 998         CHECK(type_member, ENOMEM);
 999
1000         /* Node initialization. */
1001         strcpy(type_member->name, name);
1002         type_member->func = member_func;
1003         type_member->id = type->n_funcs;
1004
1005         /* Node add to tailq. */
1006         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1007         type->n_funcs++;
1008
1009         return 0;
1010 }
1011
1012 int
1013 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1014                                       const char *extern_type_name,
1015                                       const char *name,
1016                                       const char *args)
1017 {
1018         struct extern_type *type;
1019         struct extern_obj *obj;
1020         void *obj_handle;
1021
1022         CHECK(p, EINVAL);
1023
1024         CHECK_NAME(extern_type_name, EINVAL);
1025         type = extern_type_find(p, extern_type_name);
1026         CHECK(type, EINVAL);
1027
1028         CHECK_NAME(name, EINVAL);
1029         CHECK(!extern_obj_find(p, name), EEXIST);
1030
1031         /* Node allocation. */
1032         obj = calloc(1, sizeof(struct extern_obj));
1033         CHECK(obj, ENOMEM);
1034
1035         /* Object construction. */
1036         obj_handle = type->constructor(args);
1037         if (!obj_handle) {
1038                 free(obj);
1039                 CHECK(0, ENODEV);
1040         }
1041
1042         /* Node initialization. */
1043         strcpy(obj->name, name);
1044         obj->type = type;
1045         obj->obj = obj_handle;
1046         obj->struct_id = p->n_structs;
1047         obj->id = p->n_extern_objs;
1048
1049         /* Node add to tailq. */
1050         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1051         p->n_extern_objs++;
1052         p->n_structs++;
1053
1054         return 0;
1055 }
1056
1057 static int
1058 extern_obj_build(struct rte_swx_pipeline *p)
1059 {
1060         uint32_t i;
1061
1062         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1063                 struct thread *t = &p->threads[i];
1064                 struct extern_obj *obj;
1065
1066                 t->extern_objs = calloc(p->n_extern_objs,
1067                                         sizeof(struct extern_obj_runtime));
1068                 CHECK(t->extern_objs, ENOMEM);
1069
1070                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1071                         struct extern_obj_runtime *r =
1072                                 &t->extern_objs[obj->id];
1073                         struct extern_type_member_func *func;
1074                         uint32_t mailbox_size =
1075                                 obj->type->mailbox_struct_type->n_bits / 8;
1076
1077                         r->obj = obj->obj;
1078
1079                         r->mailbox = calloc(1, mailbox_size);
1080                         CHECK(r->mailbox, ENOMEM);
1081
1082                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1083                                 r->funcs[func->id] = func->func;
1084
1085                         t->structs[obj->struct_id] = r->mailbox;
1086                 }
1087         }
1088
1089         return 0;
1090 }
1091
1092 static void
1093 extern_obj_build_free(struct rte_swx_pipeline *p)
1094 {
1095         uint32_t i;
1096
1097         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1098                 struct thread *t = &p->threads[i];
1099                 uint32_t j;
1100
1101                 if (!t->extern_objs)
1102                         continue;
1103
1104                 for (j = 0; j < p->n_extern_objs; j++) {
1105                         struct extern_obj_runtime *r = &t->extern_objs[j];
1106
1107                         free(r->mailbox);
1108                 }
1109
1110                 free(t->extern_objs);
1111                 t->extern_objs = NULL;
1112         }
1113 }
1114
1115 static void
1116 extern_obj_free(struct rte_swx_pipeline *p)
1117 {
1118         extern_obj_build_free(p);
1119
1120         /* Extern objects. */
1121         for ( ; ; ) {
1122                 struct extern_obj *elem;
1123
1124                 elem = TAILQ_FIRST(&p->extern_objs);
1125                 if (!elem)
1126                         break;
1127
1128                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1129                 if (elem->obj)
1130                         elem->type->destructor(elem->obj);
1131                 free(elem);
1132         }
1133
1134         /* Extern types. */
1135         for ( ; ; ) {
1136                 struct extern_type *elem;
1137
1138                 elem = TAILQ_FIRST(&p->extern_types);
1139                 if (!elem)
1140                         break;
1141
1142                 TAILQ_REMOVE(&p->extern_types, elem, node);
1143
1144                 for ( ; ; ) {
1145                         struct extern_type_member_func *func;
1146
1147                         func = TAILQ_FIRST(&elem->funcs);
1148                         if (!func)
1149                                 break;
1150
1151                         TAILQ_REMOVE(&elem->funcs, func, node);
1152                         free(func);
1153                 }
1154
1155                 free(elem);
1156         }
1157 }
1158
1159 /*
1160  * Extern function.
1161  */
1162 static struct extern_func *
1163 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1164 {
1165         struct extern_func *elem;
1166
1167         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1168                 if (strcmp(elem->name, name) == 0)
1169                         return elem;
1170
1171         return NULL;
1172 }
1173
1174 int
1175 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1176                                       const char *name,
1177                                       const char *mailbox_struct_type_name,
1178                                       rte_swx_extern_func_t func)
1179 {
1180         struct extern_func *f;
1181         struct struct_type *mailbox_struct_type;
1182
1183         CHECK(p, EINVAL);
1184
1185         CHECK_NAME(name, EINVAL);
1186         CHECK(!extern_func_find(p, name), EEXIST);
1187
1188         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1189         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1190         CHECK(mailbox_struct_type, EINVAL);
1191
1192         CHECK(func, EINVAL);
1193
1194         /* Node allocation. */
1195         f = calloc(1, sizeof(struct extern_func));
1196         CHECK(func, ENOMEM);
1197
1198         /* Node initialization. */
1199         strcpy(f->name, name);
1200         f->mailbox_struct_type = mailbox_struct_type;
1201         f->func = func;
1202         f->struct_id = p->n_structs;
1203         f->id = p->n_extern_funcs;
1204
1205         /* Node add to tailq. */
1206         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1207         p->n_extern_funcs++;
1208         p->n_structs++;
1209
1210         return 0;
1211 }
1212
1213 static int
1214 extern_func_build(struct rte_swx_pipeline *p)
1215 {
1216         uint32_t i;
1217
1218         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1219                 struct thread *t = &p->threads[i];
1220                 struct extern_func *func;
1221
1222                 /* Memory allocation. */
1223                 t->extern_funcs = calloc(p->n_extern_funcs,
1224                                          sizeof(struct extern_func_runtime));
1225                 CHECK(t->extern_funcs, ENOMEM);
1226
1227                 /* Extern function. */
1228                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1229                         struct extern_func_runtime *r =
1230                                 &t->extern_funcs[func->id];
1231                         uint32_t mailbox_size =
1232                                 func->mailbox_struct_type->n_bits / 8;
1233
1234                         r->func = func->func;
1235
1236                         r->mailbox = calloc(1, mailbox_size);
1237                         CHECK(r->mailbox, ENOMEM);
1238
1239                         t->structs[func->struct_id] = r->mailbox;
1240                 }
1241         }
1242
1243         return 0;
1244 }
1245
1246 static void
1247 extern_func_build_free(struct rte_swx_pipeline *p)
1248 {
1249         uint32_t i;
1250
1251         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1252                 struct thread *t = &p->threads[i];
1253                 uint32_t j;
1254
1255                 if (!t->extern_funcs)
1256                         continue;
1257
1258                 for (j = 0; j < p->n_extern_funcs; j++) {
1259                         struct extern_func_runtime *r = &t->extern_funcs[j];
1260
1261                         free(r->mailbox);
1262                 }
1263
1264                 free(t->extern_funcs);
1265                 t->extern_funcs = NULL;
1266         }
1267 }
1268
1269 static void
1270 extern_func_free(struct rte_swx_pipeline *p)
1271 {
1272         extern_func_build_free(p);
1273
1274         for ( ; ; ) {
1275                 struct extern_func *elem;
1276
1277                 elem = TAILQ_FIRST(&p->extern_funcs);
1278                 if (!elem)
1279                         break;
1280
1281                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1282                 free(elem);
1283         }
1284 }
1285
1286 /*
1287  * Header.
1288  */
1289 static struct header *
1290 header_find(struct rte_swx_pipeline *p, const char *name)
1291 {
1292         struct header *elem;
1293
1294         TAILQ_FOREACH(elem, &p->headers, node)
1295                 if (strcmp(elem->name, name) == 0)
1296                         return elem;
1297
1298         return NULL;
1299 }
1300
1301 static struct header *
1302 header_parse(struct rte_swx_pipeline *p,
1303              const char *name)
1304 {
1305         if (name[0] != 'h' || name[1] != '.')
1306                 return NULL;
1307
1308         return header_find(p, &name[2]);
1309 }
1310
1311 static struct field *
1312 header_field_parse(struct rte_swx_pipeline *p,
1313                    const char *name,
1314                    struct header **header)
1315 {
1316         struct header *h;
1317         struct field *f;
1318         char *header_name, *field_name;
1319
1320         if ((name[0] != 'h') || (name[1] != '.'))
1321                 return NULL;
1322
1323         header_name = strdup(&name[2]);
1324         if (!header_name)
1325                 return NULL;
1326
1327         field_name = strchr(header_name, '.');
1328         if (!field_name) {
1329                 free(header_name);
1330                 return NULL;
1331         }
1332
1333         *field_name = 0;
1334         field_name++;
1335
1336         h = header_find(p, header_name);
1337         if (!h) {
1338                 free(header_name);
1339                 return NULL;
1340         }
1341
1342         f = struct_type_field_find(h->st, field_name);
1343         if (!f) {
1344                 free(header_name);
1345                 return NULL;
1346         }
1347
1348         if (header)
1349                 *header = h;
1350
1351         free(header_name);
1352         return f;
1353 }
1354
1355 int
1356 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1357                                         const char *name,
1358                                         const char *struct_type_name)
1359 {
1360         struct struct_type *st;
1361         struct header *h;
1362         size_t n_headers_max;
1363
1364         CHECK(p, EINVAL);
1365         CHECK_NAME(name, EINVAL);
1366         CHECK_NAME(struct_type_name, EINVAL);
1367
1368         CHECK(!header_find(p, name), EEXIST);
1369
1370         st = struct_type_find(p, struct_type_name);
1371         CHECK(st, EINVAL);
1372
1373         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1374         CHECK(p->n_headers < n_headers_max, ENOSPC);
1375
1376         /* Node allocation. */
1377         h = calloc(1, sizeof(struct header));
1378         CHECK(h, ENOMEM);
1379
1380         /* Node initialization. */
1381         strcpy(h->name, name);
1382         h->st = st;
1383         h->struct_id = p->n_structs;
1384         h->id = p->n_headers;
1385
1386         /* Node add to tailq. */
1387         TAILQ_INSERT_TAIL(&p->headers, h, node);
1388         p->n_headers++;
1389         p->n_structs++;
1390
1391         return 0;
1392 }
1393
1394 static int
1395 header_build(struct rte_swx_pipeline *p)
1396 {
1397         struct header *h;
1398         uint32_t n_bytes = 0, i;
1399
1400         TAILQ_FOREACH(h, &p->headers, node) {
1401                 n_bytes += h->st->n_bits / 8;
1402         }
1403
1404         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1405                 struct thread *t = &p->threads[i];
1406                 uint32_t offset = 0;
1407
1408                 t->headers = calloc(p->n_headers,
1409                                     sizeof(struct header_runtime));
1410                 CHECK(t->headers, ENOMEM);
1411
1412                 t->headers_out = calloc(p->n_headers,
1413                                         sizeof(struct header_out_runtime));
1414                 CHECK(t->headers_out, ENOMEM);
1415
1416                 t->header_storage = calloc(1, n_bytes);
1417                 CHECK(t->header_storage, ENOMEM);
1418
1419                 t->header_out_storage = calloc(1, n_bytes);
1420                 CHECK(t->header_out_storage, ENOMEM);
1421
1422                 TAILQ_FOREACH(h, &p->headers, node) {
1423                         uint8_t *header_storage;
1424
1425                         header_storage = &t->header_storage[offset];
1426                         offset += h->st->n_bits / 8;
1427
1428                         t->headers[h->id].ptr0 = header_storage;
1429                         t->structs[h->struct_id] = header_storage;
1430                 }
1431         }
1432
1433         return 0;
1434 }
1435
1436 static void
1437 header_build_free(struct rte_swx_pipeline *p)
1438 {
1439         uint32_t i;
1440
1441         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1442                 struct thread *t = &p->threads[i];
1443
1444                 free(t->headers_out);
1445                 t->headers_out = NULL;
1446
1447                 free(t->headers);
1448                 t->headers = NULL;
1449
1450                 free(t->header_out_storage);
1451                 t->header_out_storage = NULL;
1452
1453                 free(t->header_storage);
1454                 t->header_storage = NULL;
1455         }
1456 }
1457
1458 static void
1459 header_free(struct rte_swx_pipeline *p)
1460 {
1461         header_build_free(p);
1462
1463         for ( ; ; ) {
1464                 struct header *elem;
1465
1466                 elem = TAILQ_FIRST(&p->headers);
1467                 if (!elem)
1468                         break;
1469
1470                 TAILQ_REMOVE(&p->headers, elem, node);
1471                 free(elem);
1472         }
1473 }
1474
1475 /*
1476  * Meta-data.
1477  */
1478 static struct field *
1479 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1480 {
1481         if (!p->metadata_st)
1482                 return NULL;
1483
1484         if (name[0] != 'm' || name[1] != '.')
1485                 return NULL;
1486
1487         return struct_type_field_find(p->metadata_st, &name[2]);
1488 }
1489
1490 int
1491 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1492                                           const char *struct_type_name)
1493 {
1494         struct struct_type *st = NULL;
1495
1496         CHECK(p, EINVAL);
1497
1498         CHECK_NAME(struct_type_name, EINVAL);
1499         st  = struct_type_find(p, struct_type_name);
1500         CHECK(st, EINVAL);
1501         CHECK(!p->metadata_st, EINVAL);
1502
1503         p->metadata_st = st;
1504         p->metadata_struct_id = p->n_structs;
1505
1506         p->n_structs++;
1507
1508         return 0;
1509 }
1510
1511 static int
1512 metadata_build(struct rte_swx_pipeline *p)
1513 {
1514         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1515         uint32_t i;
1516
1517         /* Thread-level initialization. */
1518         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1519                 struct thread *t = &p->threads[i];
1520                 uint8_t *metadata;
1521
1522                 metadata = calloc(1, n_bytes);
1523                 CHECK(metadata, ENOMEM);
1524
1525                 t->metadata = metadata;
1526                 t->structs[p->metadata_struct_id] = metadata;
1527         }
1528
1529         return 0;
1530 }
1531
1532 static void
1533 metadata_build_free(struct rte_swx_pipeline *p)
1534 {
1535         uint32_t i;
1536
1537         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1538                 struct thread *t = &p->threads[i];
1539
1540                 free(t->metadata);
1541                 t->metadata = NULL;
1542         }
1543 }
1544
1545 static void
1546 metadata_free(struct rte_swx_pipeline *p)
1547 {
1548         metadata_build_free(p);
1549 }
1550
1551 /*
1552  * Instruction.
1553  */
1554 static inline void
1555 pipeline_port_inc(struct rte_swx_pipeline *p)
1556 {
1557         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1558 }
1559
1560 static inline void
1561 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1562 {
1563         t->ip = p->instructions;
1564 }
1565
1566 static inline void
1567 thread_ip_inc(struct rte_swx_pipeline *p);
1568
1569 static inline void
1570 thread_ip_inc(struct rte_swx_pipeline *p)
1571 {
1572         struct thread *t = &p->threads[p->thread_id];
1573
1574         t->ip++;
1575 }
1576
1577 static inline void
1578 thread_ip_inc_cond(struct thread *t, int cond)
1579 {
1580         t->ip += cond;
1581 }
1582
1583 static inline void
1584 thread_yield(struct rte_swx_pipeline *p)
1585 {
1586         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1587 }
1588
1589 /*
1590  * rx.
1591  */
1592 static int
1593 instr_rx_translate(struct rte_swx_pipeline *p,
1594                    struct action *action,
1595                    char **tokens,
1596                    int n_tokens,
1597                    struct instruction *instr,
1598                    struct instruction_data *data __rte_unused)
1599 {
1600         struct field *f;
1601
1602         CHECK(!action, EINVAL);
1603         CHECK(n_tokens == 2, EINVAL);
1604
1605         f = metadata_field_parse(p, tokens[1]);
1606         CHECK(f, EINVAL);
1607
1608         instr->type = INSTR_RX;
1609         instr->io.io.offset = f->offset / 8;
1610         instr->io.io.n_bits = f->n_bits;
1611         return 0;
1612 }
1613
1614 static inline void
1615 instr_rx_exec(struct rte_swx_pipeline *p);
1616
1617 static inline void
1618 instr_rx_exec(struct rte_swx_pipeline *p)
1619 {
1620         struct thread *t = &p->threads[p->thread_id];
1621         struct instruction *ip = t->ip;
1622         struct port_in_runtime *port = &p->in[p->port_id];
1623         struct rte_swx_pkt *pkt = &t->pkt;
1624         int pkt_received;
1625
1626         /* Packet. */
1627         pkt_received = port->pkt_rx(port->obj, pkt);
1628         t->ptr = &pkt->pkt[pkt->offset];
1629         rte_prefetch0(t->ptr);
1630
1631         TRACE("[Thread %2u] rx %s from port %u\n",
1632               p->thread_id,
1633               pkt_received ? "1 pkt" : "0 pkts",
1634               p->port_id);
1635
1636         /* Headers. */
1637         t->valid_headers = 0;
1638         t->n_headers_out = 0;
1639
1640         /* Meta-data. */
1641         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1642
1643         /* Tables. */
1644         t->table_state = p->table_state;
1645
1646         /* Thread. */
1647         pipeline_port_inc(p);
1648         thread_ip_inc_cond(t, pkt_received);
1649         thread_yield(p);
1650 }
1651
1652 /*
1653  * tx.
1654  */
1655 static int
1656 instr_tx_translate(struct rte_swx_pipeline *p,
1657                    struct action *action __rte_unused,
1658                    char **tokens,
1659                    int n_tokens,
1660                    struct instruction *instr,
1661                    struct instruction_data *data __rte_unused)
1662 {
1663         struct field *f;
1664
1665         CHECK(n_tokens == 2, EINVAL);
1666
1667         f = metadata_field_parse(p, tokens[1]);
1668         CHECK(f, EINVAL);
1669
1670         instr->type = INSTR_TX;
1671         instr->io.io.offset = f->offset / 8;
1672         instr->io.io.n_bits = f->n_bits;
1673         return 0;
1674 }
1675
1676 static inline void
1677 emit_handler(struct thread *t)
1678 {
1679         struct header_out_runtime *h0 = &t->headers_out[0];
1680         struct header_out_runtime *h1 = &t->headers_out[1];
1681         uint32_t offset = 0, i;
1682
1683         /* No header change or header decapsulation. */
1684         if ((t->n_headers_out == 1) &&
1685             (h0->ptr + h0->n_bytes == t->ptr)) {
1686                 TRACE("Emit handler: no header change or header decap.\n");
1687
1688                 t->pkt.offset -= h0->n_bytes;
1689                 t->pkt.length += h0->n_bytes;
1690
1691                 return;
1692         }
1693
1694         /* Header encapsulation (optionally, with prior header decasulation). */
1695         if ((t->n_headers_out == 2) &&
1696             (h1->ptr + h1->n_bytes == t->ptr) &&
1697             (h0->ptr == h0->ptr0)) {
1698                 uint32_t offset;
1699
1700                 TRACE("Emit handler: header encapsulation.\n");
1701
1702                 offset = h0->n_bytes + h1->n_bytes;
1703                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1704                 t->pkt.offset -= offset;
1705                 t->pkt.length += offset;
1706
1707                 return;
1708         }
1709
1710         /* Header insertion. */
1711         /* TBD */
1712
1713         /* Header extraction. */
1714         /* TBD */
1715
1716         /* For any other case. */
1717         TRACE("Emit handler: complex case.\n");
1718
1719         for (i = 0; i < t->n_headers_out; i++) {
1720                 struct header_out_runtime *h = &t->headers_out[i];
1721
1722                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1723                 offset += h->n_bytes;
1724         }
1725
1726         if (offset) {
1727                 memcpy(t->ptr - offset, t->header_out_storage, offset);
1728                 t->pkt.offset -= offset;
1729                 t->pkt.length += offset;
1730         }
1731 }
1732
1733 static inline void
1734 instr_tx_exec(struct rte_swx_pipeline *p);
1735
1736 static inline void
1737 instr_tx_exec(struct rte_swx_pipeline *p)
1738 {
1739         struct thread *t = &p->threads[p->thread_id];
1740         struct instruction *ip = t->ip;
1741         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1742         struct port_out_runtime *port = &p->out[port_id];
1743         struct rte_swx_pkt *pkt = &t->pkt;
1744
1745         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1746               p->thread_id,
1747               (uint32_t)port_id);
1748
1749         /* Headers. */
1750         emit_handler(t);
1751
1752         /* Packet. */
1753         port->pkt_tx(port->obj, pkt);
1754
1755         /* Thread. */
1756         thread_ip_reset(p, t);
1757         instr_rx_exec(p);
1758 }
1759
1760 /*
1761  * extract.
1762  */
1763 static int
1764 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1765                             struct action *action,
1766                             char **tokens,
1767                             int n_tokens,
1768                             struct instruction *instr,
1769                             struct instruction_data *data __rte_unused)
1770 {
1771         struct header *h;
1772
1773         CHECK(!action, EINVAL);
1774         CHECK(n_tokens == 2, EINVAL);
1775
1776         h = header_parse(p, tokens[1]);
1777         CHECK(h, EINVAL);
1778
1779         instr->type = INSTR_HDR_EXTRACT;
1780         instr->io.hdr.header_id[0] = h->id;
1781         instr->io.hdr.struct_id[0] = h->struct_id;
1782         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1783         return 0;
1784 }
1785
1786 static inline void
1787 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
1788
1789 static inline void
1790 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
1791 {
1792         struct thread *t = &p->threads[p->thread_id];
1793         struct instruction *ip = t->ip;
1794         uint64_t valid_headers = t->valid_headers;
1795         uint8_t *ptr = t->ptr;
1796         uint32_t offset = t->pkt.offset;
1797         uint32_t length = t->pkt.length;
1798         uint32_t i;
1799
1800         for (i = 0; i < n_extract; i++) {
1801                 uint32_t header_id = ip->io.hdr.header_id[i];
1802                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1803                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1804
1805                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1806                       p->thread_id,
1807                       header_id,
1808                       n_bytes);
1809
1810                 /* Headers. */
1811                 t->structs[struct_id] = ptr;
1812                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1813
1814                 /* Packet. */
1815                 offset += n_bytes;
1816                 length -= n_bytes;
1817                 ptr += n_bytes;
1818         }
1819
1820         /* Headers. */
1821         t->valid_headers = valid_headers;
1822
1823         /* Packet. */
1824         t->pkt.offset = offset;
1825         t->pkt.length = length;
1826         t->ptr = ptr;
1827 }
1828
1829 static inline void
1830 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1831 {
1832         __instr_hdr_extract_exec(p, 1);
1833
1834         /* Thread. */
1835         thread_ip_inc(p);
1836 }
1837
1838 static inline void
1839 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1840 {
1841         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
1842               p->thread_id);
1843
1844         __instr_hdr_extract_exec(p, 2);
1845
1846         /* Thread. */
1847         thread_ip_inc(p);
1848 }
1849
1850 static inline void
1851 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1852 {
1853         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
1854               p->thread_id);
1855
1856         __instr_hdr_extract_exec(p, 3);
1857
1858         /* Thread. */
1859         thread_ip_inc(p);
1860 }
1861
1862 static inline void
1863 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1864 {
1865         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
1866               p->thread_id);
1867
1868         __instr_hdr_extract_exec(p, 4);
1869
1870         /* Thread. */
1871         thread_ip_inc(p);
1872 }
1873
1874 static inline void
1875 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1876 {
1877         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
1878               p->thread_id);
1879
1880         __instr_hdr_extract_exec(p, 5);
1881
1882         /* Thread. */
1883         thread_ip_inc(p);
1884 }
1885
1886 static inline void
1887 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1888 {
1889         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
1890               p->thread_id);
1891
1892         __instr_hdr_extract_exec(p, 6);
1893
1894         /* Thread. */
1895         thread_ip_inc(p);
1896 }
1897
1898 static inline void
1899 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1900 {
1901         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
1902               p->thread_id);
1903
1904         __instr_hdr_extract_exec(p, 7);
1905
1906         /* Thread. */
1907         thread_ip_inc(p);
1908 }
1909
1910 static inline void
1911 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1912 {
1913         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
1914               p->thread_id);
1915
1916         __instr_hdr_extract_exec(p, 8);
1917
1918         /* Thread. */
1919         thread_ip_inc(p);
1920 }
1921
1922 /*
1923  * emit.
1924  */
1925 static int
1926 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1927                          struct action *action __rte_unused,
1928                          char **tokens,
1929                          int n_tokens,
1930                          struct instruction *instr,
1931                          struct instruction_data *data __rte_unused)
1932 {
1933         struct header *h;
1934
1935         CHECK(n_tokens == 2, EINVAL);
1936
1937         h = header_parse(p, tokens[1]);
1938         CHECK(h, EINVAL);
1939
1940         instr->type = INSTR_HDR_EMIT;
1941         instr->io.hdr.header_id[0] = h->id;
1942         instr->io.hdr.struct_id[0] = h->struct_id;
1943         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1944         return 0;
1945 }
1946
1947 static inline void
1948 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
1949
1950 static inline void
1951 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
1952 {
1953         struct thread *t = &p->threads[p->thread_id];
1954         struct instruction *ip = t->ip;
1955         uint32_t n_headers_out = t->n_headers_out;
1956         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
1957         uint8_t *ho_ptr = NULL;
1958         uint32_t ho_nbytes = 0, i;
1959
1960         for (i = 0; i < n_emit; i++) {
1961                 uint32_t header_id = ip->io.hdr.header_id[i];
1962                 uint32_t struct_id = ip->io.hdr.struct_id[i];
1963                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1964
1965                 struct header_runtime *hi = &t->headers[header_id];
1966                 uint8_t *hi_ptr = t->structs[struct_id];
1967
1968                 TRACE("[Thread %2u]: emit header %u\n",
1969                       p->thread_id,
1970                       header_id);
1971
1972                 /* Headers. */
1973                 if (!i) {
1974                         if (!t->n_headers_out) {
1975                                 ho = &t->headers_out[0];
1976
1977                                 ho->ptr0 = hi->ptr0;
1978                                 ho->ptr = hi_ptr;
1979
1980                                 ho_ptr = hi_ptr;
1981                                 ho_nbytes = n_bytes;
1982
1983                                 n_headers_out = 1;
1984
1985                                 continue;
1986                         } else {
1987                                 ho_ptr = ho->ptr;
1988                                 ho_nbytes = ho->n_bytes;
1989                         }
1990                 }
1991
1992                 if (ho_ptr + ho_nbytes == hi_ptr) {
1993                         ho_nbytes += n_bytes;
1994                 } else {
1995                         ho->n_bytes = ho_nbytes;
1996
1997                         ho++;
1998                         ho->ptr0 = hi->ptr0;
1999                         ho->ptr = hi_ptr;
2000
2001                         ho_ptr = hi_ptr;
2002                         ho_nbytes = n_bytes;
2003
2004                         n_headers_out++;
2005                 }
2006         }
2007
2008         ho->n_bytes = ho_nbytes;
2009         t->n_headers_out = n_headers_out;
2010 }
2011
2012 static inline void
2013 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2014 {
2015         __instr_hdr_emit_exec(p, 1);
2016
2017         /* Thread. */
2018         thread_ip_inc(p);
2019 }
2020
2021 static inline void
2022 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2023 {
2024         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2025               p->thread_id);
2026
2027         __instr_hdr_emit_exec(p, 1);
2028         instr_tx_exec(p);
2029 }
2030
2031 static inline void
2032 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2033 {
2034         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2035               p->thread_id);
2036
2037         __instr_hdr_emit_exec(p, 2);
2038         instr_tx_exec(p);
2039 }
2040
2041 static inline void
2042 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2043 {
2044         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2045               p->thread_id);
2046
2047         __instr_hdr_emit_exec(p, 3);
2048         instr_tx_exec(p);
2049 }
2050
2051 static inline void
2052 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2053 {
2054         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2055               p->thread_id);
2056
2057         __instr_hdr_emit_exec(p, 4);
2058         instr_tx_exec(p);
2059 }
2060
2061 static inline void
2062 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2063 {
2064         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2065               p->thread_id);
2066
2067         __instr_hdr_emit_exec(p, 5);
2068         instr_tx_exec(p);
2069 }
2070
2071 static inline void
2072 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2073 {
2074         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2075               p->thread_id);
2076
2077         __instr_hdr_emit_exec(p, 6);
2078         instr_tx_exec(p);
2079 }
2080
2081 static inline void
2082 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2083 {
2084         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2085               p->thread_id);
2086
2087         __instr_hdr_emit_exec(p, 7);
2088         instr_tx_exec(p);
2089 }
2090
2091 static inline void
2092 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2093 {
2094         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2095               p->thread_id);
2096
2097         __instr_hdr_emit_exec(p, 8);
2098         instr_tx_exec(p);
2099 }
2100
2101 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
2102
2103 static int
2104 instr_translate(struct rte_swx_pipeline *p,
2105                 struct action *action,
2106                 char *string,
2107                 struct instruction *instr,
2108                 struct instruction_data *data)
2109 {
2110         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
2111         int n_tokens = 0, tpos = 0;
2112
2113         /* Parse the instruction string into tokens. */
2114         for ( ; ; ) {
2115                 char *token;
2116
2117                 token = strtok_r(string, " \t\v", &string);
2118                 if (!token)
2119                         break;
2120
2121                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
2122
2123                 tokens[n_tokens] = token;
2124                 n_tokens++;
2125         }
2126
2127         CHECK(n_tokens, EINVAL);
2128
2129         /* Handle the optional instruction label. */
2130         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
2131                 strcpy(data->label, tokens[0]);
2132
2133                 tpos += 2;
2134                 CHECK(n_tokens - tpos, EINVAL);
2135         }
2136
2137         /* Identify the instruction type. */
2138         if (!strcmp(tokens[tpos], "rx"))
2139                 return instr_rx_translate(p,
2140                                           action,
2141                                           &tokens[tpos],
2142                                           n_tokens - tpos,
2143                                           instr,
2144                                           data);
2145
2146         if (!strcmp(tokens[tpos], "tx"))
2147                 return instr_tx_translate(p,
2148                                           action,
2149                                           &tokens[tpos],
2150                                           n_tokens - tpos,
2151                                           instr,
2152                                           data);
2153
2154         if (!strcmp(tokens[tpos], "extract"))
2155                 return instr_hdr_extract_translate(p,
2156                                                    action,
2157                                                    &tokens[tpos],
2158                                                    n_tokens - tpos,
2159                                                    instr,
2160                                                    data);
2161
2162         if (!strcmp(tokens[tpos], "emit"))
2163                 return instr_hdr_emit_translate(p,
2164                                                 action,
2165                                                 &tokens[tpos],
2166                                                 n_tokens - tpos,
2167                                                 instr,
2168                                                 data);
2169
2170         CHECK(0, EINVAL);
2171 }
2172
2173 static uint32_t
2174 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
2175 {
2176         uint32_t count = 0, i;
2177
2178         if (!label[0])
2179                 return 0;
2180
2181         for (i = 0; i < n; i++)
2182                 if (!strcmp(label, data[i].jmp_label))
2183                         count++;
2184
2185         return count;
2186 }
2187
2188 static int
2189 instr_label_check(struct instruction_data *instruction_data,
2190                   uint32_t n_instructions)
2191 {
2192         uint32_t i;
2193
2194         /* Check that all instruction labels are unique. */
2195         for (i = 0; i < n_instructions; i++) {
2196                 struct instruction_data *data = &instruction_data[i];
2197                 char *label = data->label;
2198                 uint32_t j;
2199
2200                 if (!label[0])
2201                         continue;
2202
2203                 for (j = i + 1; j < n_instructions; j++)
2204                         CHECK(strcmp(label, data[j].label), EINVAL);
2205         }
2206
2207         /* Get users for each instruction label. */
2208         for (i = 0; i < n_instructions; i++) {
2209                 struct instruction_data *data = &instruction_data[i];
2210                 char *label = data->label;
2211
2212                 data->n_users = label_is_used(instruction_data,
2213                                               n_instructions,
2214                                               label);
2215         }
2216
2217         return 0;
2218 }
2219
2220 static int
2221 instruction_config(struct rte_swx_pipeline *p,
2222                    struct action *a,
2223                    const char **instructions,
2224                    uint32_t n_instructions)
2225 {
2226         struct instruction *instr = NULL;
2227         struct instruction_data *data = NULL;
2228         char *string = NULL;
2229         int err = 0;
2230         uint32_t i;
2231
2232         CHECK(n_instructions, EINVAL);
2233         CHECK(instructions, EINVAL);
2234         for (i = 0; i < n_instructions; i++)
2235                 CHECK(instructions[i], EINVAL);
2236
2237         /* Memory allocation. */
2238         instr = calloc(n_instructions, sizeof(struct instruction));
2239         if (!instr) {
2240                 err = ENOMEM;
2241                 goto error;
2242         }
2243
2244         data = calloc(n_instructions, sizeof(struct instruction_data));
2245         if (!data) {
2246                 err = ENOMEM;
2247                 goto error;
2248         }
2249
2250         for (i = 0; i < n_instructions; i++) {
2251                 string = strdup(instructions[i]);
2252                 if (!string) {
2253                         err = ENOMEM;
2254                         goto error;
2255                 }
2256
2257                 err = instr_translate(p, a, string, &instr[i], &data[i]);
2258                 if (err)
2259                         goto error;
2260
2261                 free(string);
2262         }
2263
2264         err = instr_label_check(data, n_instructions);
2265         if (err)
2266                 goto error;
2267
2268         free(data);
2269
2270         if (a) {
2271                 a->instructions = instr;
2272                 a->n_instructions = n_instructions;
2273         } else {
2274                 p->instructions = instr;
2275                 p->n_instructions = n_instructions;
2276         }
2277
2278         return 0;
2279
2280 error:
2281         free(string);
2282         free(data);
2283         free(instr);
2284         return err;
2285 }
2286
2287 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
2288
2289 static instr_exec_t instruction_table[] = {
2290         [INSTR_RX] = instr_rx_exec,
2291         [INSTR_TX] = instr_tx_exec,
2292
2293         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
2294         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
2295         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
2296         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
2297         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
2298         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
2299         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
2300         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
2301
2302         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
2303         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
2304         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
2305         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
2306         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
2307         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
2308         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
2309         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
2310         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
2311 };
2312
2313 static inline void
2314 instr_exec(struct rte_swx_pipeline *p)
2315 {
2316         struct thread *t = &p->threads[p->thread_id];
2317         struct instruction *ip = t->ip;
2318         instr_exec_t instr = instruction_table[ip->type];
2319
2320         instr(p);
2321 }
2322
2323 /*
2324  * Action.
2325  */
2326 static struct action *
2327 action_find(struct rte_swx_pipeline *p, const char *name)
2328 {
2329         struct action *elem;
2330
2331         if (!name)
2332                 return NULL;
2333
2334         TAILQ_FOREACH(elem, &p->actions, node)
2335                 if (strcmp(elem->name, name) == 0)
2336                         return elem;
2337
2338         return NULL;
2339 }
2340
2341 int
2342 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
2343                                const char *name,
2344                                const char *args_struct_type_name,
2345                                const char **instructions,
2346                                uint32_t n_instructions)
2347 {
2348         struct struct_type *args_struct_type;
2349         struct action *a;
2350         int err;
2351
2352         CHECK(p, EINVAL);
2353
2354         CHECK_NAME(name, EINVAL);
2355         CHECK(!action_find(p, name), EEXIST);
2356
2357         if (args_struct_type_name) {
2358                 CHECK_NAME(args_struct_type_name, EINVAL);
2359                 args_struct_type = struct_type_find(p, args_struct_type_name);
2360                 CHECK(args_struct_type, EINVAL);
2361         } else {
2362                 args_struct_type = NULL;
2363         }
2364
2365         /* Node allocation. */
2366         a = calloc(1, sizeof(struct action));
2367         CHECK(a, ENOMEM);
2368
2369         /* Node initialization. */
2370         strcpy(a->name, name);
2371         a->st = args_struct_type;
2372         a->id = p->n_actions;
2373
2374         /* Instruction translation. */
2375         err = instruction_config(p, a, instructions, n_instructions);
2376         if (err) {
2377                 free(a);
2378                 return err;
2379         }
2380
2381         /* Node add to tailq. */
2382         TAILQ_INSERT_TAIL(&p->actions, a, node);
2383         p->n_actions++;
2384
2385         return 0;
2386 }
2387
2388 static int
2389 action_build(struct rte_swx_pipeline *p)
2390 {
2391         struct action *action;
2392
2393         p->action_instructions = calloc(p->n_actions,
2394                                         sizeof(struct instruction *));
2395         CHECK(p->action_instructions, ENOMEM);
2396
2397         TAILQ_FOREACH(action, &p->actions, node)
2398                 p->action_instructions[action->id] = action->instructions;
2399
2400         return 0;
2401 }
2402
2403 static void
2404 action_build_free(struct rte_swx_pipeline *p)
2405 {
2406         free(p->action_instructions);
2407         p->action_instructions = NULL;
2408 }
2409
2410 static void
2411 action_free(struct rte_swx_pipeline *p)
2412 {
2413         action_build_free(p);
2414
2415         for ( ; ; ) {
2416                 struct action *action;
2417
2418                 action = TAILQ_FIRST(&p->actions);
2419                 if (!action)
2420                         break;
2421
2422                 TAILQ_REMOVE(&p->actions, action, node);
2423                 free(action->instructions);
2424                 free(action);
2425         }
2426 }
2427
2428 /*
2429  * Table.
2430  */
2431 static struct table_type *
2432 table_type_find(struct rte_swx_pipeline *p, const char *name)
2433 {
2434         struct table_type *elem;
2435
2436         TAILQ_FOREACH(elem, &p->table_types, node)
2437                 if (strcmp(elem->name, name) == 0)
2438                         return elem;
2439
2440         return NULL;
2441 }
2442
2443 static struct table_type *
2444 table_type_resolve(struct rte_swx_pipeline *p,
2445                    const char *recommended_type_name,
2446                    enum rte_swx_table_match_type match_type)
2447 {
2448         struct table_type *elem;
2449
2450         /* Only consider the recommended type if the match type is correct. */
2451         if (recommended_type_name)
2452                 TAILQ_FOREACH(elem, &p->table_types, node)
2453                         if (!strcmp(elem->name, recommended_type_name) &&
2454                             (elem->match_type == match_type))
2455                                 return elem;
2456
2457         /* Ignore the recommended type and get the first element with this match
2458          * type.
2459          */
2460         TAILQ_FOREACH(elem, &p->table_types, node)
2461                 if (elem->match_type == match_type)
2462                         return elem;
2463
2464         return NULL;
2465 }
2466
2467 static struct table *
2468 table_find(struct rte_swx_pipeline *p, const char *name)
2469 {
2470         struct table *elem;
2471
2472         TAILQ_FOREACH(elem, &p->tables, node)
2473                 if (strcmp(elem->name, name) == 0)
2474                         return elem;
2475
2476         return NULL;
2477 }
2478
2479 static struct table *
2480 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
2481 {
2482         struct table *table = NULL;
2483
2484         TAILQ_FOREACH(table, &p->tables, node)
2485                 if (table->id == id)
2486                         return table;
2487
2488         return NULL;
2489 }
2490
2491 int
2492 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
2493                                      const char *name,
2494                                      enum rte_swx_table_match_type match_type,
2495                                      struct rte_swx_table_ops *ops)
2496 {
2497         struct table_type *elem;
2498
2499         CHECK(p, EINVAL);
2500
2501         CHECK_NAME(name, EINVAL);
2502         CHECK(!table_type_find(p, name), EEXIST);
2503
2504         CHECK(ops, EINVAL);
2505         CHECK(ops->create, EINVAL);
2506         CHECK(ops->lkp, EINVAL);
2507         CHECK(ops->free, EINVAL);
2508
2509         /* Node allocation. */
2510         elem = calloc(1, sizeof(struct table_type));
2511         CHECK(elem, ENOMEM);
2512
2513         /* Node initialization. */
2514         strcpy(elem->name, name);
2515         elem->match_type = match_type;
2516         memcpy(&elem->ops, ops, sizeof(*ops));
2517
2518         /* Node add to tailq. */
2519         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
2520
2521         return 0;
2522 }
2523
2524 static enum rte_swx_table_match_type
2525 table_match_type_resolve(struct rte_swx_match_field_params *fields,
2526                          uint32_t n_fields)
2527 {
2528         uint32_t i;
2529
2530         for (i = 0; i < n_fields; i++)
2531                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
2532                         break;
2533
2534         if (i == n_fields)
2535                 return RTE_SWX_TABLE_MATCH_EXACT;
2536
2537         if ((i == n_fields - 1) &&
2538             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
2539                 return RTE_SWX_TABLE_MATCH_LPM;
2540
2541         return RTE_SWX_TABLE_MATCH_WILDCARD;
2542 }
2543
2544 int
2545 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
2546                               const char *name,
2547                               struct rte_swx_pipeline_table_params *params,
2548                               const char *recommended_table_type_name,
2549                               const char *args,
2550                               uint32_t size)
2551 {
2552         struct table_type *type;
2553         struct table *t;
2554         struct action *default_action;
2555         struct header *header = NULL;
2556         int is_header = 0;
2557         uint32_t offset_prev = 0, action_data_size_max = 0, i;
2558
2559         CHECK(p, EINVAL);
2560
2561         CHECK_NAME(name, EINVAL);
2562         CHECK(!table_find(p, name), EEXIST);
2563
2564         CHECK(params, EINVAL);
2565
2566         /* Match checks. */
2567         CHECK(!params->n_fields || params->fields, EINVAL);
2568         for (i = 0; i < params->n_fields; i++) {
2569                 struct rte_swx_match_field_params *field = &params->fields[i];
2570                 struct header *h;
2571                 struct field *hf, *mf;
2572                 uint32_t offset;
2573
2574                 CHECK_NAME(field->name, EINVAL);
2575
2576                 hf = header_field_parse(p, field->name, &h);
2577                 mf = metadata_field_parse(p, field->name);
2578                 CHECK(hf || mf, EINVAL);
2579
2580                 offset = hf ? hf->offset : mf->offset;
2581
2582                 if (i == 0) {
2583                         is_header = hf ? 1 : 0;
2584                         header = hf ? h : NULL;
2585                         offset_prev = offset;
2586
2587                         continue;
2588                 }
2589
2590                 CHECK((is_header && hf && (h->id == header->id)) ||
2591                       (!is_header && mf), EINVAL);
2592
2593                 CHECK(offset > offset_prev, EINVAL);
2594                 offset_prev = offset;
2595         }
2596
2597         /* Action checks. */
2598         CHECK(params->n_actions, EINVAL);
2599         CHECK(params->action_names, EINVAL);
2600         for (i = 0; i < params->n_actions; i++) {
2601                 const char *action_name = params->action_names[i];
2602                 struct action *a;
2603                 uint32_t action_data_size;
2604
2605                 CHECK(action_name, EINVAL);
2606
2607                 a = action_find(p, action_name);
2608                 CHECK(a, EINVAL);
2609
2610                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
2611                 if (action_data_size > action_data_size_max)
2612                         action_data_size_max = action_data_size;
2613         }
2614
2615         CHECK(params->default_action_name, EINVAL);
2616         for (i = 0; i < p->n_actions; i++)
2617                 if (!strcmp(params->action_names[i],
2618                             params->default_action_name))
2619                         break;
2620         CHECK(i < params->n_actions, EINVAL);
2621         default_action = action_find(p, params->default_action_name);
2622         CHECK((default_action->st && params->default_action_data) ||
2623               !params->default_action_data, EINVAL);
2624
2625         /* Table type checks. */
2626         if (params->n_fields) {
2627                 enum rte_swx_table_match_type match_type;
2628
2629                 match_type = table_match_type_resolve(params->fields,
2630                                                       params->n_fields);
2631                 type = table_type_resolve(p,
2632                                           recommended_table_type_name,
2633                                           match_type);
2634                 CHECK(type, EINVAL);
2635         } else {
2636                 type = NULL;
2637         }
2638
2639         /* Memory allocation. */
2640         t = calloc(1, sizeof(struct table));
2641         CHECK(t, ENOMEM);
2642
2643         t->fields = calloc(params->n_fields, sizeof(struct match_field));
2644         if (!t->fields) {
2645                 free(t);
2646                 CHECK(0, ENOMEM);
2647         }
2648
2649         t->actions = calloc(params->n_actions, sizeof(struct action *));
2650         if (!t->actions) {
2651                 free(t->fields);
2652                 free(t);
2653                 CHECK(0, ENOMEM);
2654         }
2655
2656         if (action_data_size_max) {
2657                 t->default_action_data = calloc(1, action_data_size_max);
2658                 if (!t->default_action_data) {
2659                         free(t->actions);
2660                         free(t->fields);
2661                         free(t);
2662                         CHECK(0, ENOMEM);
2663                 }
2664         }
2665
2666         /* Node initialization. */
2667         strcpy(t->name, name);
2668         if (args && args[0])
2669                 strcpy(t->args, args);
2670         t->type = type;
2671
2672         for (i = 0; i < params->n_fields; i++) {
2673                 struct rte_swx_match_field_params *field = &params->fields[i];
2674                 struct match_field *f = &t->fields[i];
2675
2676                 f->match_type = field->match_type;
2677                 f->field = is_header ?
2678                         header_field_parse(p, field->name, NULL) :
2679                         metadata_field_parse(p, field->name);
2680         }
2681         t->n_fields = params->n_fields;
2682         t->is_header = is_header;
2683         t->header = header;
2684
2685         for (i = 0; i < params->n_actions; i++)
2686                 t->actions[i] = action_find(p, params->action_names[i]);
2687         t->default_action = default_action;
2688         if (default_action->st)
2689                 memcpy(t->default_action_data,
2690                        params->default_action_data,
2691                        default_action->st->n_bits / 8);
2692         t->n_actions = params->n_actions;
2693         t->default_action_is_const = params->default_action_is_const;
2694         t->action_data_size_max = action_data_size_max;
2695
2696         t->size = size;
2697         t->id = p->n_tables;
2698
2699         /* Node add to tailq. */
2700         TAILQ_INSERT_TAIL(&p->tables, t, node);
2701         p->n_tables++;
2702
2703         return 0;
2704 }
2705
2706 static struct rte_swx_table_params *
2707 table_params_get(struct table *table)
2708 {
2709         struct rte_swx_table_params *params;
2710         struct field *first, *last;
2711         uint8_t *key_mask;
2712         uint32_t key_size, key_offset, action_data_size, i;
2713
2714         /* Memory allocation. */
2715         params = calloc(1, sizeof(struct rte_swx_table_params));
2716         if (!params)
2717                 return NULL;
2718
2719         /* Key offset and size. */
2720         first = table->fields[0].field;
2721         last = table->fields[table->n_fields - 1].field;
2722         key_offset = first->offset / 8;
2723         key_size = (last->offset + last->n_bits - first->offset) / 8;
2724
2725         /* Memory allocation. */
2726         key_mask = calloc(1, key_size);
2727         if (!key_mask) {
2728                 free(params);
2729                 return NULL;
2730         }
2731
2732         /* Key mask. */
2733         for (i = 0; i < table->n_fields; i++) {
2734                 struct field *f = table->fields[i].field;
2735                 uint32_t start = (f->offset - first->offset) / 8;
2736                 size_t size = f->n_bits / 8;
2737
2738                 memset(&key_mask[start], 0xFF, size);
2739         }
2740
2741         /* Action data size. */
2742         action_data_size = 0;
2743         for (i = 0; i < table->n_actions; i++) {
2744                 struct action *action = table->actions[i];
2745                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
2746
2747                 if (ads > action_data_size)
2748                         action_data_size = ads;
2749         }
2750
2751         /* Fill in. */
2752         params->match_type = table->type->match_type;
2753         params->key_size = key_size;
2754         params->key_offset = key_offset;
2755         params->key_mask0 = key_mask;
2756         params->action_data_size = action_data_size;
2757         params->n_keys_max = table->size;
2758
2759         return params;
2760 }
2761
2762 static void
2763 table_params_free(struct rte_swx_table_params *params)
2764 {
2765         if (!params)
2766                 return;
2767
2768         free(params->key_mask0);
2769         free(params);
2770 }
2771
2772 static int
2773 table_state_build(struct rte_swx_pipeline *p)
2774 {
2775         struct table *table;
2776
2777         p->table_state = calloc(p->n_tables,
2778                                 sizeof(struct rte_swx_table_state));
2779         CHECK(p->table_state, ENOMEM);
2780
2781         TAILQ_FOREACH(table, &p->tables, node) {
2782                 struct rte_swx_table_state *ts = &p->table_state[table->id];
2783
2784                 if (table->type) {
2785                         struct rte_swx_table_params *params;
2786
2787                         /* ts->obj. */
2788                         params = table_params_get(table);
2789                         CHECK(params, ENOMEM);
2790
2791                         ts->obj = table->type->ops.create(params,
2792                                 NULL,
2793                                 table->args,
2794                                 p->numa_node);
2795
2796                         table_params_free(params);
2797                         CHECK(ts->obj, ENODEV);
2798                 }
2799
2800                 /* ts->default_action_data. */
2801                 if (table->action_data_size_max) {
2802                         ts->default_action_data =
2803                                 malloc(table->action_data_size_max);
2804                         CHECK(ts->default_action_data, ENOMEM);
2805
2806                         memcpy(ts->default_action_data,
2807                                table->default_action_data,
2808                                table->action_data_size_max);
2809                 }
2810
2811                 /* ts->default_action_id. */
2812                 ts->default_action_id = table->default_action->id;
2813         }
2814
2815         return 0;
2816 }
2817
2818 static void
2819 table_state_build_free(struct rte_swx_pipeline *p)
2820 {
2821         uint32_t i;
2822
2823         if (!p->table_state)
2824                 return;
2825
2826         for (i = 0; i < p->n_tables; i++) {
2827                 struct rte_swx_table_state *ts = &p->table_state[i];
2828                 struct table *table = table_find_by_id(p, i);
2829
2830                 /* ts->obj. */
2831                 if (table->type && ts->obj)
2832                         table->type->ops.free(ts->obj);
2833
2834                 /* ts->default_action_data. */
2835                 free(ts->default_action_data);
2836         }
2837
2838         free(p->table_state);
2839         p->table_state = NULL;
2840 }
2841
2842 static void
2843 table_state_free(struct rte_swx_pipeline *p)
2844 {
2845         table_state_build_free(p);
2846 }
2847
2848 static int
2849 table_stub_lkp(void *table __rte_unused,
2850                void *mailbox __rte_unused,
2851                uint8_t **key __rte_unused,
2852                uint64_t *action_id __rte_unused,
2853                uint8_t **action_data __rte_unused,
2854                int *hit)
2855 {
2856         *hit = 0;
2857         return 1; /* DONE. */
2858 }
2859
2860 static int
2861 table_build(struct rte_swx_pipeline *p)
2862 {
2863         uint32_t i;
2864
2865         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2866                 struct thread *t = &p->threads[i];
2867                 struct table *table;
2868
2869                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
2870                 CHECK(t->tables, ENOMEM);
2871
2872                 TAILQ_FOREACH(table, &p->tables, node) {
2873                         struct table_runtime *r = &t->tables[table->id];
2874
2875                         if (table->type) {
2876                                 uint64_t size;
2877
2878                                 size = table->type->ops.mailbox_size_get();
2879
2880                                 /* r->func. */
2881                                 r->func = table->type->ops.lkp;
2882
2883                                 /* r->mailbox. */
2884                                 if (size) {
2885                                         r->mailbox = calloc(1, size);
2886                                         CHECK(r->mailbox, ENOMEM);
2887                                 }
2888
2889                                 /* r->key. */
2890                                 r->key = table->is_header ?
2891                                         &t->structs[table->header->struct_id] :
2892                                         &t->structs[p->metadata_struct_id];
2893                         } else {
2894                                 r->func = table_stub_lkp;
2895                         }
2896                 }
2897         }
2898
2899         return 0;
2900 }
2901
2902 static void
2903 table_build_free(struct rte_swx_pipeline *p)
2904 {
2905         uint32_t i;
2906
2907         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2908                 struct thread *t = &p->threads[i];
2909                 uint32_t j;
2910
2911                 if (!t->tables)
2912                         continue;
2913
2914                 for (j = 0; j < p->n_tables; j++) {
2915                         struct table_runtime *r = &t->tables[j];
2916
2917                         free(r->mailbox);
2918                 }
2919
2920                 free(t->tables);
2921                 t->tables = NULL;
2922         }
2923 }
2924
2925 static void
2926 table_free(struct rte_swx_pipeline *p)
2927 {
2928         table_build_free(p);
2929
2930         /* Tables. */
2931         for ( ; ; ) {
2932                 struct table *elem;
2933
2934                 elem = TAILQ_FIRST(&p->tables);
2935                 if (!elem)
2936                         break;
2937
2938                 TAILQ_REMOVE(&p->tables, elem, node);
2939                 free(elem->fields);
2940                 free(elem->actions);
2941                 free(elem->default_action_data);
2942                 free(elem);
2943         }
2944
2945         /* Table types. */
2946         for ( ; ; ) {
2947                 struct table_type *elem;
2948
2949                 elem = TAILQ_FIRST(&p->table_types);
2950                 if (!elem)
2951                         break;
2952
2953                 TAILQ_REMOVE(&p->table_types, elem, node);
2954                 free(elem);
2955         }
2956 }
2957
2958 /*
2959  * Pipeline.
2960  */
2961 int
2962 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
2963 {
2964         struct rte_swx_pipeline *pipeline;
2965
2966         /* Check input parameters. */
2967         CHECK(p, EINVAL);
2968
2969         /* Memory allocation. */
2970         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
2971         CHECK(pipeline, ENOMEM);
2972
2973         /* Initialization. */
2974         TAILQ_INIT(&pipeline->struct_types);
2975         TAILQ_INIT(&pipeline->port_in_types);
2976         TAILQ_INIT(&pipeline->ports_in);
2977         TAILQ_INIT(&pipeline->port_out_types);
2978         TAILQ_INIT(&pipeline->ports_out);
2979         TAILQ_INIT(&pipeline->extern_types);
2980         TAILQ_INIT(&pipeline->extern_objs);
2981         TAILQ_INIT(&pipeline->extern_funcs);
2982         TAILQ_INIT(&pipeline->headers);
2983         TAILQ_INIT(&pipeline->actions);
2984         TAILQ_INIT(&pipeline->table_types);
2985         TAILQ_INIT(&pipeline->tables);
2986
2987         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
2988         pipeline->numa_node = numa_node;
2989
2990         *p = pipeline;
2991         return 0;
2992 }
2993
2994 void
2995 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
2996 {
2997         if (!p)
2998                 return;
2999
3000         free(p->instructions);
3001
3002         table_state_free(p);
3003         table_free(p);
3004         action_free(p);
3005         metadata_free(p);
3006         header_free(p);
3007         extern_func_free(p);
3008         extern_obj_free(p);
3009         port_out_free(p);
3010         port_in_free(p);
3011         struct_free(p);
3012
3013         free(p);
3014 }
3015
3016 int
3017 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
3018                                      const char **instructions,
3019                                      uint32_t n_instructions)
3020 {
3021         int err;
3022         uint32_t i;
3023
3024         err = instruction_config(p, NULL, instructions, n_instructions);
3025         if (err)
3026                 return err;
3027
3028         /* Thread instruction pointer reset. */
3029         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3030                 struct thread *t = &p->threads[i];
3031
3032                 thread_ip_reset(p, t);
3033         }
3034
3035         return 0;
3036 }
3037
3038 int
3039 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
3040 {
3041         int status;
3042
3043         CHECK(p, EINVAL);
3044         CHECK(p->build_done == 0, EEXIST);
3045
3046         status = port_in_build(p);
3047         if (status)
3048                 goto error;
3049
3050         status = port_out_build(p);
3051         if (status)
3052                 goto error;
3053
3054         status = struct_build(p);
3055         if (status)
3056                 goto error;
3057
3058         status = extern_obj_build(p);
3059         if (status)
3060                 goto error;
3061
3062         status = extern_func_build(p);
3063         if (status)
3064                 goto error;
3065
3066         status = header_build(p);
3067         if (status)
3068                 goto error;
3069
3070         status = metadata_build(p);
3071         if (status)
3072                 goto error;
3073
3074         status = action_build(p);
3075         if (status)
3076                 goto error;
3077
3078         status = table_build(p);
3079         if (status)
3080                 goto error;
3081
3082         status = table_state_build(p);
3083         if (status)
3084                 goto error;
3085
3086         p->build_done = 1;
3087         return 0;
3088
3089 error:
3090         table_state_build_free(p);
3091         table_build_free(p);
3092         action_build_free(p);
3093         metadata_build_free(p);
3094         header_build_free(p);
3095         extern_func_build_free(p);
3096         extern_obj_build_free(p);
3097         port_out_build_free(p);
3098         port_in_build_free(p);
3099         struct_build_free(p);
3100
3101         return status;
3102 }
3103
3104 void
3105 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
3106 {
3107         uint32_t i;
3108
3109         for (i = 0; i < n_instructions; i++)
3110                 instr_exec(p);
3111 }
3112
3113 /*
3114  * Control.
3115  */
3116 int
3117 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
3118                                  struct rte_swx_table_state **table_state)
3119 {
3120         if (!p || !table_state || !p->build_done)
3121                 return -EINVAL;
3122
3123         *table_state = p->table_state;
3124         return 0;
3125 }
3126
3127 int
3128 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
3129                                  struct rte_swx_table_state *table_state)
3130 {
3131         if (!p || !table_state || !p->build_done)
3132                 return -EINVAL;
3133
3134         p->table_state = table_state;
3135         return 0;
3136 }