lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258 };
 259
 260 struct instr_operand {
 261         uint8_t struct_id;
 262         uint8_t n_bits;
 263         uint8_t offset;
 264         uint8_t pad;
 265 };
 266
 267 struct instr_io {
 268         struct {
 269                 uint8_t offset;
 270                 uint8_t n_bits;
 271                 uint8_t pad[2];
 272         } io;
 273
 274         struct {
 275                 uint8_t header_id[8];
 276                 uint8_t struct_id[8];
 277                 uint8_t n_bytes[8];
 278         } hdr;
 279 };
 280
 281 struct instr_hdr_validity {
 282         uint8_t header_id;
 283 };
 284
 285 struct instr_dst_src {
 286         struct instr_operand dst;
 287         union {
 288                 struct instr_operand src;
 289                 uint32_t src_val;
 290         };
 291 };
 292
 293 struct instruction {
 294         enum instruction_type type;
 295         union {
 296                 struct instr_io io;
 297                 struct instr_hdr_validity valid;
 298                 struct instr_dst_src mov;
 299         };
 300 };
 301
 302 struct instruction_data {
 303         char label[RTE_SWX_NAME_SIZE];
 304         char jmp_label[RTE_SWX_NAME_SIZE];
 305         uint32_t n_users; /* user = jmp instruction to this instruction. */
 306         int invalid;
 307 };
 308
 309 /*
 310  * Action.
 311  */
 312 struct action {
 313         TAILQ_ENTRY(action) node;
 314         char name[RTE_SWX_NAME_SIZE];
 315         struct struct_type *st;
 316         struct instruction *instructions;
 317         uint32_t n_instructions;
 318         uint32_t id;
 319 };
 320
 321 TAILQ_HEAD(action_tailq, action);
 322
 323 /*
 324  * Table.
 325  */
 326 struct table_type {
 327         TAILQ_ENTRY(table_type) node;
 328         char name[RTE_SWX_NAME_SIZE];
 329         enum rte_swx_table_match_type match_type;
 330         struct rte_swx_table_ops ops;
 331 };
 332
 333 TAILQ_HEAD(table_type_tailq, table_type);
 334
 335 struct match_field {
 336         enum rte_swx_table_match_type match_type;
 337         struct field *field;
 338 };
 339
 340 struct table {
 341         TAILQ_ENTRY(table) node;
 342         char name[RTE_SWX_NAME_SIZE];
 343         char args[RTE_SWX_NAME_SIZE];
 344         struct table_type *type; /* NULL when n_fields == 0. */
 345
 346         /* Match. */
 347         struct match_field *fields;
 348         uint32_t n_fields;
 349         int is_header; /* Only valid when n_fields > 0. */
 350         struct header *header; /* Only valid when n_fields > 0. */
 351
 352         /* Action. */
 353         struct action **actions;
 354         struct action *default_action;
 355         uint8_t *default_action_data;
 356         uint32_t n_actions;
 357         int default_action_is_const;
 358         uint32_t action_data_size_max;
 359
 360         uint32_t size;
 361         uint32_t id;
 362 };
 363
 364 TAILQ_HEAD(table_tailq, table);
 365
 366 struct table_runtime {
 367         rte_swx_table_lookup_t func;
 368         void *mailbox;
 369         uint8_t **key;
 370 };
 371
 372 /*
 373  * Pipeline.
 374  */
 375 struct thread {
 376         /* Packet. */
 377         struct rte_swx_pkt pkt;
 378         uint8_t *ptr;
 379
 380         /* Structures. */
 381         uint8_t **structs;
 382
 383         /* Packet headers. */
 384         struct header_runtime *headers; /* Extracted or generated headers. */
 385         struct header_out_runtime *headers_out; /* Emitted headers. */
 386         uint8_t *header_storage;
 387         uint8_t *header_out_storage;
 388         uint64_t valid_headers;
 389         uint32_t n_headers_out;
 390
 391         /* Packet meta-data. */
 392         uint8_t *metadata;
 393
 394         /* Tables. */
 395         struct table_runtime *tables;
 396         struct rte_swx_table_state *table_state;
 397         uint64_t action_id;
 398         int hit; /* 0 = Miss, 1 = Hit. */
 399
 400         /* Extern objects and functions. */
 401         struct extern_obj_runtime *extern_objs;
 402         struct extern_func_runtime *extern_funcs;
 403
 404         /* Instructions. */
 405         struct instruction *ip;
 406         struct instruction *ret;
 407 };
 408
 409 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 410 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 411 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 412
 413 #define MOV(thread, ip)  \
 414 {                                                                              \
 415         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 416         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 417         uint64_t dst64 = *dst64_ptr;                                           \
 418         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 419                                                                                \
 420         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 421         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 422         uint64_t src64 = *src64_ptr;                                           \
 423         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 424         uint64_t src = src64 & src64_mask;                                     \
 425                                                                                \
 426         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 427 }
 428
 429 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 430
 431 #define MOV_S(thread, ip)  \
 432 {                                                                              \
 433         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 434         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 435         uint64_t dst64 = *dst64_ptr;                                           \
 436         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 437                                                                                \
 438         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 439         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 440         uint64_t src64 = *src64_ptr;                                           \
 441         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 442                                                                                \
 443         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 444 }
 445
 446 #else
 447
 448 #define MOV_S MOV
 449
 450 #endif
 451
 452 #define MOV_I(thread, ip)  \
 453 {                                                                              \
 454         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 455         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 456         uint64_t dst64 = *dst64_ptr;                                           \
 457         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 458                                                                                \
 459         uint64_t src = (ip)->mov.src_val;                                      \
 460                                                                                \
 461         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 462 }
 463
 464 #define METADATA_READ(thread, offset, n_bits)                                  \
 465 ({                                                                             \
 466         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 467         uint64_t m64 = *m64_ptr;                                               \
 468         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 469         (m64 & m64_mask);                                                      \
 470 })
 471
 472 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 473 {                                                                              \
 474         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 475         uint64_t m64 = *m64_ptr;                                               \
 476         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 477                                                                                \
 478         uint64_t m_new = value;                                                \
 479                                                                                \
 480         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 481 }
 482
 483 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 484 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 485 #endif
 486
 487 struct rte_swx_pipeline {
 488         struct struct_type_tailq struct_types;
 489         struct port_in_type_tailq port_in_types;
 490         struct port_in_tailq ports_in;
 491         struct port_out_type_tailq port_out_types;
 492         struct port_out_tailq ports_out;
 493         struct extern_type_tailq extern_types;
 494         struct extern_obj_tailq extern_objs;
 495         struct extern_func_tailq extern_funcs;
 496         struct header_tailq headers;
 497         struct struct_type *metadata_st;
 498         uint32_t metadata_struct_id;
 499         struct action_tailq actions;
 500         struct table_type_tailq table_types;
 501         struct table_tailq tables;
 502
 503         struct port_in_runtime *in;
 504         struct port_out_runtime *out;
 505         struct instruction **action_instructions;
 506         struct rte_swx_table_state *table_state;
 507         struct instruction *instructions;
 508         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 509
 510         uint32_t n_structs;
 511         uint32_t n_ports_in;
 512         uint32_t n_ports_out;
 513         uint32_t n_extern_objs;
 514         uint32_t n_extern_funcs;
 515         uint32_t n_actions;
 516         uint32_t n_tables;
 517         uint32_t n_headers;
 518         uint32_t thread_id;
 519         uint32_t port_id;
 520         uint32_t n_instructions;
 521         int build_done;
 522         int numa_node;
 523 };
 524
 525 /*
 526  * Struct.
 527  */
 528 static struct struct_type *
 529 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 530 {
 531         struct struct_type *elem;
 532
 533         TAILQ_FOREACH(elem, &p->struct_types, node)
 534                 if (strcmp(elem->name, name) == 0)
 535                         return elem;
 536
 537         return NULL;
 538 }
 539
 540 static struct field *
 541 struct_type_field_find(struct struct_type *st, const char *name)
 542 {
 543         uint32_t i;
 544
 545         for (i = 0; i < st->n_fields; i++) {
 546                 struct field *f = &st->fields[i];
 547
 548                 if (strcmp(f->name, name) == 0)
 549                         return f;
 550         }
 551
 552         return NULL;
 553 }
 554
 555 int
 556 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 557                                       const char *name,
 558                                       struct rte_swx_field_params *fields,
 559                                       uint32_t n_fields)
 560 {
 561         struct struct_type *st;
 562         uint32_t i;
 563
 564         CHECK(p, EINVAL);
 565         CHECK_NAME(name, EINVAL);
 566         CHECK(fields, EINVAL);
 567         CHECK(n_fields, EINVAL);
 568
 569         for (i = 0; i < n_fields; i++) {
 570                 struct rte_swx_field_params *f = &fields[i];
 571                 uint32_t j;
 572
 573                 CHECK_NAME(f->name, EINVAL);
 574                 CHECK(f->n_bits, EINVAL);
 575                 CHECK(f->n_bits <= 64, EINVAL);
 576                 CHECK((f->n_bits & 7) == 0, EINVAL);
 577
 578                 for (j = 0; j < i; j++) {
 579                         struct rte_swx_field_params *f_prev = &fields[j];
 580
 581                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 582                 }
 583         }
 584
 585         CHECK(!struct_type_find(p, name), EEXIST);
 586
 587         /* Node allocation. */
 588         st = calloc(1, sizeof(struct struct_type));
 589         CHECK(st, ENOMEM);
 590
 591         st->fields = calloc(n_fields, sizeof(struct field));
 592         if (!st->fields) {
 593                 free(st);
 594                 CHECK(0, ENOMEM);
 595         }
 596
 597         /* Node initialization. */
 598         strcpy(st->name, name);
 599         for (i = 0; i < n_fields; i++) {
 600                 struct field *dst = &st->fields[i];
 601                 struct rte_swx_field_params *src = &fields[i];
 602
 603                 strcpy(dst->name, src->name);
 604                 dst->n_bits = src->n_bits;
 605                 dst->offset = st->n_bits;
 606
 607                 st->n_bits += src->n_bits;
 608         }
 609         st->n_fields = n_fields;
 610
 611         /* Node add to tailq. */
 612         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 613
 614         return 0;
 615 }
 616
 617 static int
 618 struct_build(struct rte_swx_pipeline *p)
 619 {
 620         uint32_t i;
 621
 622         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 623                 struct thread *t = &p->threads[i];
 624
 625                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 626                 CHECK(t->structs, ENOMEM);
 627         }
 628
 629         return 0;
 630 }
 631
 632 static void
 633 struct_build_free(struct rte_swx_pipeline *p)
 634 {
 635         uint32_t i;
 636
 637         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 638                 struct thread *t = &p->threads[i];
 639
 640                 free(t->structs);
 641                 t->structs = NULL;
 642         }
 643 }
 644
 645 static void
 646 struct_free(struct rte_swx_pipeline *p)
 647 {
 648         struct_build_free(p);
 649
 650         /* Struct types. */
 651         for ( ; ; ) {
 652                 struct struct_type *elem;
 653
 654                 elem = TAILQ_FIRST(&p->struct_types);
 655                 if (!elem)
 656                         break;
 657
 658                 TAILQ_REMOVE(&p->struct_types, elem, node);
 659                 free(elem->fields);
 660                 free(elem);
 661         }
 662 }
 663
 664 /*
 665  * Input port.
 666  */
 667 static struct port_in_type *
 668 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 669 {
 670         struct port_in_type *elem;
 671
 672         if (!name)
 673                 return NULL;
 674
 675         TAILQ_FOREACH(elem, &p->port_in_types, node)
 676                 if (strcmp(elem->name, name) == 0)
 677                         return elem;
 678
 679         return NULL;
 680 }
 681
 682 int
 683 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 684                                        const char *name,
 685                                        struct rte_swx_port_in_ops *ops)
 686 {
 687         struct port_in_type *elem;
 688
 689         CHECK(p, EINVAL);
 690         CHECK_NAME(name, EINVAL);
 691         CHECK(ops, EINVAL);
 692         CHECK(ops->create, EINVAL);
 693         CHECK(ops->free, EINVAL);
 694         CHECK(ops->pkt_rx, EINVAL);
 695         CHECK(ops->stats_read, EINVAL);
 696
 697         CHECK(!port_in_type_find(p, name), EEXIST);
 698
 699         /* Node allocation. */
 700         elem = calloc(1, sizeof(struct port_in_type));
 701         CHECK(elem, ENOMEM);
 702
 703         /* Node initialization. */
 704         strcpy(elem->name, name);
 705         memcpy(&elem->ops, ops, sizeof(*ops));
 706
 707         /* Node add to tailq. */
 708         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 709
 710         return 0;
 711 }
 712
 713 static struct port_in *
 714 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 715 {
 716         struct port_in *port;
 717
 718         TAILQ_FOREACH(port, &p->ports_in, node)
 719                 if (port->id == port_id)
 720                         return port;
 721
 722         return NULL;
 723 }
 724
 725 int
 726 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 727                                 uint32_t port_id,
 728                                 const char *port_type_name,
 729                                 void *args)
 730 {
 731         struct port_in_type *type = NULL;
 732         struct port_in *port = NULL;
 733         void *obj = NULL;
 734
 735         CHECK(p, EINVAL);
 736
 737         CHECK(!port_in_find(p, port_id), EINVAL);
 738
 739         CHECK_NAME(port_type_name, EINVAL);
 740         type = port_in_type_find(p, port_type_name);
 741         CHECK(type, EINVAL);
 742
 743         obj = type->ops.create(args);
 744         CHECK(obj, ENODEV);
 745
 746         /* Node allocation. */
 747         port = calloc(1, sizeof(struct port_in));
 748         CHECK(port, ENOMEM);
 749
 750         /* Node initialization. */
 751         port->type = type;
 752         port->obj = obj;
 753         port->id = port_id;
 754
 755         /* Node add to tailq. */
 756         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 757         if (p->n_ports_in < port_id + 1)
 758                 p->n_ports_in = port_id + 1;
 759
 760         return 0;
 761 }
 762
 763 static int
 764 port_in_build(struct rte_swx_pipeline *p)
 765 {
 766         struct port_in *port;
 767         uint32_t i;
 768
 769         CHECK(p->n_ports_in, EINVAL);
 770         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 771
 772         for (i = 0; i < p->n_ports_in; i++)
 773                 CHECK(port_in_find(p, i), EINVAL);
 774
 775         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 776         CHECK(p->in, ENOMEM);
 777
 778         TAILQ_FOREACH(port, &p->ports_in, node) {
 779                 struct port_in_runtime *in = &p->in[port->id];
 780
 781                 in->pkt_rx = port->type->ops.pkt_rx;
 782                 in->obj = port->obj;
 783         }
 784
 785         return 0;
 786 }
 787
 788 static void
 789 port_in_build_free(struct rte_swx_pipeline *p)
 790 {
 791         free(p->in);
 792         p->in = NULL;
 793 }
 794
 795 static void
 796 port_in_free(struct rte_swx_pipeline *p)
 797 {
 798         port_in_build_free(p);
 799
 800         /* Input ports. */
 801         for ( ; ; ) {
 802                 struct port_in *port;
 803
 804                 port = TAILQ_FIRST(&p->ports_in);
 805                 if (!port)
 806                         break;
 807
 808                 TAILQ_REMOVE(&p->ports_in, port, node);
 809                 port->type->ops.free(port->obj);
 810                 free(port);
 811         }
 812
 813         /* Input port types. */
 814         for ( ; ; ) {
 815                 struct port_in_type *elem;
 816
 817                 elem = TAILQ_FIRST(&p->port_in_types);
 818                 if (!elem)
 819                         break;
 820
 821                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 822                 free(elem);
 823         }
 824 }
 825
 826 /*
 827  * Output port.
 828  */
 829 static struct port_out_type *
 830 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 831 {
 832         struct port_out_type *elem;
 833
 834         if (!name)
 835                 return NULL;
 836
 837         TAILQ_FOREACH(elem, &p->port_out_types, node)
 838                 if (!strcmp(elem->name, name))
 839                         return elem;
 840
 841         return NULL;
 842 }
 843
 844 int
 845 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
 846                                         const char *name,
 847                                         struct rte_swx_port_out_ops *ops)
 848 {
 849         struct port_out_type *elem;
 850
 851         CHECK(p, EINVAL);
 852         CHECK_NAME(name, EINVAL);
 853         CHECK(ops, EINVAL);
 854         CHECK(ops->create, EINVAL);
 855         CHECK(ops->free, EINVAL);
 856         CHECK(ops->pkt_tx, EINVAL);
 857         CHECK(ops->stats_read, EINVAL);
 858
 859         CHECK(!port_out_type_find(p, name), EEXIST);
 860
 861         /* Node allocation. */
 862         elem = calloc(1, sizeof(struct port_out_type));
 863         CHECK(elem, ENOMEM);
 864
 865         /* Node initialization. */
 866         strcpy(elem->name, name);
 867         memcpy(&elem->ops, ops, sizeof(*ops));
 868
 869         /* Node add to tailq. */
 870         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
 871
 872         return 0;
 873 }
 874
 875 static struct port_out *
 876 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
 877 {
 878         struct port_out *port;
 879
 880         TAILQ_FOREACH(port, &p->ports_out, node)
 881                 if (port->id == port_id)
 882                         return port;
 883
 884         return NULL;
 885 }
 886
 887 int
 888 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
 889                                  uint32_t port_id,
 890                                  const char *port_type_name,
 891                                  void *args)
 892 {
 893         struct port_out_type *type = NULL;
 894         struct port_out *port = NULL;
 895         void *obj = NULL;
 896
 897         CHECK(p, EINVAL);
 898
 899         CHECK(!port_out_find(p, port_id), EINVAL);
 900
 901         CHECK_NAME(port_type_name, EINVAL);
 902         type = port_out_type_find(p, port_type_name);
 903         CHECK(type, EINVAL);
 904
 905         obj = type->ops.create(args);
 906         CHECK(obj, ENODEV);
 907
 908         /* Node allocation. */
 909         port = calloc(1, sizeof(struct port_out));
 910         CHECK(port, ENOMEM);
 911
 912         /* Node initialization. */
 913         port->type = type;
 914         port->obj = obj;
 915         port->id = port_id;
 916
 917         /* Node add to tailq. */
 918         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
 919         if (p->n_ports_out < port_id + 1)
 920                 p->n_ports_out = port_id + 1;
 921
 922         return 0;
 923 }
 924
 925 static int
 926 port_out_build(struct rte_swx_pipeline *p)
 927 {
 928         struct port_out *port;
 929         uint32_t i;
 930
 931         CHECK(p->n_ports_out, EINVAL);
 932
 933         for (i = 0; i < p->n_ports_out; i++)
 934                 CHECK(port_out_find(p, i), EINVAL);
 935
 936         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
 937         CHECK(p->out, ENOMEM);
 938
 939         TAILQ_FOREACH(port, &p->ports_out, node) {
 940                 struct port_out_runtime *out = &p->out[port->id];
 941
 942                 out->pkt_tx = port->type->ops.pkt_tx;
 943                 out->flush = port->type->ops.flush;
 944                 out->obj = port->obj;
 945         }
 946
 947         return 0;
 948 }
 949
 950 static void
 951 port_out_build_free(struct rte_swx_pipeline *p)
 952 {
 953         free(p->out);
 954         p->out = NULL;
 955 }
 956
 957 static void
 958 port_out_free(struct rte_swx_pipeline *p)
 959 {
 960         port_out_build_free(p);
 961
 962         /* Output ports. */
 963         for ( ; ; ) {
 964                 struct port_out *port;
 965
 966                 port = TAILQ_FIRST(&p->ports_out);
 967                 if (!port)
 968                         break;
 969
 970                 TAILQ_REMOVE(&p->ports_out, port, node);
 971                 port->type->ops.free(port->obj);
 972                 free(port);
 973         }
 974
 975         /* Output port types. */
 976         for ( ; ; ) {
 977                 struct port_out_type *elem;
 978
 979                 elem = TAILQ_FIRST(&p->port_out_types);
 980                 if (!elem)
 981                         break;
 982
 983                 TAILQ_REMOVE(&p->port_out_types, elem, node);
 984                 free(elem);
 985         }
 986 }
 987
 988 /*
 989  * Extern object.
 990  */
 991 static struct extern_type *
 992 extern_type_find(struct rte_swx_pipeline *p, const char *name)
 993 {
 994         struct extern_type *elem;
 995
 996         TAILQ_FOREACH(elem, &p->extern_types, node)
 997                 if (strcmp(elem->name, name) == 0)
 998                         return elem;
 999
1000         return NULL;
1001 }
1002
1003 static struct extern_type_member_func *
1004 extern_type_member_func_find(struct extern_type *type, const char *name)
1005 {
1006         struct extern_type_member_func *elem;
1007
1008         TAILQ_FOREACH(elem, &type->funcs, node)
1009                 if (strcmp(elem->name, name) == 0)
1010                         return elem;
1011
1012         return NULL;
1013 }
1014
1015 static struct extern_obj *
1016 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1017 {
1018         struct extern_obj *elem;
1019
1020         TAILQ_FOREACH(elem, &p->extern_objs, node)
1021                 if (strcmp(elem->name, name) == 0)
1022                         return elem;
1023
1024         return NULL;
1025 }
1026
1027 static struct field *
1028 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1029                                const char *name,
1030                                struct extern_obj **object)
1031 {
1032         struct extern_obj *obj;
1033         struct field *f;
1034         char *obj_name, *field_name;
1035
1036         if ((name[0] != 'e') || (name[1] != '.'))
1037                 return NULL;
1038
1039         obj_name = strdup(&name[2]);
1040         if (!obj_name)
1041                 return NULL;
1042
1043         field_name = strchr(obj_name, '.');
1044         if (!field_name) {
1045                 free(obj_name);
1046                 return NULL;
1047         }
1048
1049         *field_name = 0;
1050         field_name++;
1051
1052         obj = extern_obj_find(p, obj_name);
1053         if (!obj) {
1054                 free(obj_name);
1055                 return NULL;
1056         }
1057
1058         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1059         if (!f) {
1060                 free(obj_name);
1061                 return NULL;
1062         }
1063
1064         if (object)
1065                 *object = obj;
1066
1067         free(obj_name);
1068         return f;
1069 }
1070
1071 int
1072 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1073         const char *name,
1074         const char *mailbox_struct_type_name,
1075         rte_swx_extern_type_constructor_t constructor,
1076         rte_swx_extern_type_destructor_t destructor)
1077 {
1078         struct extern_type *elem;
1079         struct struct_type *mailbox_struct_type;
1080
1081         CHECK(p, EINVAL);
1082
1083         CHECK_NAME(name, EINVAL);
1084         CHECK(!extern_type_find(p, name), EEXIST);
1085
1086         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1087         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1088         CHECK(mailbox_struct_type, EINVAL);
1089
1090         CHECK(constructor, EINVAL);
1091         CHECK(destructor, EINVAL);
1092
1093         /* Node allocation. */
1094         elem = calloc(1, sizeof(struct extern_type));
1095         CHECK(elem, ENOMEM);
1096
1097         /* Node initialization. */
1098         strcpy(elem->name, name);
1099         elem->mailbox_struct_type = mailbox_struct_type;
1100         elem->constructor = constructor;
1101         elem->destructor = destructor;
1102         TAILQ_INIT(&elem->funcs);
1103
1104         /* Node add to tailq. */
1105         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1106
1107         return 0;
1108 }
1109
1110 int
1111 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1112         const char *extern_type_name,
1113         const char *name,
1114         rte_swx_extern_type_member_func_t member_func)
1115 {
1116         struct extern_type *type;
1117         struct extern_type_member_func *type_member;
1118
1119         CHECK(p, EINVAL);
1120
1121         CHECK(extern_type_name, EINVAL);
1122         type = extern_type_find(p, extern_type_name);
1123         CHECK(type, EINVAL);
1124         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1125
1126         CHECK(name, EINVAL);
1127         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1128
1129         CHECK(member_func, EINVAL);
1130
1131         /* Node allocation. */
1132         type_member = calloc(1, sizeof(struct extern_type_member_func));
1133         CHECK(type_member, ENOMEM);
1134
1135         /* Node initialization. */
1136         strcpy(type_member->name, name);
1137         type_member->func = member_func;
1138         type_member->id = type->n_funcs;
1139
1140         /* Node add to tailq. */
1141         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1142         type->n_funcs++;
1143
1144         return 0;
1145 }
1146
1147 int
1148 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1149                                       const char *extern_type_name,
1150                                       const char *name,
1151                                       const char *args)
1152 {
1153         struct extern_type *type;
1154         struct extern_obj *obj;
1155         void *obj_handle;
1156
1157         CHECK(p, EINVAL);
1158
1159         CHECK_NAME(extern_type_name, EINVAL);
1160         type = extern_type_find(p, extern_type_name);
1161         CHECK(type, EINVAL);
1162
1163         CHECK_NAME(name, EINVAL);
1164         CHECK(!extern_obj_find(p, name), EEXIST);
1165
1166         /* Node allocation. */
1167         obj = calloc(1, sizeof(struct extern_obj));
1168         CHECK(obj, ENOMEM);
1169
1170         /* Object construction. */
1171         obj_handle = type->constructor(args);
1172         if (!obj_handle) {
1173                 free(obj);
1174                 CHECK(0, ENODEV);
1175         }
1176
1177         /* Node initialization. */
1178         strcpy(obj->name, name);
1179         obj->type = type;
1180         obj->obj = obj_handle;
1181         obj->struct_id = p->n_structs;
1182         obj->id = p->n_extern_objs;
1183
1184         /* Node add to tailq. */
1185         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1186         p->n_extern_objs++;
1187         p->n_structs++;
1188
1189         return 0;
1190 }
1191
1192 static int
1193 extern_obj_build(struct rte_swx_pipeline *p)
1194 {
1195         uint32_t i;
1196
1197         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1198                 struct thread *t = &p->threads[i];
1199                 struct extern_obj *obj;
1200
1201                 t->extern_objs = calloc(p->n_extern_objs,
1202                                         sizeof(struct extern_obj_runtime));
1203                 CHECK(t->extern_objs, ENOMEM);
1204
1205                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1206                         struct extern_obj_runtime *r =
1207                                 &t->extern_objs[obj->id];
1208                         struct extern_type_member_func *func;
1209                         uint32_t mailbox_size =
1210                                 obj->type->mailbox_struct_type->n_bits / 8;
1211
1212                         r->obj = obj->obj;
1213
1214                         r->mailbox = calloc(1, mailbox_size);
1215                         CHECK(r->mailbox, ENOMEM);
1216
1217                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1218                                 r->funcs[func->id] = func->func;
1219
1220                         t->structs[obj->struct_id] = r->mailbox;
1221                 }
1222         }
1223
1224         return 0;
1225 }
1226
1227 static void
1228 extern_obj_build_free(struct rte_swx_pipeline *p)
1229 {
1230         uint32_t i;
1231
1232         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1233                 struct thread *t = &p->threads[i];
1234                 uint32_t j;
1235
1236                 if (!t->extern_objs)
1237                         continue;
1238
1239                 for (j = 0; j < p->n_extern_objs; j++) {
1240                         struct extern_obj_runtime *r = &t->extern_objs[j];
1241
1242                         free(r->mailbox);
1243                 }
1244
1245                 free(t->extern_objs);
1246                 t->extern_objs = NULL;
1247         }
1248 }
1249
1250 static void
1251 extern_obj_free(struct rte_swx_pipeline *p)
1252 {
1253         extern_obj_build_free(p);
1254
1255         /* Extern objects. */
1256         for ( ; ; ) {
1257                 struct extern_obj *elem;
1258
1259                 elem = TAILQ_FIRST(&p->extern_objs);
1260                 if (!elem)
1261                         break;
1262
1263                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1264                 if (elem->obj)
1265                         elem->type->destructor(elem->obj);
1266                 free(elem);
1267         }
1268
1269         /* Extern types. */
1270         for ( ; ; ) {
1271                 struct extern_type *elem;
1272
1273                 elem = TAILQ_FIRST(&p->extern_types);
1274                 if (!elem)
1275                         break;
1276
1277                 TAILQ_REMOVE(&p->extern_types, elem, node);
1278
1279                 for ( ; ; ) {
1280                         struct extern_type_member_func *func;
1281
1282                         func = TAILQ_FIRST(&elem->funcs);
1283                         if (!func)
1284                                 break;
1285
1286                         TAILQ_REMOVE(&elem->funcs, func, node);
1287                         free(func);
1288                 }
1289
1290                 free(elem);
1291         }
1292 }
1293
1294 /*
1295  * Extern function.
1296  */
1297 static struct extern_func *
1298 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1299 {
1300         struct extern_func *elem;
1301
1302         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1303                 if (strcmp(elem->name, name) == 0)
1304                         return elem;
1305
1306         return NULL;
1307 }
1308
1309 static struct field *
1310 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1311                                 const char *name,
1312                                 struct extern_func **function)
1313 {
1314         struct extern_func *func;
1315         struct field *f;
1316         char *func_name, *field_name;
1317
1318         if ((name[0] != 'f') || (name[1] != '.'))
1319                 return NULL;
1320
1321         func_name = strdup(&name[2]);
1322         if (!func_name)
1323                 return NULL;
1324
1325         field_name = strchr(func_name, '.');
1326         if (!field_name) {
1327                 free(func_name);
1328                 return NULL;
1329         }
1330
1331         *field_name = 0;
1332         field_name++;
1333
1334         func = extern_func_find(p, func_name);
1335         if (!func) {
1336                 free(func_name);
1337                 return NULL;
1338         }
1339
1340         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1341         if (!f) {
1342                 free(func_name);
1343                 return NULL;
1344         }
1345
1346         if (function)
1347                 *function = func;
1348
1349         free(func_name);
1350         return f;
1351 }
1352
1353 int
1354 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1355                                       const char *name,
1356                                       const char *mailbox_struct_type_name,
1357                                       rte_swx_extern_func_t func)
1358 {
1359         struct extern_func *f;
1360         struct struct_type *mailbox_struct_type;
1361
1362         CHECK(p, EINVAL);
1363
1364         CHECK_NAME(name, EINVAL);
1365         CHECK(!extern_func_find(p, name), EEXIST);
1366
1367         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1368         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1369         CHECK(mailbox_struct_type, EINVAL);
1370
1371         CHECK(func, EINVAL);
1372
1373         /* Node allocation. */
1374         f = calloc(1, sizeof(struct extern_func));
1375         CHECK(func, ENOMEM);
1376
1377         /* Node initialization. */
1378         strcpy(f->name, name);
1379         f->mailbox_struct_type = mailbox_struct_type;
1380         f->func = func;
1381         f->struct_id = p->n_structs;
1382         f->id = p->n_extern_funcs;
1383
1384         /* Node add to tailq. */
1385         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1386         p->n_extern_funcs++;
1387         p->n_structs++;
1388
1389         return 0;
1390 }
1391
1392 static int
1393 extern_func_build(struct rte_swx_pipeline *p)
1394 {
1395         uint32_t i;
1396
1397         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1398                 struct thread *t = &p->threads[i];
1399                 struct extern_func *func;
1400
1401                 /* Memory allocation. */
1402                 t->extern_funcs = calloc(p->n_extern_funcs,
1403                                          sizeof(struct extern_func_runtime));
1404                 CHECK(t->extern_funcs, ENOMEM);
1405
1406                 /* Extern function. */
1407                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1408                         struct extern_func_runtime *r =
1409                                 &t->extern_funcs[func->id];
1410                         uint32_t mailbox_size =
1411                                 func->mailbox_struct_type->n_bits / 8;
1412
1413                         r->func = func->func;
1414
1415                         r->mailbox = calloc(1, mailbox_size);
1416                         CHECK(r->mailbox, ENOMEM);
1417
1418                         t->structs[func->struct_id] = r->mailbox;
1419                 }
1420         }
1421
1422         return 0;
1423 }
1424
1425 static void
1426 extern_func_build_free(struct rte_swx_pipeline *p)
1427 {
1428         uint32_t i;
1429
1430         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1431                 struct thread *t = &p->threads[i];
1432                 uint32_t j;
1433
1434                 if (!t->extern_funcs)
1435                         continue;
1436
1437                 for (j = 0; j < p->n_extern_funcs; j++) {
1438                         struct extern_func_runtime *r = &t->extern_funcs[j];
1439
1440                         free(r->mailbox);
1441                 }
1442
1443                 free(t->extern_funcs);
1444                 t->extern_funcs = NULL;
1445         }
1446 }
1447
1448 static void
1449 extern_func_free(struct rte_swx_pipeline *p)
1450 {
1451         extern_func_build_free(p);
1452
1453         for ( ; ; ) {
1454                 struct extern_func *elem;
1455
1456                 elem = TAILQ_FIRST(&p->extern_funcs);
1457                 if (!elem)
1458                         break;
1459
1460                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1461                 free(elem);
1462         }
1463 }
1464
1465 /*
1466  * Header.
1467  */
1468 static struct header *
1469 header_find(struct rte_swx_pipeline *p, const char *name)
1470 {
1471         struct header *elem;
1472
1473         TAILQ_FOREACH(elem, &p->headers, node)
1474                 if (strcmp(elem->name, name) == 0)
1475                         return elem;
1476
1477         return NULL;
1478 }
1479
1480 static struct header *
1481 header_parse(struct rte_swx_pipeline *p,
1482              const char *name)
1483 {
1484         if (name[0] != 'h' || name[1] != '.')
1485                 return NULL;
1486
1487         return header_find(p, &name[2]);
1488 }
1489
1490 static struct field *
1491 header_field_parse(struct rte_swx_pipeline *p,
1492                    const char *name,
1493                    struct header **header)
1494 {
1495         struct header *h;
1496         struct field *f;
1497         char *header_name, *field_name;
1498
1499         if ((name[0] != 'h') || (name[1] != '.'))
1500                 return NULL;
1501
1502         header_name = strdup(&name[2]);
1503         if (!header_name)
1504                 return NULL;
1505
1506         field_name = strchr(header_name, '.');
1507         if (!field_name) {
1508                 free(header_name);
1509                 return NULL;
1510         }
1511
1512         *field_name = 0;
1513         field_name++;
1514
1515         h = header_find(p, header_name);
1516         if (!h) {
1517                 free(header_name);
1518                 return NULL;
1519         }
1520
1521         f = struct_type_field_find(h->st, field_name);
1522         if (!f) {
1523                 free(header_name);
1524                 return NULL;
1525         }
1526
1527         if (header)
1528                 *header = h;
1529
1530         free(header_name);
1531         return f;
1532 }
1533
1534 int
1535 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1536                                         const char *name,
1537                                         const char *struct_type_name)
1538 {
1539         struct struct_type *st;
1540         struct header *h;
1541         size_t n_headers_max;
1542
1543         CHECK(p, EINVAL);
1544         CHECK_NAME(name, EINVAL);
1545         CHECK_NAME(struct_type_name, EINVAL);
1546
1547         CHECK(!header_find(p, name), EEXIST);
1548
1549         st = struct_type_find(p, struct_type_name);
1550         CHECK(st, EINVAL);
1551
1552         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1553         CHECK(p->n_headers < n_headers_max, ENOSPC);
1554
1555         /* Node allocation. */
1556         h = calloc(1, sizeof(struct header));
1557         CHECK(h, ENOMEM);
1558
1559         /* Node initialization. */
1560         strcpy(h->name, name);
1561         h->st = st;
1562         h->struct_id = p->n_structs;
1563         h->id = p->n_headers;
1564
1565         /* Node add to tailq. */
1566         TAILQ_INSERT_TAIL(&p->headers, h, node);
1567         p->n_headers++;
1568         p->n_structs++;
1569
1570         return 0;
1571 }
1572
1573 static int
1574 header_build(struct rte_swx_pipeline *p)
1575 {
1576         struct header *h;
1577         uint32_t n_bytes = 0, i;
1578
1579         TAILQ_FOREACH(h, &p->headers, node) {
1580                 n_bytes += h->st->n_bits / 8;
1581         }
1582
1583         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1584                 struct thread *t = &p->threads[i];
1585                 uint32_t offset = 0;
1586
1587                 t->headers = calloc(p->n_headers,
1588                                     sizeof(struct header_runtime));
1589                 CHECK(t->headers, ENOMEM);
1590
1591                 t->headers_out = calloc(p->n_headers,
1592                                         sizeof(struct header_out_runtime));
1593                 CHECK(t->headers_out, ENOMEM);
1594
1595                 t->header_storage = calloc(1, n_bytes);
1596                 CHECK(t->header_storage, ENOMEM);
1597
1598                 t->header_out_storage = calloc(1, n_bytes);
1599                 CHECK(t->header_out_storage, ENOMEM);
1600
1601                 TAILQ_FOREACH(h, &p->headers, node) {
1602                         uint8_t *header_storage;
1603
1604                         header_storage = &t->header_storage[offset];
1605                         offset += h->st->n_bits / 8;
1606
1607                         t->headers[h->id].ptr0 = header_storage;
1608                         t->structs[h->struct_id] = header_storage;
1609                 }
1610         }
1611
1612         return 0;
1613 }
1614
1615 static void
1616 header_build_free(struct rte_swx_pipeline *p)
1617 {
1618         uint32_t i;
1619
1620         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1621                 struct thread *t = &p->threads[i];
1622
1623                 free(t->headers_out);
1624                 t->headers_out = NULL;
1625
1626                 free(t->headers);
1627                 t->headers = NULL;
1628
1629                 free(t->header_out_storage);
1630                 t->header_out_storage = NULL;
1631
1632                 free(t->header_storage);
1633                 t->header_storage = NULL;
1634         }
1635 }
1636
1637 static void
1638 header_free(struct rte_swx_pipeline *p)
1639 {
1640         header_build_free(p);
1641
1642         for ( ; ; ) {
1643                 struct header *elem;
1644
1645                 elem = TAILQ_FIRST(&p->headers);
1646                 if (!elem)
1647                         break;
1648
1649                 TAILQ_REMOVE(&p->headers, elem, node);
1650                 free(elem);
1651         }
1652 }
1653
1654 /*
1655  * Meta-data.
1656  */
1657 static struct field *
1658 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1659 {
1660         if (!p->metadata_st)
1661                 return NULL;
1662
1663         if (name[0] != 'm' || name[1] != '.')
1664                 return NULL;
1665
1666         return struct_type_field_find(p->metadata_st, &name[2]);
1667 }
1668
1669 int
1670 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1671                                           const char *struct_type_name)
1672 {
1673         struct struct_type *st = NULL;
1674
1675         CHECK(p, EINVAL);
1676
1677         CHECK_NAME(struct_type_name, EINVAL);
1678         st  = struct_type_find(p, struct_type_name);
1679         CHECK(st, EINVAL);
1680         CHECK(!p->metadata_st, EINVAL);
1681
1682         p->metadata_st = st;
1683         p->metadata_struct_id = p->n_structs;
1684
1685         p->n_structs++;
1686
1687         return 0;
1688 }
1689
1690 static int
1691 metadata_build(struct rte_swx_pipeline *p)
1692 {
1693         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1694         uint32_t i;
1695
1696         /* Thread-level initialization. */
1697         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1698                 struct thread *t = &p->threads[i];
1699                 uint8_t *metadata;
1700
1701                 metadata = calloc(1, n_bytes);
1702                 CHECK(metadata, ENOMEM);
1703
1704                 t->metadata = metadata;
1705                 t->structs[p->metadata_struct_id] = metadata;
1706         }
1707
1708         return 0;
1709 }
1710
1711 static void
1712 metadata_build_free(struct rte_swx_pipeline *p)
1713 {
1714         uint32_t i;
1715
1716         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1717                 struct thread *t = &p->threads[i];
1718
1719                 free(t->metadata);
1720                 t->metadata = NULL;
1721         }
1722 }
1723
1724 static void
1725 metadata_free(struct rte_swx_pipeline *p)
1726 {
1727         metadata_build_free(p);
1728 }
1729
1730 /*
1731  * Instruction.
1732  */
1733 static struct field *
1734 action_field_parse(struct action *action, const char *name);
1735
1736 static struct field *
1737 struct_field_parse(struct rte_swx_pipeline *p,
1738                    struct action *action,
1739                    const char *name,
1740                    uint32_t *struct_id)
1741 {
1742         struct field *f;
1743
1744         switch (name[0]) {
1745         case 'h':
1746         {
1747                 struct header *header;
1748
1749                 f = header_field_parse(p, name, &header);
1750                 if (!f)
1751                         return NULL;
1752
1753                 *struct_id = header->struct_id;
1754                 return f;
1755         }
1756
1757         case 'm':
1758         {
1759                 f = metadata_field_parse(p, name);
1760                 if (!f)
1761                         return NULL;
1762
1763                 *struct_id = p->metadata_struct_id;
1764                 return f;
1765         }
1766
1767         case 't':
1768         {
1769                 if (!action)
1770                         return NULL;
1771
1772                 f = action_field_parse(action, name);
1773                 if (!f)
1774                         return NULL;
1775
1776                 *struct_id = 0;
1777                 return f;
1778         }
1779
1780         case 'e':
1781         {
1782                 struct extern_obj *obj;
1783
1784                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1785                 if (!f)
1786                         return NULL;
1787
1788                 *struct_id = obj->struct_id;
1789                 return f;
1790         }
1791
1792         case 'f':
1793         {
1794                 struct extern_func *func;
1795
1796                 f = extern_func_mailbox_field_parse(p, name, &func);
1797                 if (!f)
1798                         return NULL;
1799
1800                 *struct_id = func->struct_id;
1801                 return f;
1802         }
1803
1804         default:
1805                 return NULL;
1806         }
1807 }
1808
1809 static inline void
1810 pipeline_port_inc(struct rte_swx_pipeline *p)
1811 {
1812         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1813 }
1814
1815 static inline void
1816 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1817 {
1818         t->ip = p->instructions;
1819 }
1820
1821 static inline void
1822 thread_ip_inc(struct rte_swx_pipeline *p);
1823
1824 static inline void
1825 thread_ip_inc(struct rte_swx_pipeline *p)
1826 {
1827         struct thread *t = &p->threads[p->thread_id];
1828
1829         t->ip++;
1830 }
1831
1832 static inline void
1833 thread_ip_inc_cond(struct thread *t, int cond)
1834 {
1835         t->ip += cond;
1836 }
1837
1838 static inline void
1839 thread_yield(struct rte_swx_pipeline *p)
1840 {
1841         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1842 }
1843
1844 /*
1845  * rx.
1846  */
1847 static int
1848 instr_rx_translate(struct rte_swx_pipeline *p,
1849                    struct action *action,
1850                    char **tokens,
1851                    int n_tokens,
1852                    struct instruction *instr,
1853                    struct instruction_data *data __rte_unused)
1854 {
1855         struct field *f;
1856
1857         CHECK(!action, EINVAL);
1858         CHECK(n_tokens == 2, EINVAL);
1859
1860         f = metadata_field_parse(p, tokens[1]);
1861         CHECK(f, EINVAL);
1862
1863         instr->type = INSTR_RX;
1864         instr->io.io.offset = f->offset / 8;
1865         instr->io.io.n_bits = f->n_bits;
1866         return 0;
1867 }
1868
1869 static inline void
1870 instr_rx_exec(struct rte_swx_pipeline *p);
1871
1872 static inline void
1873 instr_rx_exec(struct rte_swx_pipeline *p)
1874 {
1875         struct thread *t = &p->threads[p->thread_id];
1876         struct instruction *ip = t->ip;
1877         struct port_in_runtime *port = &p->in[p->port_id];
1878         struct rte_swx_pkt *pkt = &t->pkt;
1879         int pkt_received;
1880
1881         /* Packet. */
1882         pkt_received = port->pkt_rx(port->obj, pkt);
1883         t->ptr = &pkt->pkt[pkt->offset];
1884         rte_prefetch0(t->ptr);
1885
1886         TRACE("[Thread %2u] rx %s from port %u\n",
1887               p->thread_id,
1888               pkt_received ? "1 pkt" : "0 pkts",
1889               p->port_id);
1890
1891         /* Headers. */
1892         t->valid_headers = 0;
1893         t->n_headers_out = 0;
1894
1895         /* Meta-data. */
1896         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1897
1898         /* Tables. */
1899         t->table_state = p->table_state;
1900
1901         /* Thread. */
1902         pipeline_port_inc(p);
1903         thread_ip_inc_cond(t, pkt_received);
1904         thread_yield(p);
1905 }
1906
1907 /*
1908  * tx.
1909  */
1910 static int
1911 instr_tx_translate(struct rte_swx_pipeline *p,
1912                    struct action *action __rte_unused,
1913                    char **tokens,
1914                    int n_tokens,
1915                    struct instruction *instr,
1916                    struct instruction_data *data __rte_unused)
1917 {
1918         struct field *f;
1919
1920         CHECK(n_tokens == 2, EINVAL);
1921
1922         f = metadata_field_parse(p, tokens[1]);
1923         CHECK(f, EINVAL);
1924
1925         instr->type = INSTR_TX;
1926         instr->io.io.offset = f->offset / 8;
1927         instr->io.io.n_bits = f->n_bits;
1928         return 0;
1929 }
1930
1931 static inline void
1932 emit_handler(struct thread *t)
1933 {
1934         struct header_out_runtime *h0 = &t->headers_out[0];
1935         struct header_out_runtime *h1 = &t->headers_out[1];
1936         uint32_t offset = 0, i;
1937
1938         /* No header change or header decapsulation. */
1939         if ((t->n_headers_out == 1) &&
1940             (h0->ptr + h0->n_bytes == t->ptr)) {
1941                 TRACE("Emit handler: no header change or header decap.\n");
1942
1943                 t->pkt.offset -= h0->n_bytes;
1944                 t->pkt.length += h0->n_bytes;
1945
1946                 return;
1947         }
1948
1949         /* Header encapsulation (optionally, with prior header decasulation). */
1950         if ((t->n_headers_out == 2) &&
1951             (h1->ptr + h1->n_bytes == t->ptr) &&
1952             (h0->ptr == h0->ptr0)) {
1953                 uint32_t offset;
1954
1955                 TRACE("Emit handler: header encapsulation.\n");
1956
1957                 offset = h0->n_bytes + h1->n_bytes;
1958                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1959                 t->pkt.offset -= offset;
1960                 t->pkt.length += offset;
1961
1962                 return;
1963         }
1964
1965         /* Header insertion. */
1966         /* TBD */
1967
1968         /* Header extraction. */
1969         /* TBD */
1970
1971         /* For any other case. */
1972         TRACE("Emit handler: complex case.\n");
1973
1974         for (i = 0; i < t->n_headers_out; i++) {
1975                 struct header_out_runtime *h = &t->headers_out[i];
1976
1977                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1978                 offset += h->n_bytes;
1979         }
1980
1981         if (offset) {
1982                 memcpy(t->ptr - offset, t->header_out_storage, offset);
1983                 t->pkt.offset -= offset;
1984                 t->pkt.length += offset;
1985         }
1986 }
1987
1988 static inline void
1989 instr_tx_exec(struct rte_swx_pipeline *p);
1990
1991 static inline void
1992 instr_tx_exec(struct rte_swx_pipeline *p)
1993 {
1994         struct thread *t = &p->threads[p->thread_id];
1995         struct instruction *ip = t->ip;
1996         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1997         struct port_out_runtime *port = &p->out[port_id];
1998         struct rte_swx_pkt *pkt = &t->pkt;
1999
2000         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2001               p->thread_id,
2002               (uint32_t)port_id);
2003
2004         /* Headers. */
2005         emit_handler(t);
2006
2007         /* Packet. */
2008         port->pkt_tx(port->obj, pkt);
2009
2010         /* Thread. */
2011         thread_ip_reset(p, t);
2012         instr_rx_exec(p);
2013 }
2014
2015 /*
2016  * extract.
2017  */
2018 static int
2019 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2020                             struct action *action,
2021                             char **tokens,
2022                             int n_tokens,
2023                             struct instruction *instr,
2024                             struct instruction_data *data __rte_unused)
2025 {
2026         struct header *h;
2027
2028         CHECK(!action, EINVAL);
2029         CHECK(n_tokens == 2, EINVAL);
2030
2031         h = header_parse(p, tokens[1]);
2032         CHECK(h, EINVAL);
2033
2034         instr->type = INSTR_HDR_EXTRACT;
2035         instr->io.hdr.header_id[0] = h->id;
2036         instr->io.hdr.struct_id[0] = h->struct_id;
2037         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2038         return 0;
2039 }
2040
2041 static inline void
2042 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2043
2044 static inline void
2045 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2046 {
2047         struct thread *t = &p->threads[p->thread_id];
2048         struct instruction *ip = t->ip;
2049         uint64_t valid_headers = t->valid_headers;
2050         uint8_t *ptr = t->ptr;
2051         uint32_t offset = t->pkt.offset;
2052         uint32_t length = t->pkt.length;
2053         uint32_t i;
2054
2055         for (i = 0; i < n_extract; i++) {
2056                 uint32_t header_id = ip->io.hdr.header_id[i];
2057                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2058                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2059
2060                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2061                       p->thread_id,
2062                       header_id,
2063                       n_bytes);
2064
2065                 /* Headers. */
2066                 t->structs[struct_id] = ptr;
2067                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2068
2069                 /* Packet. */
2070                 offset += n_bytes;
2071                 length -= n_bytes;
2072                 ptr += n_bytes;
2073         }
2074
2075         /* Headers. */
2076         t->valid_headers = valid_headers;
2077
2078         /* Packet. */
2079         t->pkt.offset = offset;
2080         t->pkt.length = length;
2081         t->ptr = ptr;
2082 }
2083
2084 static inline void
2085 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2086 {
2087         __instr_hdr_extract_exec(p, 1);
2088
2089         /* Thread. */
2090         thread_ip_inc(p);
2091 }
2092
2093 static inline void
2094 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2095 {
2096         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2097               p->thread_id);
2098
2099         __instr_hdr_extract_exec(p, 2);
2100
2101         /* Thread. */
2102         thread_ip_inc(p);
2103 }
2104
2105 static inline void
2106 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2107 {
2108         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2109               p->thread_id);
2110
2111         __instr_hdr_extract_exec(p, 3);
2112
2113         /* Thread. */
2114         thread_ip_inc(p);
2115 }
2116
2117 static inline void
2118 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2119 {
2120         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2121               p->thread_id);
2122
2123         __instr_hdr_extract_exec(p, 4);
2124
2125         /* Thread. */
2126         thread_ip_inc(p);
2127 }
2128
2129 static inline void
2130 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2131 {
2132         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2133               p->thread_id);
2134
2135         __instr_hdr_extract_exec(p, 5);
2136
2137         /* Thread. */
2138         thread_ip_inc(p);
2139 }
2140
2141 static inline void
2142 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2143 {
2144         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2145               p->thread_id);
2146
2147         __instr_hdr_extract_exec(p, 6);
2148
2149         /* Thread. */
2150         thread_ip_inc(p);
2151 }
2152
2153 static inline void
2154 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2155 {
2156         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2157               p->thread_id);
2158
2159         __instr_hdr_extract_exec(p, 7);
2160
2161         /* Thread. */
2162         thread_ip_inc(p);
2163 }
2164
2165 static inline void
2166 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2167 {
2168         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2169               p->thread_id);
2170
2171         __instr_hdr_extract_exec(p, 8);
2172
2173         /* Thread. */
2174         thread_ip_inc(p);
2175 }
2176
2177 /*
2178  * emit.
2179  */
2180 static int
2181 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2182                          struct action *action __rte_unused,
2183                          char **tokens,
2184                          int n_tokens,
2185                          struct instruction *instr,
2186                          struct instruction_data *data __rte_unused)
2187 {
2188         struct header *h;
2189
2190         CHECK(n_tokens == 2, EINVAL);
2191
2192         h = header_parse(p, tokens[1]);
2193         CHECK(h, EINVAL);
2194
2195         instr->type = INSTR_HDR_EMIT;
2196         instr->io.hdr.header_id[0] = h->id;
2197         instr->io.hdr.struct_id[0] = h->struct_id;
2198         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2199         return 0;
2200 }
2201
2202 static inline void
2203 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2204
2205 static inline void
2206 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2207 {
2208         struct thread *t = &p->threads[p->thread_id];
2209         struct instruction *ip = t->ip;
2210         uint32_t n_headers_out = t->n_headers_out;
2211         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2212         uint8_t *ho_ptr = NULL;
2213         uint32_t ho_nbytes = 0, i;
2214
2215         for (i = 0; i < n_emit; i++) {
2216                 uint32_t header_id = ip->io.hdr.header_id[i];
2217                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2218                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2219
2220                 struct header_runtime *hi = &t->headers[header_id];
2221                 uint8_t *hi_ptr = t->structs[struct_id];
2222
2223                 TRACE("[Thread %2u]: emit header %u\n",
2224                       p->thread_id,
2225                       header_id);
2226
2227                 /* Headers. */
2228                 if (!i) {
2229                         if (!t->n_headers_out) {
2230                                 ho = &t->headers_out[0];
2231
2232                                 ho->ptr0 = hi->ptr0;
2233                                 ho->ptr = hi_ptr;
2234
2235                                 ho_ptr = hi_ptr;
2236                                 ho_nbytes = n_bytes;
2237
2238                                 n_headers_out = 1;
2239
2240                                 continue;
2241                         } else {
2242                                 ho_ptr = ho->ptr;
2243                                 ho_nbytes = ho->n_bytes;
2244                         }
2245                 }
2246
2247                 if (ho_ptr + ho_nbytes == hi_ptr) {
2248                         ho_nbytes += n_bytes;
2249                 } else {
2250                         ho->n_bytes = ho_nbytes;
2251
2252                         ho++;
2253                         ho->ptr0 = hi->ptr0;
2254                         ho->ptr = hi_ptr;
2255
2256                         ho_ptr = hi_ptr;
2257                         ho_nbytes = n_bytes;
2258
2259                         n_headers_out++;
2260                 }
2261         }
2262
2263         ho->n_bytes = ho_nbytes;
2264         t->n_headers_out = n_headers_out;
2265 }
2266
2267 static inline void
2268 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2269 {
2270         __instr_hdr_emit_exec(p, 1);
2271
2272         /* Thread. */
2273         thread_ip_inc(p);
2274 }
2275
2276 static inline void
2277 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2278 {
2279         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2280               p->thread_id);
2281
2282         __instr_hdr_emit_exec(p, 1);
2283         instr_tx_exec(p);
2284 }
2285
2286 static inline void
2287 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2288 {
2289         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2290               p->thread_id);
2291
2292         __instr_hdr_emit_exec(p, 2);
2293         instr_tx_exec(p);
2294 }
2295
2296 static inline void
2297 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2298 {
2299         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2300               p->thread_id);
2301
2302         __instr_hdr_emit_exec(p, 3);
2303         instr_tx_exec(p);
2304 }
2305
2306 static inline void
2307 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2308 {
2309         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2310               p->thread_id);
2311
2312         __instr_hdr_emit_exec(p, 4);
2313         instr_tx_exec(p);
2314 }
2315
2316 static inline void
2317 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2318 {
2319         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2320               p->thread_id);
2321
2322         __instr_hdr_emit_exec(p, 5);
2323         instr_tx_exec(p);
2324 }
2325
2326 static inline void
2327 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2328 {
2329         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2330               p->thread_id);
2331
2332         __instr_hdr_emit_exec(p, 6);
2333         instr_tx_exec(p);
2334 }
2335
2336 static inline void
2337 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2338 {
2339         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2340               p->thread_id);
2341
2342         __instr_hdr_emit_exec(p, 7);
2343         instr_tx_exec(p);
2344 }
2345
2346 static inline void
2347 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2348 {
2349         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2350               p->thread_id);
2351
2352         __instr_hdr_emit_exec(p, 8);
2353         instr_tx_exec(p);
2354 }
2355
2356 /*
2357  * validate.
2358  */
2359 static int
2360 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2361                              struct action *action __rte_unused,
2362                              char **tokens,
2363                              int n_tokens,
2364                              struct instruction *instr,
2365                              struct instruction_data *data __rte_unused)
2366 {
2367         struct header *h;
2368
2369         CHECK(n_tokens == 2, EINVAL);
2370
2371         h = header_parse(p, tokens[1]);
2372         CHECK(h, EINVAL);
2373
2374         instr->type = INSTR_HDR_VALIDATE;
2375         instr->valid.header_id = h->id;
2376         return 0;
2377 }
2378
2379 static inline void
2380 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2381 {
2382         struct thread *t = &p->threads[p->thread_id];
2383         struct instruction *ip = t->ip;
2384         uint32_t header_id = ip->valid.header_id;
2385
2386         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2387
2388         /* Headers. */
2389         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2390
2391         /* Thread. */
2392         thread_ip_inc(p);
2393 }
2394
2395 /*
2396  * invalidate.
2397  */
2398 static int
2399 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2400                                struct action *action __rte_unused,
2401                                char **tokens,
2402                                int n_tokens,
2403                                struct instruction *instr,
2404                                struct instruction_data *data __rte_unused)
2405 {
2406         struct header *h;
2407
2408         CHECK(n_tokens == 2, EINVAL);
2409
2410         h = header_parse(p, tokens[1]);
2411         CHECK(h, EINVAL);
2412
2413         instr->type = INSTR_HDR_INVALIDATE;
2414         instr->valid.header_id = h->id;
2415         return 0;
2416 }
2417
2418 static inline void
2419 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2420 {
2421         struct thread *t = &p->threads[p->thread_id];
2422         struct instruction *ip = t->ip;
2423         uint32_t header_id = ip->valid.header_id;
2424
2425         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2426
2427         /* Headers. */
2428         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2429
2430         /* Thread. */
2431         thread_ip_inc(p);
2432 }
2433
2434 /*
2435  * mov.
2436  */
2437 static int
2438 instr_mov_translate(struct rte_swx_pipeline *p,
2439                     struct action *action,
2440                     char **tokens,
2441                     int n_tokens,
2442                     struct instruction *instr,
2443                     struct instruction_data *data __rte_unused)
2444 {
2445         char *dst = tokens[1], *src = tokens[2];
2446         struct field *fdst, *fsrc;
2447         uint32_t dst_struct_id, src_struct_id, src_val;
2448
2449         CHECK(n_tokens == 3, EINVAL);
2450
2451         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2452         CHECK(fdst, EINVAL);
2453
2454         /* MOV or MOV_S. */
2455         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2456         if (fsrc) {
2457                 instr->type = INSTR_MOV;
2458                 if ((dst[0] == 'h' && src[0] != 'h') ||
2459                     (dst[0] != 'h' && src[0] == 'h'))
2460                         instr->type = INSTR_MOV_S;
2461
2462                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2463                 instr->mov.dst.n_bits = fdst->n_bits;
2464                 instr->mov.dst.offset = fdst->offset / 8;
2465                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2466                 instr->mov.src.n_bits = fsrc->n_bits;
2467                 instr->mov.src.offset = fsrc->offset / 8;
2468                 return 0;
2469         }
2470
2471         /* MOV_I. */
2472         src_val = strtoul(src, &src, 0);
2473         CHECK(!src[0], EINVAL);
2474
2475         if (dst[0] == 'h')
2476                 src_val = htonl(src_val);
2477
2478         instr->type = INSTR_MOV_I;
2479         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2480         instr->mov.dst.n_bits = fdst->n_bits;
2481         instr->mov.dst.offset = fdst->offset / 8;
2482         instr->mov.src_val = (uint32_t)src_val;
2483         return 0;
2484 }
2485
2486 static inline void
2487 instr_mov_exec(struct rte_swx_pipeline *p)
2488 {
2489         struct thread *t = &p->threads[p->thread_id];
2490         struct instruction *ip = t->ip;
2491
2492         TRACE("[Thread %2u] mov\n",
2493               p->thread_id);
2494
2495         MOV(t, ip);
2496
2497         /* Thread. */
2498         thread_ip_inc(p);
2499 }
2500
2501 static inline void
2502 instr_mov_s_exec(struct rte_swx_pipeline *p)
2503 {
2504         struct thread *t = &p->threads[p->thread_id];
2505         struct instruction *ip = t->ip;
2506
2507         TRACE("[Thread %2u] mov (s)\n",
2508               p->thread_id);
2509
2510         MOV_S(t, ip);
2511
2512         /* Thread. */
2513         thread_ip_inc(p);
2514 }
2515
2516 static inline void
2517 instr_mov_i_exec(struct rte_swx_pipeline *p)
2518 {
2519         struct thread *t = &p->threads[p->thread_id];
2520         struct instruction *ip = t->ip;
2521
2522         TRACE("[Thread %2u] mov m.f %x\n",
2523               p->thread_id,
2524               ip->mov.src_val);
2525
2526         MOV_I(t, ip);
2527
2528         /* Thread. */
2529         thread_ip_inc(p);
2530 }
2531
2532 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
2533
2534 static int
2535 instr_translate(struct rte_swx_pipeline *p,
2536                 struct action *action,
2537                 char *string,
2538                 struct instruction *instr,
2539                 struct instruction_data *data)
2540 {
2541         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
2542         int n_tokens = 0, tpos = 0;
2543
2544         /* Parse the instruction string into tokens. */
2545         for ( ; ; ) {
2546                 char *token;
2547
2548                 token = strtok_r(string, " \t\v", &string);
2549                 if (!token)
2550                         break;
2551
2552                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
2553
2554                 tokens[n_tokens] = token;
2555                 n_tokens++;
2556         }
2557
2558         CHECK(n_tokens, EINVAL);
2559
2560         /* Handle the optional instruction label. */
2561         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
2562                 strcpy(data->label, tokens[0]);
2563
2564                 tpos += 2;
2565                 CHECK(n_tokens - tpos, EINVAL);
2566         }
2567
2568         /* Identify the instruction type. */
2569         if (!strcmp(tokens[tpos], "rx"))
2570                 return instr_rx_translate(p,
2571                                           action,
2572                                           &tokens[tpos],
2573                                           n_tokens - tpos,
2574                                           instr,
2575                                           data);
2576
2577         if (!strcmp(tokens[tpos], "tx"))
2578                 return instr_tx_translate(p,
2579                                           action,
2580                                           &tokens[tpos],
2581                                           n_tokens - tpos,
2582                                           instr,
2583                                           data);
2584
2585         if (!strcmp(tokens[tpos], "extract"))
2586                 return instr_hdr_extract_translate(p,
2587                                                    action,
2588                                                    &tokens[tpos],
2589                                                    n_tokens - tpos,
2590                                                    instr,
2591                                                    data);
2592
2593         if (!strcmp(tokens[tpos], "emit"))
2594                 return instr_hdr_emit_translate(p,
2595                                                 action,
2596                                                 &tokens[tpos],
2597                                                 n_tokens - tpos,
2598                                                 instr,
2599                                                 data);
2600
2601         if (!strcmp(tokens[tpos], "validate"))
2602                 return instr_hdr_validate_translate(p,
2603                                                     action,
2604                                                     &tokens[tpos],
2605                                                     n_tokens - tpos,
2606                                                     instr,
2607                                                     data);
2608
2609         if (!strcmp(tokens[tpos], "invalidate"))
2610                 return instr_hdr_invalidate_translate(p,
2611                                                       action,
2612                                                       &tokens[tpos],
2613                                                       n_tokens - tpos,
2614                                                       instr,
2615                                                       data);
2616
2617         if (!strcmp(tokens[tpos], "mov"))
2618                 return instr_mov_translate(p,
2619                                            action,
2620                                            &tokens[tpos],
2621                                            n_tokens - tpos,
2622                                            instr,
2623                                            data);
2624
2625         CHECK(0, EINVAL);
2626 }
2627
2628 static uint32_t
2629 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
2630 {
2631         uint32_t count = 0, i;
2632
2633         if (!label[0])
2634                 return 0;
2635
2636         for (i = 0; i < n; i++)
2637                 if (!strcmp(label, data[i].jmp_label))
2638                         count++;
2639
2640         return count;
2641 }
2642
2643 static int
2644 instr_label_check(struct instruction_data *instruction_data,
2645                   uint32_t n_instructions)
2646 {
2647         uint32_t i;
2648
2649         /* Check that all instruction labels are unique. */
2650         for (i = 0; i < n_instructions; i++) {
2651                 struct instruction_data *data = &instruction_data[i];
2652                 char *label = data->label;
2653                 uint32_t j;
2654
2655                 if (!label[0])
2656                         continue;
2657
2658                 for (j = i + 1; j < n_instructions; j++)
2659                         CHECK(strcmp(label, data[j].label), EINVAL);
2660         }
2661
2662         /* Get users for each instruction label. */
2663         for (i = 0; i < n_instructions; i++) {
2664                 struct instruction_data *data = &instruction_data[i];
2665                 char *label = data->label;
2666
2667                 data->n_users = label_is_used(instruction_data,
2668                                               n_instructions,
2669                                               label);
2670         }
2671
2672         return 0;
2673 }
2674
2675 static int
2676 instruction_config(struct rte_swx_pipeline *p,
2677                    struct action *a,
2678                    const char **instructions,
2679                    uint32_t n_instructions)
2680 {
2681         struct instruction *instr = NULL;
2682         struct instruction_data *data = NULL;
2683         char *string = NULL;
2684         int err = 0;
2685         uint32_t i;
2686
2687         CHECK(n_instructions, EINVAL);
2688         CHECK(instructions, EINVAL);
2689         for (i = 0; i < n_instructions; i++)
2690                 CHECK(instructions[i], EINVAL);
2691
2692         /* Memory allocation. */
2693         instr = calloc(n_instructions, sizeof(struct instruction));
2694         if (!instr) {
2695                 err = ENOMEM;
2696                 goto error;
2697         }
2698
2699         data = calloc(n_instructions, sizeof(struct instruction_data));
2700         if (!data) {
2701                 err = ENOMEM;
2702                 goto error;
2703         }
2704
2705         for (i = 0; i < n_instructions; i++) {
2706                 string = strdup(instructions[i]);
2707                 if (!string) {
2708                         err = ENOMEM;
2709                         goto error;
2710                 }
2711
2712                 err = instr_translate(p, a, string, &instr[i], &data[i]);
2713                 if (err)
2714                         goto error;
2715
2716                 free(string);
2717         }
2718
2719         err = instr_label_check(data, n_instructions);
2720         if (err)
2721                 goto error;
2722
2723         free(data);
2724
2725         if (a) {
2726                 a->instructions = instr;
2727                 a->n_instructions = n_instructions;
2728         } else {
2729                 p->instructions = instr;
2730                 p->n_instructions = n_instructions;
2731         }
2732
2733         return 0;
2734
2735 error:
2736         free(string);
2737         free(data);
2738         free(instr);
2739         return err;
2740 }
2741
2742 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
2743
2744 static instr_exec_t instruction_table[] = {
2745         [INSTR_RX] = instr_rx_exec,
2746         [INSTR_TX] = instr_tx_exec,
2747
2748         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
2749         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
2750         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
2751         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
2752         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
2753         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
2754         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
2755         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
2756
2757         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
2758         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
2759         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
2760         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
2761         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
2762         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
2763         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
2764         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
2765         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
2766
2767         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
2768         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
2769
2770         [INSTR_MOV] = instr_mov_exec,
2771         [INSTR_MOV_S] = instr_mov_s_exec,
2772         [INSTR_MOV_I] = instr_mov_i_exec,
2773 };
2774
2775 static inline void
2776 instr_exec(struct rte_swx_pipeline *p)
2777 {
2778         struct thread *t = &p->threads[p->thread_id];
2779         struct instruction *ip = t->ip;
2780         instr_exec_t instr = instruction_table[ip->type];
2781
2782         instr(p);
2783 }
2784
2785 /*
2786  * Action.
2787  */
2788 static struct action *
2789 action_find(struct rte_swx_pipeline *p, const char *name)
2790 {
2791         struct action *elem;
2792
2793         if (!name)
2794                 return NULL;
2795
2796         TAILQ_FOREACH(elem, &p->actions, node)
2797                 if (strcmp(elem->name, name) == 0)
2798                         return elem;
2799
2800         return NULL;
2801 }
2802
2803 static struct field *
2804 action_field_find(struct action *a, const char *name)
2805 {
2806         return a->st ? struct_type_field_find(a->st, name) : NULL;
2807 }
2808
2809 static struct field *
2810 action_field_parse(struct action *action, const char *name)
2811 {
2812         if (name[0] != 't' || name[1] != '.')
2813                 return NULL;
2814
2815         return action_field_find(action, &name[2]);
2816 }
2817
2818 int
2819 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
2820                                const char *name,
2821                                const char *args_struct_type_name,
2822                                const char **instructions,
2823                                uint32_t n_instructions)
2824 {
2825         struct struct_type *args_struct_type;
2826         struct action *a;
2827         int err;
2828
2829         CHECK(p, EINVAL);
2830
2831         CHECK_NAME(name, EINVAL);
2832         CHECK(!action_find(p, name), EEXIST);
2833
2834         if (args_struct_type_name) {
2835                 CHECK_NAME(args_struct_type_name, EINVAL);
2836                 args_struct_type = struct_type_find(p, args_struct_type_name);
2837                 CHECK(args_struct_type, EINVAL);
2838         } else {
2839                 args_struct_type = NULL;
2840         }
2841
2842         /* Node allocation. */
2843         a = calloc(1, sizeof(struct action));
2844         CHECK(a, ENOMEM);
2845
2846         /* Node initialization. */
2847         strcpy(a->name, name);
2848         a->st = args_struct_type;
2849         a->id = p->n_actions;
2850
2851         /* Instruction translation. */
2852         err = instruction_config(p, a, instructions, n_instructions);
2853         if (err) {
2854                 free(a);
2855                 return err;
2856         }
2857
2858         /* Node add to tailq. */
2859         TAILQ_INSERT_TAIL(&p->actions, a, node);
2860         p->n_actions++;
2861
2862         return 0;
2863 }
2864
2865 static int
2866 action_build(struct rte_swx_pipeline *p)
2867 {
2868         struct action *action;
2869
2870         p->action_instructions = calloc(p->n_actions,
2871                                         sizeof(struct instruction *));
2872         CHECK(p->action_instructions, ENOMEM);
2873
2874         TAILQ_FOREACH(action, &p->actions, node)
2875                 p->action_instructions[action->id] = action->instructions;
2876
2877         return 0;
2878 }
2879
2880 static void
2881 action_build_free(struct rte_swx_pipeline *p)
2882 {
2883         free(p->action_instructions);
2884         p->action_instructions = NULL;
2885 }
2886
2887 static void
2888 action_free(struct rte_swx_pipeline *p)
2889 {
2890         action_build_free(p);
2891
2892         for ( ; ; ) {
2893                 struct action *action;
2894
2895                 action = TAILQ_FIRST(&p->actions);
2896                 if (!action)
2897                         break;
2898
2899                 TAILQ_REMOVE(&p->actions, action, node);
2900                 free(action->instructions);
2901                 free(action);
2902         }
2903 }
2904
2905 /*
2906  * Table.
2907  */
2908 static struct table_type *
2909 table_type_find(struct rte_swx_pipeline *p, const char *name)
2910 {
2911         struct table_type *elem;
2912
2913         TAILQ_FOREACH(elem, &p->table_types, node)
2914                 if (strcmp(elem->name, name) == 0)
2915                         return elem;
2916
2917         return NULL;
2918 }
2919
2920 static struct table_type *
2921 table_type_resolve(struct rte_swx_pipeline *p,
2922                    const char *recommended_type_name,
2923                    enum rte_swx_table_match_type match_type)
2924 {
2925         struct table_type *elem;
2926
2927         /* Only consider the recommended type if the match type is correct. */
2928         if (recommended_type_name)
2929                 TAILQ_FOREACH(elem, &p->table_types, node)
2930                         if (!strcmp(elem->name, recommended_type_name) &&
2931                             (elem->match_type == match_type))
2932                                 return elem;
2933
2934         /* Ignore the recommended type and get the first element with this match
2935          * type.
2936          */
2937         TAILQ_FOREACH(elem, &p->table_types, node)
2938                 if (elem->match_type == match_type)
2939                         return elem;
2940
2941         return NULL;
2942 }
2943
2944 static struct table *
2945 table_find(struct rte_swx_pipeline *p, const char *name)
2946 {
2947         struct table *elem;
2948
2949         TAILQ_FOREACH(elem, &p->tables, node)
2950                 if (strcmp(elem->name, name) == 0)
2951                         return elem;
2952
2953         return NULL;
2954 }
2955
2956 static struct table *
2957 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
2958 {
2959         struct table *table = NULL;
2960
2961         TAILQ_FOREACH(table, &p->tables, node)
2962                 if (table->id == id)
2963                         return table;
2964
2965         return NULL;
2966 }
2967
2968 int
2969 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
2970                                      const char *name,
2971                                      enum rte_swx_table_match_type match_type,
2972                                      struct rte_swx_table_ops *ops)
2973 {
2974         struct table_type *elem;
2975
2976         CHECK(p, EINVAL);
2977
2978         CHECK_NAME(name, EINVAL);
2979         CHECK(!table_type_find(p, name), EEXIST);
2980
2981         CHECK(ops, EINVAL);
2982         CHECK(ops->create, EINVAL);
2983         CHECK(ops->lkp, EINVAL);
2984         CHECK(ops->free, EINVAL);
2985
2986         /* Node allocation. */
2987         elem = calloc(1, sizeof(struct table_type));
2988         CHECK(elem, ENOMEM);
2989
2990         /* Node initialization. */
2991         strcpy(elem->name, name);
2992         elem->match_type = match_type;
2993         memcpy(&elem->ops, ops, sizeof(*ops));
2994
2995         /* Node add to tailq. */
2996         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
2997
2998         return 0;
2999 }
3000
3001 static enum rte_swx_table_match_type
3002 table_match_type_resolve(struct rte_swx_match_field_params *fields,
3003                          uint32_t n_fields)
3004 {
3005         uint32_t i;
3006
3007         for (i = 0; i < n_fields; i++)
3008                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
3009                         break;
3010
3011         if (i == n_fields)
3012                 return RTE_SWX_TABLE_MATCH_EXACT;
3013
3014         if ((i == n_fields - 1) &&
3015             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
3016                 return RTE_SWX_TABLE_MATCH_LPM;
3017
3018         return RTE_SWX_TABLE_MATCH_WILDCARD;
3019 }
3020
3021 int
3022 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
3023                               const char *name,
3024                               struct rte_swx_pipeline_table_params *params,
3025                               const char *recommended_table_type_name,
3026                               const char *args,
3027                               uint32_t size)
3028 {
3029         struct table_type *type;
3030         struct table *t;
3031         struct action *default_action;
3032         struct header *header = NULL;
3033         int is_header = 0;
3034         uint32_t offset_prev = 0, action_data_size_max = 0, i;
3035
3036         CHECK(p, EINVAL);
3037
3038         CHECK_NAME(name, EINVAL);
3039         CHECK(!table_find(p, name), EEXIST);
3040
3041         CHECK(params, EINVAL);
3042
3043         /* Match checks. */
3044         CHECK(!params->n_fields || params->fields, EINVAL);
3045         for (i = 0; i < params->n_fields; i++) {
3046                 struct rte_swx_match_field_params *field = &params->fields[i];
3047                 struct header *h;
3048                 struct field *hf, *mf;
3049                 uint32_t offset;
3050
3051                 CHECK_NAME(field->name, EINVAL);
3052
3053                 hf = header_field_parse(p, field->name, &h);
3054                 mf = metadata_field_parse(p, field->name);
3055                 CHECK(hf || mf, EINVAL);
3056
3057                 offset = hf ? hf->offset : mf->offset;
3058
3059                 if (i == 0) {
3060                         is_header = hf ? 1 : 0;
3061                         header = hf ? h : NULL;
3062                         offset_prev = offset;
3063
3064                         continue;
3065                 }
3066
3067                 CHECK((is_header && hf && (h->id == header->id)) ||
3068                       (!is_header && mf), EINVAL);
3069
3070                 CHECK(offset > offset_prev, EINVAL);
3071                 offset_prev = offset;
3072         }
3073
3074         /* Action checks. */
3075         CHECK(params->n_actions, EINVAL);
3076         CHECK(params->action_names, EINVAL);
3077         for (i = 0; i < params->n_actions; i++) {
3078                 const char *action_name = params->action_names[i];
3079                 struct action *a;
3080                 uint32_t action_data_size;
3081
3082                 CHECK(action_name, EINVAL);
3083
3084                 a = action_find(p, action_name);
3085                 CHECK(a, EINVAL);
3086
3087                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
3088                 if (action_data_size > action_data_size_max)
3089                         action_data_size_max = action_data_size;
3090         }
3091
3092         CHECK(params->default_action_name, EINVAL);
3093         for (i = 0; i < p->n_actions; i++)
3094                 if (!strcmp(params->action_names[i],
3095                             params->default_action_name))
3096                         break;
3097         CHECK(i < params->n_actions, EINVAL);
3098         default_action = action_find(p, params->default_action_name);
3099         CHECK((default_action->st && params->default_action_data) ||
3100               !params->default_action_data, EINVAL);
3101
3102         /* Table type checks. */
3103         if (params->n_fields) {
3104                 enum rte_swx_table_match_type match_type;
3105
3106                 match_type = table_match_type_resolve(params->fields,
3107                                                       params->n_fields);
3108                 type = table_type_resolve(p,
3109                                           recommended_table_type_name,
3110                                           match_type);
3111                 CHECK(type, EINVAL);
3112         } else {
3113                 type = NULL;
3114         }
3115
3116         /* Memory allocation. */
3117         t = calloc(1, sizeof(struct table));
3118         CHECK(t, ENOMEM);
3119
3120         t->fields = calloc(params->n_fields, sizeof(struct match_field));
3121         if (!t->fields) {
3122                 free(t);
3123                 CHECK(0, ENOMEM);
3124         }
3125
3126         t->actions = calloc(params->n_actions, sizeof(struct action *));
3127         if (!t->actions) {
3128                 free(t->fields);
3129                 free(t);
3130                 CHECK(0, ENOMEM);
3131         }
3132
3133         if (action_data_size_max) {
3134                 t->default_action_data = calloc(1, action_data_size_max);
3135                 if (!t->default_action_data) {
3136                         free(t->actions);
3137                         free(t->fields);
3138                         free(t);
3139                         CHECK(0, ENOMEM);
3140                 }
3141         }
3142
3143         /* Node initialization. */
3144         strcpy(t->name, name);
3145         if (args && args[0])
3146                 strcpy(t->args, args);
3147         t->type = type;
3148
3149         for (i = 0; i < params->n_fields; i++) {
3150                 struct rte_swx_match_field_params *field = &params->fields[i];
3151                 struct match_field *f = &t->fields[i];
3152
3153                 f->match_type = field->match_type;
3154                 f->field = is_header ?
3155                         header_field_parse(p, field->name, NULL) :
3156                         metadata_field_parse(p, field->name);
3157         }
3158         t->n_fields = params->n_fields;
3159         t->is_header = is_header;
3160         t->header = header;
3161
3162         for (i = 0; i < params->n_actions; i++)
3163                 t->actions[i] = action_find(p, params->action_names[i]);
3164         t->default_action = default_action;
3165         if (default_action->st)
3166                 memcpy(t->default_action_data,
3167                        params->default_action_data,
3168                        default_action->st->n_bits / 8);
3169         t->n_actions = params->n_actions;
3170         t->default_action_is_const = params->default_action_is_const;
3171         t->action_data_size_max = action_data_size_max;
3172
3173         t->size = size;
3174         t->id = p->n_tables;
3175
3176         /* Node add to tailq. */
3177         TAILQ_INSERT_TAIL(&p->tables, t, node);
3178         p->n_tables++;
3179
3180         return 0;
3181 }
3182
3183 static struct rte_swx_table_params *
3184 table_params_get(struct table *table)
3185 {
3186         struct rte_swx_table_params *params;
3187         struct field *first, *last;
3188         uint8_t *key_mask;
3189         uint32_t key_size, key_offset, action_data_size, i;
3190
3191         /* Memory allocation. */
3192         params = calloc(1, sizeof(struct rte_swx_table_params));
3193         if (!params)
3194                 return NULL;
3195
3196         /* Key offset and size. */
3197         first = table->fields[0].field;
3198         last = table->fields[table->n_fields - 1].field;
3199         key_offset = first->offset / 8;
3200         key_size = (last->offset + last->n_bits - first->offset) / 8;
3201
3202         /* Memory allocation. */
3203         key_mask = calloc(1, key_size);
3204         if (!key_mask) {
3205                 free(params);
3206                 return NULL;
3207         }
3208
3209         /* Key mask. */
3210         for (i = 0; i < table->n_fields; i++) {
3211                 struct field *f = table->fields[i].field;
3212                 uint32_t start = (f->offset - first->offset) / 8;
3213                 size_t size = f->n_bits / 8;
3214
3215                 memset(&key_mask[start], 0xFF, size);
3216         }
3217
3218         /* Action data size. */
3219         action_data_size = 0;
3220         for (i = 0; i < table->n_actions; i++) {
3221                 struct action *action = table->actions[i];
3222                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
3223
3224                 if (ads > action_data_size)
3225                         action_data_size = ads;
3226         }
3227
3228         /* Fill in. */
3229         params->match_type = table->type->match_type;
3230         params->key_size = key_size;
3231         params->key_offset = key_offset;
3232         params->key_mask0 = key_mask;
3233         params->action_data_size = action_data_size;
3234         params->n_keys_max = table->size;
3235
3236         return params;
3237 }
3238
3239 static void
3240 table_params_free(struct rte_swx_table_params *params)
3241 {
3242         if (!params)
3243                 return;
3244
3245         free(params->key_mask0);
3246         free(params);
3247 }
3248
3249 static int
3250 table_state_build(struct rte_swx_pipeline *p)
3251 {
3252         struct table *table;
3253
3254         p->table_state = calloc(p->n_tables,
3255                                 sizeof(struct rte_swx_table_state));
3256         CHECK(p->table_state, ENOMEM);
3257
3258         TAILQ_FOREACH(table, &p->tables, node) {
3259                 struct rte_swx_table_state *ts = &p->table_state[table->id];
3260
3261                 if (table->type) {
3262                         struct rte_swx_table_params *params;
3263
3264                         /* ts->obj. */
3265                         params = table_params_get(table);
3266                         CHECK(params, ENOMEM);
3267
3268                         ts->obj = table->type->ops.create(params,
3269                                 NULL,
3270                                 table->args,
3271                                 p->numa_node);
3272
3273                         table_params_free(params);
3274                         CHECK(ts->obj, ENODEV);
3275                 }
3276
3277                 /* ts->default_action_data. */
3278                 if (table->action_data_size_max) {
3279                         ts->default_action_data =
3280                                 malloc(table->action_data_size_max);
3281                         CHECK(ts->default_action_data, ENOMEM);
3282
3283                         memcpy(ts->default_action_data,
3284                                table->default_action_data,
3285                                table->action_data_size_max);
3286                 }
3287
3288                 /* ts->default_action_id. */
3289                 ts->default_action_id = table->default_action->id;
3290         }
3291
3292         return 0;
3293 }
3294
3295 static void
3296 table_state_build_free(struct rte_swx_pipeline *p)
3297 {
3298         uint32_t i;
3299
3300         if (!p->table_state)
3301                 return;
3302
3303         for (i = 0; i < p->n_tables; i++) {
3304                 struct rte_swx_table_state *ts = &p->table_state[i];
3305                 struct table *table = table_find_by_id(p, i);
3306
3307                 /* ts->obj. */
3308                 if (table->type && ts->obj)
3309                         table->type->ops.free(ts->obj);
3310
3311                 /* ts->default_action_data. */
3312                 free(ts->default_action_data);
3313         }
3314
3315         free(p->table_state);
3316         p->table_state = NULL;
3317 }
3318
3319 static void
3320 table_state_free(struct rte_swx_pipeline *p)
3321 {
3322         table_state_build_free(p);
3323 }
3324
3325 static int
3326 table_stub_lkp(void *table __rte_unused,
3327                void *mailbox __rte_unused,
3328                uint8_t **key __rte_unused,
3329                uint64_t *action_id __rte_unused,
3330                uint8_t **action_data __rte_unused,
3331                int *hit)
3332 {
3333         *hit = 0;
3334         return 1; /* DONE. */
3335 }
3336
3337 static int
3338 table_build(struct rte_swx_pipeline *p)
3339 {
3340         uint32_t i;
3341
3342         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3343                 struct thread *t = &p->threads[i];
3344                 struct table *table;
3345
3346                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
3347                 CHECK(t->tables, ENOMEM);
3348
3349                 TAILQ_FOREACH(table, &p->tables, node) {
3350                         struct table_runtime *r = &t->tables[table->id];
3351
3352                         if (table->type) {
3353                                 uint64_t size;
3354
3355                                 size = table->type->ops.mailbox_size_get();
3356
3357                                 /* r->func. */
3358                                 r->func = table->type->ops.lkp;
3359
3360                                 /* r->mailbox. */
3361                                 if (size) {
3362                                         r->mailbox = calloc(1, size);
3363                                         CHECK(r->mailbox, ENOMEM);
3364                                 }
3365
3366                                 /* r->key. */
3367                                 r->key = table->is_header ?
3368                                         &t->structs[table->header->struct_id] :
3369                                         &t->structs[p->metadata_struct_id];
3370                         } else {
3371                                 r->func = table_stub_lkp;
3372                         }
3373                 }
3374         }
3375
3376         return 0;
3377 }
3378
3379 static void
3380 table_build_free(struct rte_swx_pipeline *p)
3381 {
3382         uint32_t i;
3383
3384         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3385                 struct thread *t = &p->threads[i];
3386                 uint32_t j;
3387
3388                 if (!t->tables)
3389                         continue;
3390
3391                 for (j = 0; j < p->n_tables; j++) {
3392                         struct table_runtime *r = &t->tables[j];
3393
3394                         free(r->mailbox);
3395                 }
3396
3397                 free(t->tables);
3398                 t->tables = NULL;
3399         }
3400 }
3401
3402 static void
3403 table_free(struct rte_swx_pipeline *p)
3404 {
3405         table_build_free(p);
3406
3407         /* Tables. */
3408         for ( ; ; ) {
3409                 struct table *elem;
3410
3411                 elem = TAILQ_FIRST(&p->tables);
3412                 if (!elem)
3413                         break;
3414
3415                 TAILQ_REMOVE(&p->tables, elem, node);
3416                 free(elem->fields);
3417                 free(elem->actions);
3418                 free(elem->default_action_data);
3419                 free(elem);
3420         }
3421
3422         /* Table types. */
3423         for ( ; ; ) {
3424                 struct table_type *elem;
3425
3426                 elem = TAILQ_FIRST(&p->table_types);
3427                 if (!elem)
3428                         break;
3429
3430                 TAILQ_REMOVE(&p->table_types, elem, node);
3431                 free(elem);
3432         }
3433 }
3434
3435 /*
3436  * Pipeline.
3437  */
3438 int
3439 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
3440 {
3441         struct rte_swx_pipeline *pipeline;
3442
3443         /* Check input parameters. */
3444         CHECK(p, EINVAL);
3445
3446         /* Memory allocation. */
3447         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
3448         CHECK(pipeline, ENOMEM);
3449
3450         /* Initialization. */
3451         TAILQ_INIT(&pipeline->struct_types);
3452         TAILQ_INIT(&pipeline->port_in_types);
3453         TAILQ_INIT(&pipeline->ports_in);
3454         TAILQ_INIT(&pipeline->port_out_types);
3455         TAILQ_INIT(&pipeline->ports_out);
3456         TAILQ_INIT(&pipeline->extern_types);
3457         TAILQ_INIT(&pipeline->extern_objs);
3458         TAILQ_INIT(&pipeline->extern_funcs);
3459         TAILQ_INIT(&pipeline->headers);
3460         TAILQ_INIT(&pipeline->actions);
3461         TAILQ_INIT(&pipeline->table_types);
3462         TAILQ_INIT(&pipeline->tables);
3463
3464         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
3465         pipeline->numa_node = numa_node;
3466
3467         *p = pipeline;
3468         return 0;
3469 }
3470
3471 void
3472 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
3473 {
3474         if (!p)
3475                 return;
3476
3477         free(p->instructions);
3478
3479         table_state_free(p);
3480         table_free(p);
3481         action_free(p);
3482         metadata_free(p);
3483         header_free(p);
3484         extern_func_free(p);
3485         extern_obj_free(p);
3486         port_out_free(p);
3487         port_in_free(p);
3488         struct_free(p);
3489
3490         free(p);
3491 }
3492
3493 int
3494 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
3495                                      const char **instructions,
3496                                      uint32_t n_instructions)
3497 {
3498         int err;
3499         uint32_t i;
3500
3501         err = instruction_config(p, NULL, instructions, n_instructions);
3502         if (err)
3503                 return err;
3504
3505         /* Thread instruction pointer reset. */
3506         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3507                 struct thread *t = &p->threads[i];
3508
3509                 thread_ip_reset(p, t);
3510         }
3511
3512         return 0;
3513 }
3514
3515 int
3516 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
3517 {
3518         int status;
3519
3520         CHECK(p, EINVAL);
3521         CHECK(p->build_done == 0, EEXIST);
3522
3523         status = port_in_build(p);
3524         if (status)
3525                 goto error;
3526
3527         status = port_out_build(p);
3528         if (status)
3529                 goto error;
3530
3531         status = struct_build(p);
3532         if (status)
3533                 goto error;
3534
3535         status = extern_obj_build(p);
3536         if (status)
3537                 goto error;
3538
3539         status = extern_func_build(p);
3540         if (status)
3541                 goto error;
3542
3543         status = header_build(p);
3544         if (status)
3545                 goto error;
3546
3547         status = metadata_build(p);
3548         if (status)
3549                 goto error;
3550
3551         status = action_build(p);
3552         if (status)
3553                 goto error;
3554
3555         status = table_build(p);
3556         if (status)
3557                 goto error;
3558
3559         status = table_state_build(p);
3560         if (status)
3561                 goto error;
3562
3563         p->build_done = 1;
3564         return 0;
3565
3566 error:
3567         table_state_build_free(p);
3568         table_build_free(p);
3569         action_build_free(p);
3570         metadata_build_free(p);
3571         header_build_free(p);
3572         extern_func_build_free(p);
3573         extern_obj_build_free(p);
3574         port_out_build_free(p);
3575         port_in_build_free(p);
3576         struct_build_free(p);
3577
3578         return status;
3579 }
3580
3581 void
3582 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
3583 {
3584         uint32_t i;
3585
3586         for (i = 0; i < n_instructions; i++)
3587                 instr_exec(p);
3588 }
3589
3590 /*
3591  * Control.
3592  */
3593 int
3594 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
3595                                  struct rte_swx_table_state **table_state)
3596 {
3597         if (!p || !table_state || !p->build_done)
3598                 return -EINVAL;
3599
3600         *table_state = p->table_state;
3601         return 0;
3602 }
3603
3604 int
3605 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
3606                                  struct rte_swx_table_state *table_state)
3607 {
3608         if (!p || !table_state || !p->build_done)
3609                 return -EINVAL;
3610
3611         p->table_state = table_state;
3612         return 0;
3613 }