lib/librte_pipeline/rte_swx_pipeline.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <string.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <sys/queue.h>
   9 #include <arpa/inet.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "rte_swx_pipeline.h"
  16 #include "rte_swx_ctl.h"
  17
  18 #define CHECK(condition, err_code)                                             \
  19 do {                                                                           \
  20         if (!(condition))                                                      \
  21                 return -(err_code);                                            \
  22 } while (0)
  23
  24 #define CHECK_NAME(name, err_code)                                             \
  25         CHECK((name) && (name)[0], err_code)
  26
  27 #ifndef TRACE_LEVEL
  28 #define TRACE_LEVEL 0
  29 #endif
  30
  31 #if TRACE_LEVEL
  32 #define TRACE(...) printf(__VA_ARGS__)
  33 #else
  34 #define TRACE(...)
  35 #endif
  36
  37 #define ntoh64(x) rte_be_to_cpu_64(x)
  38 #define hton64(x) rte_cpu_to_be_64(x)
  39
  40 /*
  41  * Struct.
  42  */
  43 struct field {
  44         char name[RTE_SWX_NAME_SIZE];
  45         uint32_t n_bits;
  46         uint32_t offset;
  47 };
  48
  49 struct struct_type {
  50         TAILQ_ENTRY(struct_type) node;
  51         char name[RTE_SWX_NAME_SIZE];
  52         struct field *fields;
  53         uint32_t n_fields;
  54         uint32_t n_bits;
  55 };
  56
  57 TAILQ_HEAD(struct_type_tailq, struct_type);
  58
  59 /*
  60  * Input port.
  61  */
  62 struct port_in_type {
  63         TAILQ_ENTRY(port_in_type) node;
  64         char name[RTE_SWX_NAME_SIZE];
  65         struct rte_swx_port_in_ops ops;
  66 };
  67
  68 TAILQ_HEAD(port_in_type_tailq, port_in_type);
  69
  70 struct port_in {
  71         TAILQ_ENTRY(port_in) node;
  72         struct port_in_type *type;
  73         void *obj;
  74         uint32_t id;
  75 };
  76
  77 TAILQ_HEAD(port_in_tailq, port_in);
  78
  79 struct port_in_runtime {
  80         rte_swx_port_in_pkt_rx_t pkt_rx;
  81         void *obj;
  82 };
  83
  84 /*
  85  * Output port.
  86  */
  87 struct port_out_type {
  88         TAILQ_ENTRY(port_out_type) node;
  89         char name[RTE_SWX_NAME_SIZE];
  90         struct rte_swx_port_out_ops ops;
  91 };
  92
  93 TAILQ_HEAD(port_out_type_tailq, port_out_type);
  94
  95 struct port_out {
  96         TAILQ_ENTRY(port_out) node;
  97         struct port_out_type *type;
  98         void *obj;
  99         uint32_t id;
 100 };
 101
 102 TAILQ_HEAD(port_out_tailq, port_out);
 103
 104 struct port_out_runtime {
 105         rte_swx_port_out_pkt_tx_t pkt_tx;
 106         rte_swx_port_out_flush_t flush;
 107         void *obj;
 108 };
 109
 110 /*
 111  * Extern object.
 112  */
 113 struct extern_type_member_func {
 114         TAILQ_ENTRY(extern_type_member_func) node;
 115         char name[RTE_SWX_NAME_SIZE];
 116         rte_swx_extern_type_member_func_t func;
 117         uint32_t id;
 118 };
 119
 120 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
 121
 122 struct extern_type {
 123         TAILQ_ENTRY(extern_type) node;
 124         char name[RTE_SWX_NAME_SIZE];
 125         struct struct_type *mailbox_struct_type;
 126         rte_swx_extern_type_constructor_t constructor;
 127         rte_swx_extern_type_destructor_t destructor;
 128         struct extern_type_member_func_tailq funcs;
 129         uint32_t n_funcs;
 130 };
 131
 132 TAILQ_HEAD(extern_type_tailq, extern_type);
 133
 134 struct extern_obj {
 135         TAILQ_ENTRY(extern_obj) node;
 136         char name[RTE_SWX_NAME_SIZE];
 137         struct extern_type *type;
 138         void *obj;
 139         uint32_t struct_id;
 140         uint32_t id;
 141 };
 142
 143 TAILQ_HEAD(extern_obj_tailq, extern_obj);
 144
 145 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
 146 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
 147 #endif
 148
 149 struct extern_obj_runtime {
 150         void *obj;
 151         uint8_t *mailbox;
 152         rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
 153 };
 154
 155 /*
 156  * Extern function.
 157  */
 158 struct extern_func {
 159         TAILQ_ENTRY(extern_func) node;
 160         char name[RTE_SWX_NAME_SIZE];
 161         struct struct_type *mailbox_struct_type;
 162         rte_swx_extern_func_t func;
 163         uint32_t struct_id;
 164         uint32_t id;
 165 };
 166
 167 TAILQ_HEAD(extern_func_tailq, extern_func);
 168
 169 struct extern_func_runtime {
 170         uint8_t *mailbox;
 171         rte_swx_extern_func_t func;
 172 };
 173
 174 /*
 175  * Header.
 176  */
 177 struct header {
 178         TAILQ_ENTRY(header) node;
 179         char name[RTE_SWX_NAME_SIZE];
 180         struct struct_type *st;
 181         uint32_t struct_id;
 182         uint32_t id;
 183 };
 184
 185 TAILQ_HEAD(header_tailq, header);
 186
 187 struct header_runtime {
 188         uint8_t *ptr0;
 189 };
 190
 191 struct header_out_runtime {
 192         uint8_t *ptr0;
 193         uint8_t *ptr;
 194         uint32_t n_bytes;
 195 };
 196
 197 /*
 198  * Instruction.
 199  */
 200
 201 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 202  * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 203  * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 204  * when transferred to packet meta-data and in NBO when transferred to packet
 205  * headers.
 206  */
 207
 208 /* Notation conventions:
 209  *    -Header field: H = h.header.field (dst/src)
 210  *    -Meta-data field: M = m.field (dst/src)
 211  *    -Extern object mailbox field: E = e.field (dst/src)
 212  *    -Extern function mailbox field: F = f.field (dst/src)
 213  *    -Table action data field: T = t.field (src only)
 214  *    -Immediate value: I = 32-bit unsigned value (src only)
 215  */
 216
 217 enum instruction_type {
 218         /* rx m.port_in */
 219         INSTR_RX,
 220
 221         /* tx m.port_out */
 222         INSTR_TX,
 223
 224         /* extract h.header */
 225         INSTR_HDR_EXTRACT,
 226         INSTR_HDR_EXTRACT2,
 227         INSTR_HDR_EXTRACT3,
 228         INSTR_HDR_EXTRACT4,
 229         INSTR_HDR_EXTRACT5,
 230         INSTR_HDR_EXTRACT6,
 231         INSTR_HDR_EXTRACT7,
 232         INSTR_HDR_EXTRACT8,
 233
 234         /* emit h.header */
 235         INSTR_HDR_EMIT,
 236         INSTR_HDR_EMIT_TX,
 237         INSTR_HDR_EMIT2_TX,
 238         INSTR_HDR_EMIT3_TX,
 239         INSTR_HDR_EMIT4_TX,
 240         INSTR_HDR_EMIT5_TX,
 241         INSTR_HDR_EMIT6_TX,
 242         INSTR_HDR_EMIT7_TX,
 243         INSTR_HDR_EMIT8_TX,
 244
 245         /* validate h.header */
 246         INSTR_HDR_VALIDATE,
 247
 248         /* invalidate h.header */
 249         INSTR_HDR_INVALIDATE,
 250
 251         /* mov dst src
 252          * dst = src
 253          * dst = HMEF, src = HMEFTI
 254          */
 255         INSTR_MOV,   /* dst = MEF, src = MEFT */
 256         INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
 257         INSTR_MOV_I, /* dst = HMEF, src = I */
 258
 259         /* dma h.header t.field
 260          * memcpy(h.header, t.field, sizeof(h.header))
 261          */
 262         INSTR_DMA_HT,
 263         INSTR_DMA_HT2,
 264         INSTR_DMA_HT3,
 265         INSTR_DMA_HT4,
 266         INSTR_DMA_HT5,
 267         INSTR_DMA_HT6,
 268         INSTR_DMA_HT7,
 269         INSTR_DMA_HT8,
 270
 271         /* add dst src
 272          * dst += src
 273          * dst = HMEF, src = HMEFTI
 274          */
 275         INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
 276         INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
 277         INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
 278         INSTR_ALU_ADD_HH, /* dst = H, src = H */
 279         INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
 280         INSTR_ALU_ADD_HI, /* dst = H, src = I */
 281 };
 282
 283 struct instr_operand {
 284         uint8_t struct_id;
 285         uint8_t n_bits;
 286         uint8_t offset;
 287         uint8_t pad;
 288 };
 289
 290 struct instr_io {
 291         struct {
 292                 uint8_t offset;
 293                 uint8_t n_bits;
 294                 uint8_t pad[2];
 295         } io;
 296
 297         struct {
 298                 uint8_t header_id[8];
 299                 uint8_t struct_id[8];
 300                 uint8_t n_bytes[8];
 301         } hdr;
 302 };
 303
 304 struct instr_hdr_validity {
 305         uint8_t header_id;
 306 };
 307
 308 struct instr_dst_src {
 309         struct instr_operand dst;
 310         union {
 311                 struct instr_operand src;
 312                 uint32_t src_val;
 313         };
 314 };
 315
 316 struct instr_dma {
 317         struct {
 318                 uint8_t header_id[8];
 319                 uint8_t struct_id[8];
 320         } dst;
 321
 322         struct {
 323                 uint8_t offset[8];
 324         } src;
 325
 326         uint16_t n_bytes[8];
 327 };
 328
 329 struct instruction {
 330         enum instruction_type type;
 331         union {
 332                 struct instr_io io;
 333                 struct instr_hdr_validity valid;
 334                 struct instr_dst_src mov;
 335                 struct instr_dma dma;
 336                 struct instr_dst_src alu;
 337         };
 338 };
 339
 340 struct instruction_data {
 341         char label[RTE_SWX_NAME_SIZE];
 342         char jmp_label[RTE_SWX_NAME_SIZE];
 343         uint32_t n_users; /* user = jmp instruction to this instruction. */
 344         int invalid;
 345 };
 346
 347 /*
 348  * Action.
 349  */
 350 struct action {
 351         TAILQ_ENTRY(action) node;
 352         char name[RTE_SWX_NAME_SIZE];
 353         struct struct_type *st;
 354         struct instruction *instructions;
 355         uint32_t n_instructions;
 356         uint32_t id;
 357 };
 358
 359 TAILQ_HEAD(action_tailq, action);
 360
 361 /*
 362  * Table.
 363  */
 364 struct table_type {
 365         TAILQ_ENTRY(table_type) node;
 366         char name[RTE_SWX_NAME_SIZE];
 367         enum rte_swx_table_match_type match_type;
 368         struct rte_swx_table_ops ops;
 369 };
 370
 371 TAILQ_HEAD(table_type_tailq, table_type);
 372
 373 struct match_field {
 374         enum rte_swx_table_match_type match_type;
 375         struct field *field;
 376 };
 377
 378 struct table {
 379         TAILQ_ENTRY(table) node;
 380         char name[RTE_SWX_NAME_SIZE];
 381         char args[RTE_SWX_NAME_SIZE];
 382         struct table_type *type; /* NULL when n_fields == 0. */
 383
 384         /* Match. */
 385         struct match_field *fields;
 386         uint32_t n_fields;
 387         int is_header; /* Only valid when n_fields > 0. */
 388         struct header *header; /* Only valid when n_fields > 0. */
 389
 390         /* Action. */
 391         struct action **actions;
 392         struct action *default_action;
 393         uint8_t *default_action_data;
 394         uint32_t n_actions;
 395         int default_action_is_const;
 396         uint32_t action_data_size_max;
 397
 398         uint32_t size;
 399         uint32_t id;
 400 };
 401
 402 TAILQ_HEAD(table_tailq, table);
 403
 404 struct table_runtime {
 405         rte_swx_table_lookup_t func;
 406         void *mailbox;
 407         uint8_t **key;
 408 };
 409
 410 /*
 411  * Pipeline.
 412  */
 413 struct thread {
 414         /* Packet. */
 415         struct rte_swx_pkt pkt;
 416         uint8_t *ptr;
 417
 418         /* Structures. */
 419         uint8_t **structs;
 420
 421         /* Packet headers. */
 422         struct header_runtime *headers; /* Extracted or generated headers. */
 423         struct header_out_runtime *headers_out; /* Emitted headers. */
 424         uint8_t *header_storage;
 425         uint8_t *header_out_storage;
 426         uint64_t valid_headers;
 427         uint32_t n_headers_out;
 428
 429         /* Packet meta-data. */
 430         uint8_t *metadata;
 431
 432         /* Tables. */
 433         struct table_runtime *tables;
 434         struct rte_swx_table_state *table_state;
 435         uint64_t action_id;
 436         int hit; /* 0 = Miss, 1 = Hit. */
 437
 438         /* Extern objects and functions. */
 439         struct extern_obj_runtime *extern_objs;
 440         struct extern_func_runtime *extern_funcs;
 441
 442         /* Instructions. */
 443         struct instruction *ip;
 444         struct instruction *ret;
 445 };
 446
 447 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
 448 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
 449 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
 450
 451 #define ALU(thread, ip, operator)  \
 452 {                                                                              \
 453         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 454         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 455         uint64_t dst64 = *dst64_ptr;                                           \
 456         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 457         uint64_t dst = dst64 & dst64_mask;                                     \
 458                                                                                \
 459         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 460         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 461         uint64_t src64 = *src64_ptr;                                           \
 462         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 463         uint64_t src = src64 & src64_mask;                                     \
 464                                                                                \
 465         uint64_t result = dst operator src;                                    \
 466                                                                                \
 467         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 468 }
 469
 470 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 471
 472 #define ALU_S(thread, ip, operator)  \
 473 {                                                                              \
 474         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 475         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 476         uint64_t dst64 = *dst64_ptr;                                           \
 477         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 478         uint64_t dst = dst64 & dst64_mask;                                     \
 479                                                                                \
 480         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 481         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 482         uint64_t src64 = *src64_ptr;                                           \
 483         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 484                                                                                \
 485         uint64_t result = dst operator src;                                    \
 486                                                                                \
 487         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 488 }
 489
 490 #define ALU_MH ALU_S
 491
 492 #define ALU_HM(thread, ip, operator)  \
 493 {                                                                              \
 494         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 495         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 496         uint64_t dst64 = *dst64_ptr;                                           \
 497         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 498         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 499                                                                                \
 500         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 501         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 502         uint64_t src64 = *src64_ptr;                                           \
 503         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
 504         uint64_t src = src64 & src64_mask;                                     \
 505                                                                                \
 506         uint64_t result = dst operator src;                                    \
 507         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 508                                                                                \
 509         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 510 }
 511
 512 #define ALU_HH(thread, ip, operator)  \
 513 {                                                                              \
 514         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 515         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 516         uint64_t dst64 = *dst64_ptr;                                           \
 517         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 518         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 519                                                                                \
 520         uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
 521         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
 522         uint64_t src64 = *src64_ptr;                                           \
 523         uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
 524                                                                                \
 525         uint64_t result = dst operator src;                                    \
 526         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 527                                                                                \
 528         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 529 }
 530
 531 #else
 532
 533 #define ALU_S ALU
 534 #define ALU_MH ALU
 535 #define ALU_HM ALU
 536 #define ALU_HH ALU
 537
 538 #endif
 539
 540 #define ALU_I(thread, ip, operator)  \
 541 {                                                                              \
 542         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 543         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 544         uint64_t dst64 = *dst64_ptr;                                           \
 545         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 546         uint64_t dst = dst64 & dst64_mask;                                     \
 547                                                                                \
 548         uint64_t src = (ip)->alu.src_val;                                      \
 549                                                                                \
 550         uint64_t result = dst operator src;                                    \
 551                                                                                \
 552         *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
 553 }
 554
 555 #define ALU_MI ALU_I
 556
 557 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 558
 559 #define ALU_HI(thread, ip, operator)  \
 560 {                                                                              \
 561         uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
 562         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
 563         uint64_t dst64 = *dst64_ptr;                                           \
 564         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
 565         uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
 566                                                                                \
 567         uint64_t src = (ip)->alu.src_val;                                      \
 568                                                                                \
 569         uint64_t result = dst operator src;                                    \
 570         result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
 571                                                                                \
 572         *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
 573 }
 574
 575 #else
 576
 577 #define ALU_HI ALU_I
 578
 579 #endif
 580
 581 #define MOV(thread, ip)  \
 582 {                                                                              \
 583         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 584         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 585         uint64_t dst64 = *dst64_ptr;                                           \
 586         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 587                                                                                \
 588         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 589         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 590         uint64_t src64 = *src64_ptr;                                           \
 591         uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
 592         uint64_t src = src64 & src64_mask;                                     \
 593                                                                                \
 594         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 595 }
 596
 597 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 598
 599 #define MOV_S(thread, ip)  \
 600 {                                                                              \
 601         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 602         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 603         uint64_t dst64 = *dst64_ptr;                                           \
 604         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 605                                                                                \
 606         uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
 607         uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
 608         uint64_t src64 = *src64_ptr;                                           \
 609         uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
 610                                                                                \
 611         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 612 }
 613
 614 #else
 615
 616 #define MOV_S MOV
 617
 618 #endif
 619
 620 #define MOV_I(thread, ip)  \
 621 {                                                                              \
 622         uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
 623         uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
 624         uint64_t dst64 = *dst64_ptr;                                           \
 625         uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
 626                                                                                \
 627         uint64_t src = (ip)->mov.src_val;                                      \
 628                                                                                \
 629         *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
 630 }
 631
 632 #define METADATA_READ(thread, offset, n_bits)                                  \
 633 ({                                                                             \
 634         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 635         uint64_t m64 = *m64_ptr;                                               \
 636         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 637         (m64 & m64_mask);                                                      \
 638 })
 639
 640 #define METADATA_WRITE(thread, offset, n_bits, value)                          \
 641 {                                                                              \
 642         uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
 643         uint64_t m64 = *m64_ptr;                                               \
 644         uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
 645                                                                                \
 646         uint64_t m_new = value;                                                \
 647                                                                                \
 648         *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
 649 }
 650
 651 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
 652 #define RTE_SWX_PIPELINE_THREADS_MAX 16
 653 #endif
 654
 655 struct rte_swx_pipeline {
 656         struct struct_type_tailq struct_types;
 657         struct port_in_type_tailq port_in_types;
 658         struct port_in_tailq ports_in;
 659         struct port_out_type_tailq port_out_types;
 660         struct port_out_tailq ports_out;
 661         struct extern_type_tailq extern_types;
 662         struct extern_obj_tailq extern_objs;
 663         struct extern_func_tailq extern_funcs;
 664         struct header_tailq headers;
 665         struct struct_type *metadata_st;
 666         uint32_t metadata_struct_id;
 667         struct action_tailq actions;
 668         struct table_type_tailq table_types;
 669         struct table_tailq tables;
 670
 671         struct port_in_runtime *in;
 672         struct port_out_runtime *out;
 673         struct instruction **action_instructions;
 674         struct rte_swx_table_state *table_state;
 675         struct instruction *instructions;
 676         struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
 677
 678         uint32_t n_structs;
 679         uint32_t n_ports_in;
 680         uint32_t n_ports_out;
 681         uint32_t n_extern_objs;
 682         uint32_t n_extern_funcs;
 683         uint32_t n_actions;
 684         uint32_t n_tables;
 685         uint32_t n_headers;
 686         uint32_t thread_id;
 687         uint32_t port_id;
 688         uint32_t n_instructions;
 689         int build_done;
 690         int numa_node;
 691 };
 692
 693 /*
 694  * Struct.
 695  */
 696 static struct struct_type *
 697 struct_type_find(struct rte_swx_pipeline *p, const char *name)
 698 {
 699         struct struct_type *elem;
 700
 701         TAILQ_FOREACH(elem, &p->struct_types, node)
 702                 if (strcmp(elem->name, name) == 0)
 703                         return elem;
 704
 705         return NULL;
 706 }
 707
 708 static struct field *
 709 struct_type_field_find(struct struct_type *st, const char *name)
 710 {
 711         uint32_t i;
 712
 713         for (i = 0; i < st->n_fields; i++) {
 714                 struct field *f = &st->fields[i];
 715
 716                 if (strcmp(f->name, name) == 0)
 717                         return f;
 718         }
 719
 720         return NULL;
 721 }
 722
 723 int
 724 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
 725                                       const char *name,
 726                                       struct rte_swx_field_params *fields,
 727                                       uint32_t n_fields)
 728 {
 729         struct struct_type *st;
 730         uint32_t i;
 731
 732         CHECK(p, EINVAL);
 733         CHECK_NAME(name, EINVAL);
 734         CHECK(fields, EINVAL);
 735         CHECK(n_fields, EINVAL);
 736
 737         for (i = 0; i < n_fields; i++) {
 738                 struct rte_swx_field_params *f = &fields[i];
 739                 uint32_t j;
 740
 741                 CHECK_NAME(f->name, EINVAL);
 742                 CHECK(f->n_bits, EINVAL);
 743                 CHECK(f->n_bits <= 64, EINVAL);
 744                 CHECK((f->n_bits & 7) == 0, EINVAL);
 745
 746                 for (j = 0; j < i; j++) {
 747                         struct rte_swx_field_params *f_prev = &fields[j];
 748
 749                         CHECK(strcmp(f->name, f_prev->name), EINVAL);
 750                 }
 751         }
 752
 753         CHECK(!struct_type_find(p, name), EEXIST);
 754
 755         /* Node allocation. */
 756         st = calloc(1, sizeof(struct struct_type));
 757         CHECK(st, ENOMEM);
 758
 759         st->fields = calloc(n_fields, sizeof(struct field));
 760         if (!st->fields) {
 761                 free(st);
 762                 CHECK(0, ENOMEM);
 763         }
 764
 765         /* Node initialization. */
 766         strcpy(st->name, name);
 767         for (i = 0; i < n_fields; i++) {
 768                 struct field *dst = &st->fields[i];
 769                 struct rte_swx_field_params *src = &fields[i];
 770
 771                 strcpy(dst->name, src->name);
 772                 dst->n_bits = src->n_bits;
 773                 dst->offset = st->n_bits;
 774
 775                 st->n_bits += src->n_bits;
 776         }
 777         st->n_fields = n_fields;
 778
 779         /* Node add to tailq. */
 780         TAILQ_INSERT_TAIL(&p->struct_types, st, node);
 781
 782         return 0;
 783 }
 784
 785 static int
 786 struct_build(struct rte_swx_pipeline *p)
 787 {
 788         uint32_t i;
 789
 790         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 791                 struct thread *t = &p->threads[i];
 792
 793                 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
 794                 CHECK(t->structs, ENOMEM);
 795         }
 796
 797         return 0;
 798 }
 799
 800 static void
 801 struct_build_free(struct rte_swx_pipeline *p)
 802 {
 803         uint32_t i;
 804
 805         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
 806                 struct thread *t = &p->threads[i];
 807
 808                 free(t->structs);
 809                 t->structs = NULL;
 810         }
 811 }
 812
 813 static void
 814 struct_free(struct rte_swx_pipeline *p)
 815 {
 816         struct_build_free(p);
 817
 818         /* Struct types. */
 819         for ( ; ; ) {
 820                 struct struct_type *elem;
 821
 822                 elem = TAILQ_FIRST(&p->struct_types);
 823                 if (!elem)
 824                         break;
 825
 826                 TAILQ_REMOVE(&p->struct_types, elem, node);
 827                 free(elem->fields);
 828                 free(elem);
 829         }
 830 }
 831
 832 /*
 833  * Input port.
 834  */
 835 static struct port_in_type *
 836 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
 837 {
 838         struct port_in_type *elem;
 839
 840         if (!name)
 841                 return NULL;
 842
 843         TAILQ_FOREACH(elem, &p->port_in_types, node)
 844                 if (strcmp(elem->name, name) == 0)
 845                         return elem;
 846
 847         return NULL;
 848 }
 849
 850 int
 851 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
 852                                        const char *name,
 853                                        struct rte_swx_port_in_ops *ops)
 854 {
 855         struct port_in_type *elem;
 856
 857         CHECK(p, EINVAL);
 858         CHECK_NAME(name, EINVAL);
 859         CHECK(ops, EINVAL);
 860         CHECK(ops->create, EINVAL);
 861         CHECK(ops->free, EINVAL);
 862         CHECK(ops->pkt_rx, EINVAL);
 863         CHECK(ops->stats_read, EINVAL);
 864
 865         CHECK(!port_in_type_find(p, name), EEXIST);
 866
 867         /* Node allocation. */
 868         elem = calloc(1, sizeof(struct port_in_type));
 869         CHECK(elem, ENOMEM);
 870
 871         /* Node initialization. */
 872         strcpy(elem->name, name);
 873         memcpy(&elem->ops, ops, sizeof(*ops));
 874
 875         /* Node add to tailq. */
 876         TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
 877
 878         return 0;
 879 }
 880
 881 static struct port_in *
 882 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
 883 {
 884         struct port_in *port;
 885
 886         TAILQ_FOREACH(port, &p->ports_in, node)
 887                 if (port->id == port_id)
 888                         return port;
 889
 890         return NULL;
 891 }
 892
 893 int
 894 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
 895                                 uint32_t port_id,
 896                                 const char *port_type_name,
 897                                 void *args)
 898 {
 899         struct port_in_type *type = NULL;
 900         struct port_in *port = NULL;
 901         void *obj = NULL;
 902
 903         CHECK(p, EINVAL);
 904
 905         CHECK(!port_in_find(p, port_id), EINVAL);
 906
 907         CHECK_NAME(port_type_name, EINVAL);
 908         type = port_in_type_find(p, port_type_name);
 909         CHECK(type, EINVAL);
 910
 911         obj = type->ops.create(args);
 912         CHECK(obj, ENODEV);
 913
 914         /* Node allocation. */
 915         port = calloc(1, sizeof(struct port_in));
 916         CHECK(port, ENOMEM);
 917
 918         /* Node initialization. */
 919         port->type = type;
 920         port->obj = obj;
 921         port->id = port_id;
 922
 923         /* Node add to tailq. */
 924         TAILQ_INSERT_TAIL(&p->ports_in, port, node);
 925         if (p->n_ports_in < port_id + 1)
 926                 p->n_ports_in = port_id + 1;
 927
 928         return 0;
 929 }
 930
 931 static int
 932 port_in_build(struct rte_swx_pipeline *p)
 933 {
 934         struct port_in *port;
 935         uint32_t i;
 936
 937         CHECK(p->n_ports_in, EINVAL);
 938         CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
 939
 940         for (i = 0; i < p->n_ports_in; i++)
 941                 CHECK(port_in_find(p, i), EINVAL);
 942
 943         p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
 944         CHECK(p->in, ENOMEM);
 945
 946         TAILQ_FOREACH(port, &p->ports_in, node) {
 947                 struct port_in_runtime *in = &p->in[port->id];
 948
 949                 in->pkt_rx = port->type->ops.pkt_rx;
 950                 in->obj = port->obj;
 951         }
 952
 953         return 0;
 954 }
 955
 956 static void
 957 port_in_build_free(struct rte_swx_pipeline *p)
 958 {
 959         free(p->in);
 960         p->in = NULL;
 961 }
 962
 963 static void
 964 port_in_free(struct rte_swx_pipeline *p)
 965 {
 966         port_in_build_free(p);
 967
 968         /* Input ports. */
 969         for ( ; ; ) {
 970                 struct port_in *port;
 971
 972                 port = TAILQ_FIRST(&p->ports_in);
 973                 if (!port)
 974                         break;
 975
 976                 TAILQ_REMOVE(&p->ports_in, port, node);
 977                 port->type->ops.free(port->obj);
 978                 free(port);
 979         }
 980
 981         /* Input port types. */
 982         for ( ; ; ) {
 983                 struct port_in_type *elem;
 984
 985                 elem = TAILQ_FIRST(&p->port_in_types);
 986                 if (!elem)
 987                         break;
 988
 989                 TAILQ_REMOVE(&p->port_in_types, elem, node);
 990                 free(elem);
 991         }
 992 }
 993
 994 /*
 995  * Output port.
 996  */
 997 static struct port_out_type *
 998 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
 999 {
1000         struct port_out_type *elem;
1001
1002         if (!name)
1003                 return NULL;
1004
1005         TAILQ_FOREACH(elem, &p->port_out_types, node)
1006                 if (!strcmp(elem->name, name))
1007                         return elem;
1008
1009         return NULL;
1010 }
1011
1012 int
1013 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1014                                         const char *name,
1015                                         struct rte_swx_port_out_ops *ops)
1016 {
1017         struct port_out_type *elem;
1018
1019         CHECK(p, EINVAL);
1020         CHECK_NAME(name, EINVAL);
1021         CHECK(ops, EINVAL);
1022         CHECK(ops->create, EINVAL);
1023         CHECK(ops->free, EINVAL);
1024         CHECK(ops->pkt_tx, EINVAL);
1025         CHECK(ops->stats_read, EINVAL);
1026
1027         CHECK(!port_out_type_find(p, name), EEXIST);
1028
1029         /* Node allocation. */
1030         elem = calloc(1, sizeof(struct port_out_type));
1031         CHECK(elem, ENOMEM);
1032
1033         /* Node initialization. */
1034         strcpy(elem->name, name);
1035         memcpy(&elem->ops, ops, sizeof(*ops));
1036
1037         /* Node add to tailq. */
1038         TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1039
1040         return 0;
1041 }
1042
1043 static struct port_out *
1044 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1045 {
1046         struct port_out *port;
1047
1048         TAILQ_FOREACH(port, &p->ports_out, node)
1049                 if (port->id == port_id)
1050                         return port;
1051
1052         return NULL;
1053 }
1054
1055 int
1056 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1057                                  uint32_t port_id,
1058                                  const char *port_type_name,
1059                                  void *args)
1060 {
1061         struct port_out_type *type = NULL;
1062         struct port_out *port = NULL;
1063         void *obj = NULL;
1064
1065         CHECK(p, EINVAL);
1066
1067         CHECK(!port_out_find(p, port_id), EINVAL);
1068
1069         CHECK_NAME(port_type_name, EINVAL);
1070         type = port_out_type_find(p, port_type_name);
1071         CHECK(type, EINVAL);
1072
1073         obj = type->ops.create(args);
1074         CHECK(obj, ENODEV);
1075
1076         /* Node allocation. */
1077         port = calloc(1, sizeof(struct port_out));
1078         CHECK(port, ENOMEM);
1079
1080         /* Node initialization. */
1081         port->type = type;
1082         port->obj = obj;
1083         port->id = port_id;
1084
1085         /* Node add to tailq. */
1086         TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1087         if (p->n_ports_out < port_id + 1)
1088                 p->n_ports_out = port_id + 1;
1089
1090         return 0;
1091 }
1092
1093 static int
1094 port_out_build(struct rte_swx_pipeline *p)
1095 {
1096         struct port_out *port;
1097         uint32_t i;
1098
1099         CHECK(p->n_ports_out, EINVAL);
1100
1101         for (i = 0; i < p->n_ports_out; i++)
1102                 CHECK(port_out_find(p, i), EINVAL);
1103
1104         p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1105         CHECK(p->out, ENOMEM);
1106
1107         TAILQ_FOREACH(port, &p->ports_out, node) {
1108                 struct port_out_runtime *out = &p->out[port->id];
1109
1110                 out->pkt_tx = port->type->ops.pkt_tx;
1111                 out->flush = port->type->ops.flush;
1112                 out->obj = port->obj;
1113         }
1114
1115         return 0;
1116 }
1117
1118 static void
1119 port_out_build_free(struct rte_swx_pipeline *p)
1120 {
1121         free(p->out);
1122         p->out = NULL;
1123 }
1124
1125 static void
1126 port_out_free(struct rte_swx_pipeline *p)
1127 {
1128         port_out_build_free(p);
1129
1130         /* Output ports. */
1131         for ( ; ; ) {
1132                 struct port_out *port;
1133
1134                 port = TAILQ_FIRST(&p->ports_out);
1135                 if (!port)
1136                         break;
1137
1138                 TAILQ_REMOVE(&p->ports_out, port, node);
1139                 port->type->ops.free(port->obj);
1140                 free(port);
1141         }
1142
1143         /* Output port types. */
1144         for ( ; ; ) {
1145                 struct port_out_type *elem;
1146
1147                 elem = TAILQ_FIRST(&p->port_out_types);
1148                 if (!elem)
1149                         break;
1150
1151                 TAILQ_REMOVE(&p->port_out_types, elem, node);
1152                 free(elem);
1153         }
1154 }
1155
1156 /*
1157  * Extern object.
1158  */
1159 static struct extern_type *
1160 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1161 {
1162         struct extern_type *elem;
1163
1164         TAILQ_FOREACH(elem, &p->extern_types, node)
1165                 if (strcmp(elem->name, name) == 0)
1166                         return elem;
1167
1168         return NULL;
1169 }
1170
1171 static struct extern_type_member_func *
1172 extern_type_member_func_find(struct extern_type *type, const char *name)
1173 {
1174         struct extern_type_member_func *elem;
1175
1176         TAILQ_FOREACH(elem, &type->funcs, node)
1177                 if (strcmp(elem->name, name) == 0)
1178                         return elem;
1179
1180         return NULL;
1181 }
1182
1183 static struct extern_obj *
1184 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1185 {
1186         struct extern_obj *elem;
1187
1188         TAILQ_FOREACH(elem, &p->extern_objs, node)
1189                 if (strcmp(elem->name, name) == 0)
1190                         return elem;
1191
1192         return NULL;
1193 }
1194
1195 static struct field *
1196 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1197                                const char *name,
1198                                struct extern_obj **object)
1199 {
1200         struct extern_obj *obj;
1201         struct field *f;
1202         char *obj_name, *field_name;
1203
1204         if ((name[0] != 'e') || (name[1] != '.'))
1205                 return NULL;
1206
1207         obj_name = strdup(&name[2]);
1208         if (!obj_name)
1209                 return NULL;
1210
1211         field_name = strchr(obj_name, '.');
1212         if (!field_name) {
1213                 free(obj_name);
1214                 return NULL;
1215         }
1216
1217         *field_name = 0;
1218         field_name++;
1219
1220         obj = extern_obj_find(p, obj_name);
1221         if (!obj) {
1222                 free(obj_name);
1223                 return NULL;
1224         }
1225
1226         f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1227         if (!f) {
1228                 free(obj_name);
1229                 return NULL;
1230         }
1231
1232         if (object)
1233                 *object = obj;
1234
1235         free(obj_name);
1236         return f;
1237 }
1238
1239 int
1240 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1241         const char *name,
1242         const char *mailbox_struct_type_name,
1243         rte_swx_extern_type_constructor_t constructor,
1244         rte_swx_extern_type_destructor_t destructor)
1245 {
1246         struct extern_type *elem;
1247         struct struct_type *mailbox_struct_type;
1248
1249         CHECK(p, EINVAL);
1250
1251         CHECK_NAME(name, EINVAL);
1252         CHECK(!extern_type_find(p, name), EEXIST);
1253
1254         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1255         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1256         CHECK(mailbox_struct_type, EINVAL);
1257
1258         CHECK(constructor, EINVAL);
1259         CHECK(destructor, EINVAL);
1260
1261         /* Node allocation. */
1262         elem = calloc(1, sizeof(struct extern_type));
1263         CHECK(elem, ENOMEM);
1264
1265         /* Node initialization. */
1266         strcpy(elem->name, name);
1267         elem->mailbox_struct_type = mailbox_struct_type;
1268         elem->constructor = constructor;
1269         elem->destructor = destructor;
1270         TAILQ_INIT(&elem->funcs);
1271
1272         /* Node add to tailq. */
1273         TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1274
1275         return 0;
1276 }
1277
1278 int
1279 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1280         const char *extern_type_name,
1281         const char *name,
1282         rte_swx_extern_type_member_func_t member_func)
1283 {
1284         struct extern_type *type;
1285         struct extern_type_member_func *type_member;
1286
1287         CHECK(p, EINVAL);
1288
1289         CHECK(extern_type_name, EINVAL);
1290         type = extern_type_find(p, extern_type_name);
1291         CHECK(type, EINVAL);
1292         CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1293
1294         CHECK(name, EINVAL);
1295         CHECK(!extern_type_member_func_find(type, name), EEXIST);
1296
1297         CHECK(member_func, EINVAL);
1298
1299         /* Node allocation. */
1300         type_member = calloc(1, sizeof(struct extern_type_member_func));
1301         CHECK(type_member, ENOMEM);
1302
1303         /* Node initialization. */
1304         strcpy(type_member->name, name);
1305         type_member->func = member_func;
1306         type_member->id = type->n_funcs;
1307
1308         /* Node add to tailq. */
1309         TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1310         type->n_funcs++;
1311
1312         return 0;
1313 }
1314
1315 int
1316 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1317                                       const char *extern_type_name,
1318                                       const char *name,
1319                                       const char *args)
1320 {
1321         struct extern_type *type;
1322         struct extern_obj *obj;
1323         void *obj_handle;
1324
1325         CHECK(p, EINVAL);
1326
1327         CHECK_NAME(extern_type_name, EINVAL);
1328         type = extern_type_find(p, extern_type_name);
1329         CHECK(type, EINVAL);
1330
1331         CHECK_NAME(name, EINVAL);
1332         CHECK(!extern_obj_find(p, name), EEXIST);
1333
1334         /* Node allocation. */
1335         obj = calloc(1, sizeof(struct extern_obj));
1336         CHECK(obj, ENOMEM);
1337
1338         /* Object construction. */
1339         obj_handle = type->constructor(args);
1340         if (!obj_handle) {
1341                 free(obj);
1342                 CHECK(0, ENODEV);
1343         }
1344
1345         /* Node initialization. */
1346         strcpy(obj->name, name);
1347         obj->type = type;
1348         obj->obj = obj_handle;
1349         obj->struct_id = p->n_structs;
1350         obj->id = p->n_extern_objs;
1351
1352         /* Node add to tailq. */
1353         TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1354         p->n_extern_objs++;
1355         p->n_structs++;
1356
1357         return 0;
1358 }
1359
1360 static int
1361 extern_obj_build(struct rte_swx_pipeline *p)
1362 {
1363         uint32_t i;
1364
1365         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1366                 struct thread *t = &p->threads[i];
1367                 struct extern_obj *obj;
1368
1369                 t->extern_objs = calloc(p->n_extern_objs,
1370                                         sizeof(struct extern_obj_runtime));
1371                 CHECK(t->extern_objs, ENOMEM);
1372
1373                 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1374                         struct extern_obj_runtime *r =
1375                                 &t->extern_objs[obj->id];
1376                         struct extern_type_member_func *func;
1377                         uint32_t mailbox_size =
1378                                 obj->type->mailbox_struct_type->n_bits / 8;
1379
1380                         r->obj = obj->obj;
1381
1382                         r->mailbox = calloc(1, mailbox_size);
1383                         CHECK(r->mailbox, ENOMEM);
1384
1385                         TAILQ_FOREACH(func, &obj->type->funcs, node)
1386                                 r->funcs[func->id] = func->func;
1387
1388                         t->structs[obj->struct_id] = r->mailbox;
1389                 }
1390         }
1391
1392         return 0;
1393 }
1394
1395 static void
1396 extern_obj_build_free(struct rte_swx_pipeline *p)
1397 {
1398         uint32_t i;
1399
1400         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1401                 struct thread *t = &p->threads[i];
1402                 uint32_t j;
1403
1404                 if (!t->extern_objs)
1405                         continue;
1406
1407                 for (j = 0; j < p->n_extern_objs; j++) {
1408                         struct extern_obj_runtime *r = &t->extern_objs[j];
1409
1410                         free(r->mailbox);
1411                 }
1412
1413                 free(t->extern_objs);
1414                 t->extern_objs = NULL;
1415         }
1416 }
1417
1418 static void
1419 extern_obj_free(struct rte_swx_pipeline *p)
1420 {
1421         extern_obj_build_free(p);
1422
1423         /* Extern objects. */
1424         for ( ; ; ) {
1425                 struct extern_obj *elem;
1426
1427                 elem = TAILQ_FIRST(&p->extern_objs);
1428                 if (!elem)
1429                         break;
1430
1431                 TAILQ_REMOVE(&p->extern_objs, elem, node);
1432                 if (elem->obj)
1433                         elem->type->destructor(elem->obj);
1434                 free(elem);
1435         }
1436
1437         /* Extern types. */
1438         for ( ; ; ) {
1439                 struct extern_type *elem;
1440
1441                 elem = TAILQ_FIRST(&p->extern_types);
1442                 if (!elem)
1443                         break;
1444
1445                 TAILQ_REMOVE(&p->extern_types, elem, node);
1446
1447                 for ( ; ; ) {
1448                         struct extern_type_member_func *func;
1449
1450                         func = TAILQ_FIRST(&elem->funcs);
1451                         if (!func)
1452                                 break;
1453
1454                         TAILQ_REMOVE(&elem->funcs, func, node);
1455                         free(func);
1456                 }
1457
1458                 free(elem);
1459         }
1460 }
1461
1462 /*
1463  * Extern function.
1464  */
1465 static struct extern_func *
1466 extern_func_find(struct rte_swx_pipeline *p, const char *name)
1467 {
1468         struct extern_func *elem;
1469
1470         TAILQ_FOREACH(elem, &p->extern_funcs, node)
1471                 if (strcmp(elem->name, name) == 0)
1472                         return elem;
1473
1474         return NULL;
1475 }
1476
1477 static struct field *
1478 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1479                                 const char *name,
1480                                 struct extern_func **function)
1481 {
1482         struct extern_func *func;
1483         struct field *f;
1484         char *func_name, *field_name;
1485
1486         if ((name[0] != 'f') || (name[1] != '.'))
1487                 return NULL;
1488
1489         func_name = strdup(&name[2]);
1490         if (!func_name)
1491                 return NULL;
1492
1493         field_name = strchr(func_name, '.');
1494         if (!field_name) {
1495                 free(func_name);
1496                 return NULL;
1497         }
1498
1499         *field_name = 0;
1500         field_name++;
1501
1502         func = extern_func_find(p, func_name);
1503         if (!func) {
1504                 free(func_name);
1505                 return NULL;
1506         }
1507
1508         f = struct_type_field_find(func->mailbox_struct_type, field_name);
1509         if (!f) {
1510                 free(func_name);
1511                 return NULL;
1512         }
1513
1514         if (function)
1515                 *function = func;
1516
1517         free(func_name);
1518         return f;
1519 }
1520
1521 int
1522 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1523                                       const char *name,
1524                                       const char *mailbox_struct_type_name,
1525                                       rte_swx_extern_func_t func)
1526 {
1527         struct extern_func *f;
1528         struct struct_type *mailbox_struct_type;
1529
1530         CHECK(p, EINVAL);
1531
1532         CHECK_NAME(name, EINVAL);
1533         CHECK(!extern_func_find(p, name), EEXIST);
1534
1535         CHECK_NAME(mailbox_struct_type_name, EINVAL);
1536         mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1537         CHECK(mailbox_struct_type, EINVAL);
1538
1539         CHECK(func, EINVAL);
1540
1541         /* Node allocation. */
1542         f = calloc(1, sizeof(struct extern_func));
1543         CHECK(func, ENOMEM);
1544
1545         /* Node initialization. */
1546         strcpy(f->name, name);
1547         f->mailbox_struct_type = mailbox_struct_type;
1548         f->func = func;
1549         f->struct_id = p->n_structs;
1550         f->id = p->n_extern_funcs;
1551
1552         /* Node add to tailq. */
1553         TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1554         p->n_extern_funcs++;
1555         p->n_structs++;
1556
1557         return 0;
1558 }
1559
1560 static int
1561 extern_func_build(struct rte_swx_pipeline *p)
1562 {
1563         uint32_t i;
1564
1565         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1566                 struct thread *t = &p->threads[i];
1567                 struct extern_func *func;
1568
1569                 /* Memory allocation. */
1570                 t->extern_funcs = calloc(p->n_extern_funcs,
1571                                          sizeof(struct extern_func_runtime));
1572                 CHECK(t->extern_funcs, ENOMEM);
1573
1574                 /* Extern function. */
1575                 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1576                         struct extern_func_runtime *r =
1577                                 &t->extern_funcs[func->id];
1578                         uint32_t mailbox_size =
1579                                 func->mailbox_struct_type->n_bits / 8;
1580
1581                         r->func = func->func;
1582
1583                         r->mailbox = calloc(1, mailbox_size);
1584                         CHECK(r->mailbox, ENOMEM);
1585
1586                         t->structs[func->struct_id] = r->mailbox;
1587                 }
1588         }
1589
1590         return 0;
1591 }
1592
1593 static void
1594 extern_func_build_free(struct rte_swx_pipeline *p)
1595 {
1596         uint32_t i;
1597
1598         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1599                 struct thread *t = &p->threads[i];
1600                 uint32_t j;
1601
1602                 if (!t->extern_funcs)
1603                         continue;
1604
1605                 for (j = 0; j < p->n_extern_funcs; j++) {
1606                         struct extern_func_runtime *r = &t->extern_funcs[j];
1607
1608                         free(r->mailbox);
1609                 }
1610
1611                 free(t->extern_funcs);
1612                 t->extern_funcs = NULL;
1613         }
1614 }
1615
1616 static void
1617 extern_func_free(struct rte_swx_pipeline *p)
1618 {
1619         extern_func_build_free(p);
1620
1621         for ( ; ; ) {
1622                 struct extern_func *elem;
1623
1624                 elem = TAILQ_FIRST(&p->extern_funcs);
1625                 if (!elem)
1626                         break;
1627
1628                 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1629                 free(elem);
1630         }
1631 }
1632
1633 /*
1634  * Header.
1635  */
1636 static struct header *
1637 header_find(struct rte_swx_pipeline *p, const char *name)
1638 {
1639         struct header *elem;
1640
1641         TAILQ_FOREACH(elem, &p->headers, node)
1642                 if (strcmp(elem->name, name) == 0)
1643                         return elem;
1644
1645         return NULL;
1646 }
1647
1648 static struct header *
1649 header_parse(struct rte_swx_pipeline *p,
1650              const char *name)
1651 {
1652         if (name[0] != 'h' || name[1] != '.')
1653                 return NULL;
1654
1655         return header_find(p, &name[2]);
1656 }
1657
1658 static struct field *
1659 header_field_parse(struct rte_swx_pipeline *p,
1660                    const char *name,
1661                    struct header **header)
1662 {
1663         struct header *h;
1664         struct field *f;
1665         char *header_name, *field_name;
1666
1667         if ((name[0] != 'h') || (name[1] != '.'))
1668                 return NULL;
1669
1670         header_name = strdup(&name[2]);
1671         if (!header_name)
1672                 return NULL;
1673
1674         field_name = strchr(header_name, '.');
1675         if (!field_name) {
1676                 free(header_name);
1677                 return NULL;
1678         }
1679
1680         *field_name = 0;
1681         field_name++;
1682
1683         h = header_find(p, header_name);
1684         if (!h) {
1685                 free(header_name);
1686                 return NULL;
1687         }
1688
1689         f = struct_type_field_find(h->st, field_name);
1690         if (!f) {
1691                 free(header_name);
1692                 return NULL;
1693         }
1694
1695         if (header)
1696                 *header = h;
1697
1698         free(header_name);
1699         return f;
1700 }
1701
1702 int
1703 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1704                                         const char *name,
1705                                         const char *struct_type_name)
1706 {
1707         struct struct_type *st;
1708         struct header *h;
1709         size_t n_headers_max;
1710
1711         CHECK(p, EINVAL);
1712         CHECK_NAME(name, EINVAL);
1713         CHECK_NAME(struct_type_name, EINVAL);
1714
1715         CHECK(!header_find(p, name), EEXIST);
1716
1717         st = struct_type_find(p, struct_type_name);
1718         CHECK(st, EINVAL);
1719
1720         n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1721         CHECK(p->n_headers < n_headers_max, ENOSPC);
1722
1723         /* Node allocation. */
1724         h = calloc(1, sizeof(struct header));
1725         CHECK(h, ENOMEM);
1726
1727         /* Node initialization. */
1728         strcpy(h->name, name);
1729         h->st = st;
1730         h->struct_id = p->n_structs;
1731         h->id = p->n_headers;
1732
1733         /* Node add to tailq. */
1734         TAILQ_INSERT_TAIL(&p->headers, h, node);
1735         p->n_headers++;
1736         p->n_structs++;
1737
1738         return 0;
1739 }
1740
1741 static int
1742 header_build(struct rte_swx_pipeline *p)
1743 {
1744         struct header *h;
1745         uint32_t n_bytes = 0, i;
1746
1747         TAILQ_FOREACH(h, &p->headers, node) {
1748                 n_bytes += h->st->n_bits / 8;
1749         }
1750
1751         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1752                 struct thread *t = &p->threads[i];
1753                 uint32_t offset = 0;
1754
1755                 t->headers = calloc(p->n_headers,
1756                                     sizeof(struct header_runtime));
1757                 CHECK(t->headers, ENOMEM);
1758
1759                 t->headers_out = calloc(p->n_headers,
1760                                         sizeof(struct header_out_runtime));
1761                 CHECK(t->headers_out, ENOMEM);
1762
1763                 t->header_storage = calloc(1, n_bytes);
1764                 CHECK(t->header_storage, ENOMEM);
1765
1766                 t->header_out_storage = calloc(1, n_bytes);
1767                 CHECK(t->header_out_storage, ENOMEM);
1768
1769                 TAILQ_FOREACH(h, &p->headers, node) {
1770                         uint8_t *header_storage;
1771
1772                         header_storage = &t->header_storage[offset];
1773                         offset += h->st->n_bits / 8;
1774
1775                         t->headers[h->id].ptr0 = header_storage;
1776                         t->structs[h->struct_id] = header_storage;
1777                 }
1778         }
1779
1780         return 0;
1781 }
1782
1783 static void
1784 header_build_free(struct rte_swx_pipeline *p)
1785 {
1786         uint32_t i;
1787
1788         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1789                 struct thread *t = &p->threads[i];
1790
1791                 free(t->headers_out);
1792                 t->headers_out = NULL;
1793
1794                 free(t->headers);
1795                 t->headers = NULL;
1796
1797                 free(t->header_out_storage);
1798                 t->header_out_storage = NULL;
1799
1800                 free(t->header_storage);
1801                 t->header_storage = NULL;
1802         }
1803 }
1804
1805 static void
1806 header_free(struct rte_swx_pipeline *p)
1807 {
1808         header_build_free(p);
1809
1810         for ( ; ; ) {
1811                 struct header *elem;
1812
1813                 elem = TAILQ_FIRST(&p->headers);
1814                 if (!elem)
1815                         break;
1816
1817                 TAILQ_REMOVE(&p->headers, elem, node);
1818                 free(elem);
1819         }
1820 }
1821
1822 /*
1823  * Meta-data.
1824  */
1825 static struct field *
1826 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1827 {
1828         if (!p->metadata_st)
1829                 return NULL;
1830
1831         if (name[0] != 'm' || name[1] != '.')
1832                 return NULL;
1833
1834         return struct_type_field_find(p->metadata_st, &name[2]);
1835 }
1836
1837 int
1838 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1839                                           const char *struct_type_name)
1840 {
1841         struct struct_type *st = NULL;
1842
1843         CHECK(p, EINVAL);
1844
1845         CHECK_NAME(struct_type_name, EINVAL);
1846         st  = struct_type_find(p, struct_type_name);
1847         CHECK(st, EINVAL);
1848         CHECK(!p->metadata_st, EINVAL);
1849
1850         p->metadata_st = st;
1851         p->metadata_struct_id = p->n_structs;
1852
1853         p->n_structs++;
1854
1855         return 0;
1856 }
1857
1858 static int
1859 metadata_build(struct rte_swx_pipeline *p)
1860 {
1861         uint32_t n_bytes = p->metadata_st->n_bits / 8;
1862         uint32_t i;
1863
1864         /* Thread-level initialization. */
1865         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1866                 struct thread *t = &p->threads[i];
1867                 uint8_t *metadata;
1868
1869                 metadata = calloc(1, n_bytes);
1870                 CHECK(metadata, ENOMEM);
1871
1872                 t->metadata = metadata;
1873                 t->structs[p->metadata_struct_id] = metadata;
1874         }
1875
1876         return 0;
1877 }
1878
1879 static void
1880 metadata_build_free(struct rte_swx_pipeline *p)
1881 {
1882         uint32_t i;
1883
1884         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1885                 struct thread *t = &p->threads[i];
1886
1887                 free(t->metadata);
1888                 t->metadata = NULL;
1889         }
1890 }
1891
1892 static void
1893 metadata_free(struct rte_swx_pipeline *p)
1894 {
1895         metadata_build_free(p);
1896 }
1897
1898 /*
1899  * Instruction.
1900  */
1901 static struct field *
1902 action_field_parse(struct action *action, const char *name);
1903
1904 static struct field *
1905 struct_field_parse(struct rte_swx_pipeline *p,
1906                    struct action *action,
1907                    const char *name,
1908                    uint32_t *struct_id)
1909 {
1910         struct field *f;
1911
1912         switch (name[0]) {
1913         case 'h':
1914         {
1915                 struct header *header;
1916
1917                 f = header_field_parse(p, name, &header);
1918                 if (!f)
1919                         return NULL;
1920
1921                 *struct_id = header->struct_id;
1922                 return f;
1923         }
1924
1925         case 'm':
1926         {
1927                 f = metadata_field_parse(p, name);
1928                 if (!f)
1929                         return NULL;
1930
1931                 *struct_id = p->metadata_struct_id;
1932                 return f;
1933         }
1934
1935         case 't':
1936         {
1937                 if (!action)
1938                         return NULL;
1939
1940                 f = action_field_parse(action, name);
1941                 if (!f)
1942                         return NULL;
1943
1944                 *struct_id = 0;
1945                 return f;
1946         }
1947
1948         case 'e':
1949         {
1950                 struct extern_obj *obj;
1951
1952                 f = extern_obj_mailbox_field_parse(p, name, &obj);
1953                 if (!f)
1954                         return NULL;
1955
1956                 *struct_id = obj->struct_id;
1957                 return f;
1958         }
1959
1960         case 'f':
1961         {
1962                 struct extern_func *func;
1963
1964                 f = extern_func_mailbox_field_parse(p, name, &func);
1965                 if (!f)
1966                         return NULL;
1967
1968                 *struct_id = func->struct_id;
1969                 return f;
1970         }
1971
1972         default:
1973                 return NULL;
1974         }
1975 }
1976
1977 static inline void
1978 pipeline_port_inc(struct rte_swx_pipeline *p)
1979 {
1980         p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1981 }
1982
1983 static inline void
1984 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1985 {
1986         t->ip = p->instructions;
1987 }
1988
1989 static inline void
1990 thread_ip_inc(struct rte_swx_pipeline *p);
1991
1992 static inline void
1993 thread_ip_inc(struct rte_swx_pipeline *p)
1994 {
1995         struct thread *t = &p->threads[p->thread_id];
1996
1997         t->ip++;
1998 }
1999
2000 static inline void
2001 thread_ip_inc_cond(struct thread *t, int cond)
2002 {
2003         t->ip += cond;
2004 }
2005
2006 static inline void
2007 thread_yield(struct rte_swx_pipeline *p)
2008 {
2009         p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2010 }
2011
2012 /*
2013  * rx.
2014  */
2015 static int
2016 instr_rx_translate(struct rte_swx_pipeline *p,
2017                    struct action *action,
2018                    char **tokens,
2019                    int n_tokens,
2020                    struct instruction *instr,
2021                    struct instruction_data *data __rte_unused)
2022 {
2023         struct field *f;
2024
2025         CHECK(!action, EINVAL);
2026         CHECK(n_tokens == 2, EINVAL);
2027
2028         f = metadata_field_parse(p, tokens[1]);
2029         CHECK(f, EINVAL);
2030
2031         instr->type = INSTR_RX;
2032         instr->io.io.offset = f->offset / 8;
2033         instr->io.io.n_bits = f->n_bits;
2034         return 0;
2035 }
2036
2037 static inline void
2038 instr_rx_exec(struct rte_swx_pipeline *p);
2039
2040 static inline void
2041 instr_rx_exec(struct rte_swx_pipeline *p)
2042 {
2043         struct thread *t = &p->threads[p->thread_id];
2044         struct instruction *ip = t->ip;
2045         struct port_in_runtime *port = &p->in[p->port_id];
2046         struct rte_swx_pkt *pkt = &t->pkt;
2047         int pkt_received;
2048
2049         /* Packet. */
2050         pkt_received = port->pkt_rx(port->obj, pkt);
2051         t->ptr = &pkt->pkt[pkt->offset];
2052         rte_prefetch0(t->ptr);
2053
2054         TRACE("[Thread %2u] rx %s from port %u\n",
2055               p->thread_id,
2056               pkt_received ? "1 pkt" : "0 pkts",
2057               p->port_id);
2058
2059         /* Headers. */
2060         t->valid_headers = 0;
2061         t->n_headers_out = 0;
2062
2063         /* Meta-data. */
2064         METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2065
2066         /* Tables. */
2067         t->table_state = p->table_state;
2068
2069         /* Thread. */
2070         pipeline_port_inc(p);
2071         thread_ip_inc_cond(t, pkt_received);
2072         thread_yield(p);
2073 }
2074
2075 /*
2076  * tx.
2077  */
2078 static int
2079 instr_tx_translate(struct rte_swx_pipeline *p,
2080                    struct action *action __rte_unused,
2081                    char **tokens,
2082                    int n_tokens,
2083                    struct instruction *instr,
2084                    struct instruction_data *data __rte_unused)
2085 {
2086         struct field *f;
2087
2088         CHECK(n_tokens == 2, EINVAL);
2089
2090         f = metadata_field_parse(p, tokens[1]);
2091         CHECK(f, EINVAL);
2092
2093         instr->type = INSTR_TX;
2094         instr->io.io.offset = f->offset / 8;
2095         instr->io.io.n_bits = f->n_bits;
2096         return 0;
2097 }
2098
2099 static inline void
2100 emit_handler(struct thread *t)
2101 {
2102         struct header_out_runtime *h0 = &t->headers_out[0];
2103         struct header_out_runtime *h1 = &t->headers_out[1];
2104         uint32_t offset = 0, i;
2105
2106         /* No header change or header decapsulation. */
2107         if ((t->n_headers_out == 1) &&
2108             (h0->ptr + h0->n_bytes == t->ptr)) {
2109                 TRACE("Emit handler: no header change or header decap.\n");
2110
2111                 t->pkt.offset -= h0->n_bytes;
2112                 t->pkt.length += h0->n_bytes;
2113
2114                 return;
2115         }
2116
2117         /* Header encapsulation (optionally, with prior header decasulation). */
2118         if ((t->n_headers_out == 2) &&
2119             (h1->ptr + h1->n_bytes == t->ptr) &&
2120             (h0->ptr == h0->ptr0)) {
2121                 uint32_t offset;
2122
2123                 TRACE("Emit handler: header encapsulation.\n");
2124
2125                 offset = h0->n_bytes + h1->n_bytes;
2126                 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2127                 t->pkt.offset -= offset;
2128                 t->pkt.length += offset;
2129
2130                 return;
2131         }
2132
2133         /* Header insertion. */
2134         /* TBD */
2135
2136         /* Header extraction. */
2137         /* TBD */
2138
2139         /* For any other case. */
2140         TRACE("Emit handler: complex case.\n");
2141
2142         for (i = 0; i < t->n_headers_out; i++) {
2143                 struct header_out_runtime *h = &t->headers_out[i];
2144
2145                 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2146                 offset += h->n_bytes;
2147         }
2148
2149         if (offset) {
2150                 memcpy(t->ptr - offset, t->header_out_storage, offset);
2151                 t->pkt.offset -= offset;
2152                 t->pkt.length += offset;
2153         }
2154 }
2155
2156 static inline void
2157 instr_tx_exec(struct rte_swx_pipeline *p);
2158
2159 static inline void
2160 instr_tx_exec(struct rte_swx_pipeline *p)
2161 {
2162         struct thread *t = &p->threads[p->thread_id];
2163         struct instruction *ip = t->ip;
2164         uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2165         struct port_out_runtime *port = &p->out[port_id];
2166         struct rte_swx_pkt *pkt = &t->pkt;
2167
2168         TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2169               p->thread_id,
2170               (uint32_t)port_id);
2171
2172         /* Headers. */
2173         emit_handler(t);
2174
2175         /* Packet. */
2176         port->pkt_tx(port->obj, pkt);
2177
2178         /* Thread. */
2179         thread_ip_reset(p, t);
2180         instr_rx_exec(p);
2181 }
2182
2183 /*
2184  * extract.
2185  */
2186 static int
2187 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2188                             struct action *action,
2189                             char **tokens,
2190                             int n_tokens,
2191                             struct instruction *instr,
2192                             struct instruction_data *data __rte_unused)
2193 {
2194         struct header *h;
2195
2196         CHECK(!action, EINVAL);
2197         CHECK(n_tokens == 2, EINVAL);
2198
2199         h = header_parse(p, tokens[1]);
2200         CHECK(h, EINVAL);
2201
2202         instr->type = INSTR_HDR_EXTRACT;
2203         instr->io.hdr.header_id[0] = h->id;
2204         instr->io.hdr.struct_id[0] = h->struct_id;
2205         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2206         return 0;
2207 }
2208
2209 static inline void
2210 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2211
2212 static inline void
2213 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2214 {
2215         struct thread *t = &p->threads[p->thread_id];
2216         struct instruction *ip = t->ip;
2217         uint64_t valid_headers = t->valid_headers;
2218         uint8_t *ptr = t->ptr;
2219         uint32_t offset = t->pkt.offset;
2220         uint32_t length = t->pkt.length;
2221         uint32_t i;
2222
2223         for (i = 0; i < n_extract; i++) {
2224                 uint32_t header_id = ip->io.hdr.header_id[i];
2225                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2226                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2227
2228                 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2229                       p->thread_id,
2230                       header_id,
2231                       n_bytes);
2232
2233                 /* Headers. */
2234                 t->structs[struct_id] = ptr;
2235                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2236
2237                 /* Packet. */
2238                 offset += n_bytes;
2239                 length -= n_bytes;
2240                 ptr += n_bytes;
2241         }
2242
2243         /* Headers. */
2244         t->valid_headers = valid_headers;
2245
2246         /* Packet. */
2247         t->pkt.offset = offset;
2248         t->pkt.length = length;
2249         t->ptr = ptr;
2250 }
2251
2252 static inline void
2253 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2254 {
2255         __instr_hdr_extract_exec(p, 1);
2256
2257         /* Thread. */
2258         thread_ip_inc(p);
2259 }
2260
2261 static inline void
2262 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2263 {
2264         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2265               p->thread_id);
2266
2267         __instr_hdr_extract_exec(p, 2);
2268
2269         /* Thread. */
2270         thread_ip_inc(p);
2271 }
2272
2273 static inline void
2274 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2275 {
2276         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2277               p->thread_id);
2278
2279         __instr_hdr_extract_exec(p, 3);
2280
2281         /* Thread. */
2282         thread_ip_inc(p);
2283 }
2284
2285 static inline void
2286 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2287 {
2288         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2289               p->thread_id);
2290
2291         __instr_hdr_extract_exec(p, 4);
2292
2293         /* Thread. */
2294         thread_ip_inc(p);
2295 }
2296
2297 static inline void
2298 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2299 {
2300         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2301               p->thread_id);
2302
2303         __instr_hdr_extract_exec(p, 5);
2304
2305         /* Thread. */
2306         thread_ip_inc(p);
2307 }
2308
2309 static inline void
2310 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2311 {
2312         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2313               p->thread_id);
2314
2315         __instr_hdr_extract_exec(p, 6);
2316
2317         /* Thread. */
2318         thread_ip_inc(p);
2319 }
2320
2321 static inline void
2322 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2323 {
2324         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2325               p->thread_id);
2326
2327         __instr_hdr_extract_exec(p, 7);
2328
2329         /* Thread. */
2330         thread_ip_inc(p);
2331 }
2332
2333 static inline void
2334 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2335 {
2336         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2337               p->thread_id);
2338
2339         __instr_hdr_extract_exec(p, 8);
2340
2341         /* Thread. */
2342         thread_ip_inc(p);
2343 }
2344
2345 /*
2346  * emit.
2347  */
2348 static int
2349 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2350                          struct action *action __rte_unused,
2351                          char **tokens,
2352                          int n_tokens,
2353                          struct instruction *instr,
2354                          struct instruction_data *data __rte_unused)
2355 {
2356         struct header *h;
2357
2358         CHECK(n_tokens == 2, EINVAL);
2359
2360         h = header_parse(p, tokens[1]);
2361         CHECK(h, EINVAL);
2362
2363         instr->type = INSTR_HDR_EMIT;
2364         instr->io.hdr.header_id[0] = h->id;
2365         instr->io.hdr.struct_id[0] = h->struct_id;
2366         instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2367         return 0;
2368 }
2369
2370 static inline void
2371 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
2372
2373 static inline void
2374 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
2375 {
2376         struct thread *t = &p->threads[p->thread_id];
2377         struct instruction *ip = t->ip;
2378         uint32_t n_headers_out = t->n_headers_out;
2379         struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
2380         uint8_t *ho_ptr = NULL;
2381         uint32_t ho_nbytes = 0, i;
2382
2383         for (i = 0; i < n_emit; i++) {
2384                 uint32_t header_id = ip->io.hdr.header_id[i];
2385                 uint32_t struct_id = ip->io.hdr.struct_id[i];
2386                 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2387
2388                 struct header_runtime *hi = &t->headers[header_id];
2389                 uint8_t *hi_ptr = t->structs[struct_id];
2390
2391                 TRACE("[Thread %2u]: emit header %u\n",
2392                       p->thread_id,
2393                       header_id);
2394
2395                 /* Headers. */
2396                 if (!i) {
2397                         if (!t->n_headers_out) {
2398                                 ho = &t->headers_out[0];
2399
2400                                 ho->ptr0 = hi->ptr0;
2401                                 ho->ptr = hi_ptr;
2402
2403                                 ho_ptr = hi_ptr;
2404                                 ho_nbytes = n_bytes;
2405
2406                                 n_headers_out = 1;
2407
2408                                 continue;
2409                         } else {
2410                                 ho_ptr = ho->ptr;
2411                                 ho_nbytes = ho->n_bytes;
2412                         }
2413                 }
2414
2415                 if (ho_ptr + ho_nbytes == hi_ptr) {
2416                         ho_nbytes += n_bytes;
2417                 } else {
2418                         ho->n_bytes = ho_nbytes;
2419
2420                         ho++;
2421                         ho->ptr0 = hi->ptr0;
2422                         ho->ptr = hi_ptr;
2423
2424                         ho_ptr = hi_ptr;
2425                         ho_nbytes = n_bytes;
2426
2427                         n_headers_out++;
2428                 }
2429         }
2430
2431         ho->n_bytes = ho_nbytes;
2432         t->n_headers_out = n_headers_out;
2433 }
2434
2435 static inline void
2436 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2437 {
2438         __instr_hdr_emit_exec(p, 1);
2439
2440         /* Thread. */
2441         thread_ip_inc(p);
2442 }
2443
2444 static inline void
2445 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2446 {
2447         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2448               p->thread_id);
2449
2450         __instr_hdr_emit_exec(p, 1);
2451         instr_tx_exec(p);
2452 }
2453
2454 static inline void
2455 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2456 {
2457         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2458               p->thread_id);
2459
2460         __instr_hdr_emit_exec(p, 2);
2461         instr_tx_exec(p);
2462 }
2463
2464 static inline void
2465 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2466 {
2467         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2468               p->thread_id);
2469
2470         __instr_hdr_emit_exec(p, 3);
2471         instr_tx_exec(p);
2472 }
2473
2474 static inline void
2475 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2476 {
2477         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2478               p->thread_id);
2479
2480         __instr_hdr_emit_exec(p, 4);
2481         instr_tx_exec(p);
2482 }
2483
2484 static inline void
2485 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2486 {
2487         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2488               p->thread_id);
2489
2490         __instr_hdr_emit_exec(p, 5);
2491         instr_tx_exec(p);
2492 }
2493
2494 static inline void
2495 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2496 {
2497         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2498               p->thread_id);
2499
2500         __instr_hdr_emit_exec(p, 6);
2501         instr_tx_exec(p);
2502 }
2503
2504 static inline void
2505 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2506 {
2507         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2508               p->thread_id);
2509
2510         __instr_hdr_emit_exec(p, 7);
2511         instr_tx_exec(p);
2512 }
2513
2514 static inline void
2515 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2516 {
2517         TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
2518               p->thread_id);
2519
2520         __instr_hdr_emit_exec(p, 8);
2521         instr_tx_exec(p);
2522 }
2523
2524 /*
2525  * validate.
2526  */
2527 static int
2528 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2529                              struct action *action __rte_unused,
2530                              char **tokens,
2531                              int n_tokens,
2532                              struct instruction *instr,
2533                              struct instruction_data *data __rte_unused)
2534 {
2535         struct header *h;
2536
2537         CHECK(n_tokens == 2, EINVAL);
2538
2539         h = header_parse(p, tokens[1]);
2540         CHECK(h, EINVAL);
2541
2542         instr->type = INSTR_HDR_VALIDATE;
2543         instr->valid.header_id = h->id;
2544         return 0;
2545 }
2546
2547 static inline void
2548 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2549 {
2550         struct thread *t = &p->threads[p->thread_id];
2551         struct instruction *ip = t->ip;
2552         uint32_t header_id = ip->valid.header_id;
2553
2554         TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2555
2556         /* Headers. */
2557         t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2558
2559         /* Thread. */
2560         thread_ip_inc(p);
2561 }
2562
2563 /*
2564  * invalidate.
2565  */
2566 static int
2567 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2568                                struct action *action __rte_unused,
2569                                char **tokens,
2570                                int n_tokens,
2571                                struct instruction *instr,
2572                                struct instruction_data *data __rte_unused)
2573 {
2574         struct header *h;
2575
2576         CHECK(n_tokens == 2, EINVAL);
2577
2578         h = header_parse(p, tokens[1]);
2579         CHECK(h, EINVAL);
2580
2581         instr->type = INSTR_HDR_INVALIDATE;
2582         instr->valid.header_id = h->id;
2583         return 0;
2584 }
2585
2586 static inline void
2587 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2588 {
2589         struct thread *t = &p->threads[p->thread_id];
2590         struct instruction *ip = t->ip;
2591         uint32_t header_id = ip->valid.header_id;
2592
2593         TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2594
2595         /* Headers. */
2596         t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2597
2598         /* Thread. */
2599         thread_ip_inc(p);
2600 }
2601
2602 /*
2603  * mov.
2604  */
2605 static int
2606 instr_mov_translate(struct rte_swx_pipeline *p,
2607                     struct action *action,
2608                     char **tokens,
2609                     int n_tokens,
2610                     struct instruction *instr,
2611                     struct instruction_data *data __rte_unused)
2612 {
2613         char *dst = tokens[1], *src = tokens[2];
2614         struct field *fdst, *fsrc;
2615         uint32_t dst_struct_id, src_struct_id, src_val;
2616
2617         CHECK(n_tokens == 3, EINVAL);
2618
2619         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2620         CHECK(fdst, EINVAL);
2621
2622         /* MOV or MOV_S. */
2623         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2624         if (fsrc) {
2625                 instr->type = INSTR_MOV;
2626                 if ((dst[0] == 'h' && src[0] != 'h') ||
2627                     (dst[0] != 'h' && src[0] == 'h'))
2628                         instr->type = INSTR_MOV_S;
2629
2630                 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2631                 instr->mov.dst.n_bits = fdst->n_bits;
2632                 instr->mov.dst.offset = fdst->offset / 8;
2633                 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2634                 instr->mov.src.n_bits = fsrc->n_bits;
2635                 instr->mov.src.offset = fsrc->offset / 8;
2636                 return 0;
2637         }
2638
2639         /* MOV_I. */
2640         src_val = strtoul(src, &src, 0);
2641         CHECK(!src[0], EINVAL);
2642
2643         if (dst[0] == 'h')
2644                 src_val = htonl(src_val);
2645
2646         instr->type = INSTR_MOV_I;
2647         instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2648         instr->mov.dst.n_bits = fdst->n_bits;
2649         instr->mov.dst.offset = fdst->offset / 8;
2650         instr->mov.src_val = (uint32_t)src_val;
2651         return 0;
2652 }
2653
2654 static inline void
2655 instr_mov_exec(struct rte_swx_pipeline *p)
2656 {
2657         struct thread *t = &p->threads[p->thread_id];
2658         struct instruction *ip = t->ip;
2659
2660         TRACE("[Thread %2u] mov\n",
2661               p->thread_id);
2662
2663         MOV(t, ip);
2664
2665         /* Thread. */
2666         thread_ip_inc(p);
2667 }
2668
2669 static inline void
2670 instr_mov_s_exec(struct rte_swx_pipeline *p)
2671 {
2672         struct thread *t = &p->threads[p->thread_id];
2673         struct instruction *ip = t->ip;
2674
2675         TRACE("[Thread %2u] mov (s)\n",
2676               p->thread_id);
2677
2678         MOV_S(t, ip);
2679
2680         /* Thread. */
2681         thread_ip_inc(p);
2682 }
2683
2684 static inline void
2685 instr_mov_i_exec(struct rte_swx_pipeline *p)
2686 {
2687         struct thread *t = &p->threads[p->thread_id];
2688         struct instruction *ip = t->ip;
2689
2690         TRACE("[Thread %2u] mov m.f %x\n",
2691               p->thread_id,
2692               ip->mov.src_val);
2693
2694         MOV_I(t, ip);
2695
2696         /* Thread. */
2697         thread_ip_inc(p);
2698 }
2699
2700 /*
2701  * dma.
2702  */
2703 static int
2704 instr_dma_translate(struct rte_swx_pipeline *p,
2705                     struct action *action,
2706                     char **tokens,
2707                     int n_tokens,
2708                     struct instruction *instr,
2709                     struct instruction_data *data __rte_unused)
2710 {
2711         char *dst = tokens[1];
2712         char *src = tokens[2];
2713         struct header *h;
2714         struct field *tf;
2715
2716         CHECK(action, EINVAL);
2717         CHECK(n_tokens == 3, EINVAL);
2718
2719         h = header_parse(p, dst);
2720         CHECK(h, EINVAL);
2721
2722         tf = action_field_parse(action, src);
2723         CHECK(tf, EINVAL);
2724
2725         instr->type = INSTR_DMA_HT;
2726         instr->dma.dst.header_id[0] = h->id;
2727         instr->dma.dst.struct_id[0] = h->struct_id;
2728         instr->dma.n_bytes[0] = h->st->n_bits / 8;
2729         instr->dma.src.offset[0] = tf->offset / 8;
2730
2731         return 0;
2732 }
2733
2734 static inline void
2735 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
2736
2737 static inline void
2738 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
2739 {
2740         struct thread *t = &p->threads[p->thread_id];
2741         struct instruction *ip = t->ip;
2742         uint8_t *action_data = t->structs[0];
2743         uint64_t valid_headers = t->valid_headers;
2744         uint32_t i;
2745
2746         for (i = 0; i < n_dma; i++) {
2747                 uint32_t header_id = ip->dma.dst.header_id[i];
2748                 uint32_t struct_id = ip->dma.dst.struct_id[i];
2749                 uint32_t offset = ip->dma.src.offset[i];
2750                 uint32_t n_bytes = ip->dma.n_bytes[i];
2751
2752                 struct header_runtime *h = &t->headers[header_id];
2753                 uint8_t *h_ptr0 = h->ptr0;
2754                 uint8_t *h_ptr = t->structs[struct_id];
2755
2756                 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2757                         h_ptr : h_ptr0;
2758                 void *src = &action_data[offset];
2759
2760                 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2761
2762                 /* Headers. */
2763                 memcpy(dst, src, n_bytes);
2764                 t->structs[struct_id] = dst;
2765                 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2766         }
2767
2768         t->valid_headers = valid_headers;
2769 }
2770
2771 static inline void
2772 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2773 {
2774         __instr_dma_ht_exec(p, 1);
2775
2776         /* Thread. */
2777         thread_ip_inc(p);
2778 }
2779
2780 static inline void
2781 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2782 {
2783         TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2784               p->thread_id);
2785
2786         __instr_dma_ht_exec(p, 2);
2787
2788         /* Thread. */
2789         thread_ip_inc(p);
2790 }
2791
2792 static inline void
2793 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2794 {
2795         TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2796               p->thread_id);
2797
2798         __instr_dma_ht_exec(p, 3);
2799
2800         /* Thread. */
2801         thread_ip_inc(p);
2802 }
2803
2804 static inline void
2805 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2806 {
2807         TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2808               p->thread_id);
2809
2810         __instr_dma_ht_exec(p, 4);
2811
2812         /* Thread. */
2813         thread_ip_inc(p);
2814 }
2815
2816 static inline void
2817 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2818 {
2819         TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2820               p->thread_id);
2821
2822         __instr_dma_ht_exec(p, 5);
2823
2824         /* Thread. */
2825         thread_ip_inc(p);
2826 }
2827
2828 static inline void
2829 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2830 {
2831         TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2832               p->thread_id);
2833
2834         __instr_dma_ht_exec(p, 6);
2835
2836         /* Thread. */
2837         thread_ip_inc(p);
2838 }
2839
2840 static inline void
2841 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2842 {
2843         TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2844               p->thread_id);
2845
2846         __instr_dma_ht_exec(p, 7);
2847
2848         /* Thread. */
2849         thread_ip_inc(p);
2850 }
2851
2852 static inline void
2853 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2854 {
2855         TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2856               p->thread_id);
2857
2858         __instr_dma_ht_exec(p, 8);
2859
2860         /* Thread. */
2861         thread_ip_inc(p);
2862 }
2863
2864 /*
2865  * alu.
2866  */
2867 static int
2868 instr_alu_add_translate(struct rte_swx_pipeline *p,
2869                         struct action *action,
2870                         char **tokens,
2871                         int n_tokens,
2872                         struct instruction *instr,
2873                         struct instruction_data *data __rte_unused)
2874 {
2875         char *dst = tokens[1], *src = tokens[2];
2876         struct field *fdst, *fsrc;
2877         uint32_t dst_struct_id, src_struct_id, src_val;
2878
2879         CHECK(n_tokens == 3, EINVAL);
2880
2881         fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2882         CHECK(fdst, EINVAL);
2883
2884         /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2885         fsrc = struct_field_parse(p, action, src, &src_struct_id);
2886         if (fsrc) {
2887                 instr->type = INSTR_ALU_ADD;
2888                 if (dst[0] == 'h' && src[0] == 'm')
2889                         instr->type = INSTR_ALU_ADD_HM;
2890                 if (dst[0] == 'm' && src[0] == 'h')
2891                         instr->type = INSTR_ALU_ADD_MH;
2892                 if (dst[0] == 'h' && src[0] == 'h')
2893                         instr->type = INSTR_ALU_ADD_HH;
2894
2895                 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2896                 instr->alu.dst.n_bits = fdst->n_bits;
2897                 instr->alu.dst.offset = fdst->offset / 8;
2898                 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2899                 instr->alu.src.n_bits = fsrc->n_bits;
2900                 instr->alu.src.offset = fsrc->offset / 8;
2901                 return 0;
2902         }
2903
2904         /* ADD_MI, ADD_HI. */
2905         src_val = strtoul(src, &src, 0);
2906         CHECK(!src[0], EINVAL);
2907
2908         instr->type = INSTR_ALU_ADD_MI;
2909         if (dst[0] == 'h')
2910                 instr->type = INSTR_ALU_ADD_HI;
2911
2912         instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2913         instr->alu.dst.n_bits = fdst->n_bits;
2914         instr->alu.dst.offset = fdst->offset / 8;
2915         instr->alu.src_val = (uint32_t)src_val;
2916         return 0;
2917 }
2918
2919 static inline void
2920 instr_alu_add_exec(struct rte_swx_pipeline *p)
2921 {
2922         struct thread *t = &p->threads[p->thread_id];
2923         struct instruction *ip = t->ip;
2924
2925         TRACE("[Thread %2u] add\n", p->thread_id);
2926
2927         /* Structs. */
2928         ALU(t, ip, +);
2929
2930         /* Thread. */
2931         thread_ip_inc(p);
2932 }
2933
2934 static inline void
2935 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
2936 {
2937         struct thread *t = &p->threads[p->thread_id];
2938         struct instruction *ip = t->ip;
2939
2940         TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2941
2942         /* Structs. */
2943         ALU_MH(t, ip, +);
2944
2945         /* Thread. */
2946         thread_ip_inc(p);
2947 }
2948
2949 static inline void
2950 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
2951 {
2952         struct thread *t = &p->threads[p->thread_id];
2953         struct instruction *ip = t->ip;
2954
2955         TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2956
2957         /* Structs. */
2958         ALU_HM(t, ip, +);
2959
2960         /* Thread. */
2961         thread_ip_inc(p);
2962 }
2963
2964 static inline void
2965 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
2966 {
2967         struct thread *t = &p->threads[p->thread_id];
2968         struct instruction *ip = t->ip;
2969
2970         TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2971
2972         /* Structs. */
2973         ALU_HH(t, ip, +);
2974
2975         /* Thread. */
2976         thread_ip_inc(p);
2977 }
2978
2979 static inline void
2980 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
2981 {
2982         struct thread *t = &p->threads[p->thread_id];
2983         struct instruction *ip = t->ip;
2984
2985         TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2986
2987         /* Structs. */
2988         ALU_MI(t, ip, +);
2989
2990         /* Thread. */
2991         thread_ip_inc(p);
2992 }
2993
2994 static inline void
2995 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
2996 {
2997         struct thread *t = &p->threads[p->thread_id];
2998         struct instruction *ip = t->ip;
2999
3000         TRACE("[Thread %2u] add (hi)\n", p->thread_id);
3001
3002         /* Structs. */
3003         ALU_HI(t, ip, +);
3004
3005         /* Thread. */
3006         thread_ip_inc(p);
3007 }
3008
3009 #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
3010
3011 static int
3012 instr_translate(struct rte_swx_pipeline *p,
3013                 struct action *action,
3014                 char *string,
3015                 struct instruction *instr,
3016                 struct instruction_data *data)
3017 {
3018         char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
3019         int n_tokens = 0, tpos = 0;
3020
3021         /* Parse the instruction string into tokens. */
3022         for ( ; ; ) {
3023                 char *token;
3024
3025                 token = strtok_r(string, " \t\v", &string);
3026                 if (!token)
3027                         break;
3028
3029                 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
3030
3031                 tokens[n_tokens] = token;
3032                 n_tokens++;
3033         }
3034
3035         CHECK(n_tokens, EINVAL);
3036
3037         /* Handle the optional instruction label. */
3038         if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
3039                 strcpy(data->label, tokens[0]);
3040
3041                 tpos += 2;
3042                 CHECK(n_tokens - tpos, EINVAL);
3043         }
3044
3045         /* Identify the instruction type. */
3046         if (!strcmp(tokens[tpos], "rx"))
3047                 return instr_rx_translate(p,
3048                                           action,
3049                                           &tokens[tpos],
3050                                           n_tokens - tpos,
3051                                           instr,
3052                                           data);
3053
3054         if (!strcmp(tokens[tpos], "tx"))
3055                 return instr_tx_translate(p,
3056                                           action,
3057                                           &tokens[tpos],
3058                                           n_tokens - tpos,
3059                                           instr,
3060                                           data);
3061
3062         if (!strcmp(tokens[tpos], "extract"))
3063                 return instr_hdr_extract_translate(p,
3064                                                    action,
3065                                                    &tokens[tpos],
3066                                                    n_tokens - tpos,
3067                                                    instr,
3068                                                    data);
3069
3070         if (!strcmp(tokens[tpos], "emit"))
3071                 return instr_hdr_emit_translate(p,
3072                                                 action,
3073                                                 &tokens[tpos],
3074                                                 n_tokens - tpos,
3075                                                 instr,
3076                                                 data);
3077
3078         if (!strcmp(tokens[tpos], "validate"))
3079                 return instr_hdr_validate_translate(p,
3080                                                     action,
3081                                                     &tokens[tpos],
3082                                                     n_tokens - tpos,
3083                                                     instr,
3084                                                     data);
3085
3086         if (!strcmp(tokens[tpos], "invalidate"))
3087                 return instr_hdr_invalidate_translate(p,
3088                                                       action,
3089                                                       &tokens[tpos],
3090                                                       n_tokens - tpos,
3091                                                       instr,
3092                                                       data);
3093
3094         if (!strcmp(tokens[tpos], "mov"))
3095                 return instr_mov_translate(p,
3096                                            action,
3097                                            &tokens[tpos],
3098                                            n_tokens - tpos,
3099                                            instr,
3100                                            data);
3101
3102         if (!strcmp(tokens[tpos], "dma"))
3103                 return instr_dma_translate(p,
3104                                            action,
3105                                            &tokens[tpos],
3106                                            n_tokens - tpos,
3107                                            instr,
3108                                            data);
3109
3110         if (!strcmp(tokens[tpos], "add"))
3111                 return instr_alu_add_translate(p,
3112                                                action,
3113                                                &tokens[tpos],
3114                                                n_tokens - tpos,
3115                                                instr,
3116                                                data);
3117
3118         CHECK(0, EINVAL);
3119 }
3120
3121 static uint32_t
3122 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
3123 {
3124         uint32_t count = 0, i;
3125
3126         if (!label[0])
3127                 return 0;
3128
3129         for (i = 0; i < n; i++)
3130                 if (!strcmp(label, data[i].jmp_label))
3131                         count++;
3132
3133         return count;
3134 }
3135
3136 static int
3137 instr_label_check(struct instruction_data *instruction_data,
3138                   uint32_t n_instructions)
3139 {
3140         uint32_t i;
3141
3142         /* Check that all instruction labels are unique. */
3143         for (i = 0; i < n_instructions; i++) {
3144                 struct instruction_data *data = &instruction_data[i];
3145                 char *label = data->label;
3146                 uint32_t j;
3147
3148                 if (!label[0])
3149                         continue;
3150
3151                 for (j = i + 1; j < n_instructions; j++)
3152                         CHECK(strcmp(label, data[j].label), EINVAL);
3153         }
3154
3155         /* Get users for each instruction label. */
3156         for (i = 0; i < n_instructions; i++) {
3157                 struct instruction_data *data = &instruction_data[i];
3158                 char *label = data->label;
3159
3160                 data->n_users = label_is_used(instruction_data,
3161                                               n_instructions,
3162                                               label);
3163         }
3164
3165         return 0;
3166 }
3167
3168 static int
3169 instruction_config(struct rte_swx_pipeline *p,
3170                    struct action *a,
3171                    const char **instructions,
3172                    uint32_t n_instructions)
3173 {
3174         struct instruction *instr = NULL;
3175         struct instruction_data *data = NULL;
3176         char *string = NULL;
3177         int err = 0;
3178         uint32_t i;
3179
3180         CHECK(n_instructions, EINVAL);
3181         CHECK(instructions, EINVAL);
3182         for (i = 0; i < n_instructions; i++)
3183                 CHECK(instructions[i], EINVAL);
3184
3185         /* Memory allocation. */
3186         instr = calloc(n_instructions, sizeof(struct instruction));
3187         if (!instr) {
3188                 err = ENOMEM;
3189                 goto error;
3190         }
3191
3192         data = calloc(n_instructions, sizeof(struct instruction_data));
3193         if (!data) {
3194                 err = ENOMEM;
3195                 goto error;
3196         }
3197
3198         for (i = 0; i < n_instructions; i++) {
3199                 string = strdup(instructions[i]);
3200                 if (!string) {
3201                         err = ENOMEM;
3202                         goto error;
3203                 }
3204
3205                 err = instr_translate(p, a, string, &instr[i], &data[i]);
3206                 if (err)
3207                         goto error;
3208
3209                 free(string);
3210         }
3211
3212         err = instr_label_check(data, n_instructions);
3213         if (err)
3214                 goto error;
3215
3216         free(data);
3217
3218         if (a) {
3219                 a->instructions = instr;
3220                 a->n_instructions = n_instructions;
3221         } else {
3222                 p->instructions = instr;
3223                 p->n_instructions = n_instructions;
3224         }
3225
3226         return 0;
3227
3228 error:
3229         free(string);
3230         free(data);
3231         free(instr);
3232         return err;
3233 }
3234
3235 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
3236
3237 static instr_exec_t instruction_table[] = {
3238         [INSTR_RX] = instr_rx_exec,
3239         [INSTR_TX] = instr_tx_exec,
3240
3241         [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
3242         [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
3243         [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
3244         [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
3245         [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
3246         [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
3247         [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
3248         [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
3249
3250         [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
3251         [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
3252         [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
3253         [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
3254         [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
3255         [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
3256         [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
3257         [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
3258         [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
3259
3260         [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
3261         [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
3262
3263         [INSTR_MOV] = instr_mov_exec,
3264         [INSTR_MOV_S] = instr_mov_s_exec,
3265         [INSTR_MOV_I] = instr_mov_i_exec,
3266
3267         [INSTR_DMA_HT] = instr_dma_ht_exec,
3268         [INSTR_DMA_HT2] = instr_dma_ht2_exec,
3269         [INSTR_DMA_HT3] = instr_dma_ht3_exec,
3270         [INSTR_DMA_HT4] = instr_dma_ht4_exec,
3271         [INSTR_DMA_HT5] = instr_dma_ht5_exec,
3272         [INSTR_DMA_HT6] = instr_dma_ht6_exec,
3273         [INSTR_DMA_HT7] = instr_dma_ht7_exec,
3274         [INSTR_DMA_HT8] = instr_dma_ht8_exec,
3275
3276         [INSTR_ALU_ADD] = instr_alu_add_exec,
3277         [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
3278         [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
3279         [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
3280         [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
3281         [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
3282 };
3283
3284 static inline void
3285 instr_exec(struct rte_swx_pipeline *p)
3286 {
3287         struct thread *t = &p->threads[p->thread_id];
3288         struct instruction *ip = t->ip;
3289         instr_exec_t instr = instruction_table[ip->type];
3290
3291         instr(p);
3292 }
3293
3294 /*
3295  * Action.
3296  */
3297 static struct action *
3298 action_find(struct rte_swx_pipeline *p, const char *name)
3299 {
3300         struct action *elem;
3301
3302         if (!name)
3303                 return NULL;
3304
3305         TAILQ_FOREACH(elem, &p->actions, node)
3306                 if (strcmp(elem->name, name) == 0)
3307                         return elem;
3308
3309         return NULL;
3310 }
3311
3312 static struct field *
3313 action_field_find(struct action *a, const char *name)
3314 {
3315         return a->st ? struct_type_field_find(a->st, name) : NULL;
3316 }
3317
3318 static struct field *
3319 action_field_parse(struct action *action, const char *name)
3320 {
3321         if (name[0] != 't' || name[1] != '.')
3322                 return NULL;
3323
3324         return action_field_find(action, &name[2]);
3325 }
3326
3327 int
3328 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
3329                                const char *name,
3330                                const char *args_struct_type_name,
3331                                const char **instructions,
3332                                uint32_t n_instructions)
3333 {
3334         struct struct_type *args_struct_type;
3335         struct action *a;
3336         int err;
3337
3338         CHECK(p, EINVAL);
3339
3340         CHECK_NAME(name, EINVAL);
3341         CHECK(!action_find(p, name), EEXIST);
3342
3343         if (args_struct_type_name) {
3344                 CHECK_NAME(args_struct_type_name, EINVAL);
3345                 args_struct_type = struct_type_find(p, args_struct_type_name);
3346                 CHECK(args_struct_type, EINVAL);
3347         } else {
3348                 args_struct_type = NULL;
3349         }
3350
3351         /* Node allocation. */
3352         a = calloc(1, sizeof(struct action));
3353         CHECK(a, ENOMEM);
3354
3355         /* Node initialization. */
3356         strcpy(a->name, name);
3357         a->st = args_struct_type;
3358         a->id = p->n_actions;
3359
3360         /* Instruction translation. */
3361         err = instruction_config(p, a, instructions, n_instructions);
3362         if (err) {
3363                 free(a);
3364                 return err;
3365         }
3366
3367         /* Node add to tailq. */
3368         TAILQ_INSERT_TAIL(&p->actions, a, node);
3369         p->n_actions++;
3370
3371         return 0;
3372 }
3373
3374 static int
3375 action_build(struct rte_swx_pipeline *p)
3376 {
3377         struct action *action;
3378
3379         p->action_instructions = calloc(p->n_actions,
3380                                         sizeof(struct instruction *));
3381         CHECK(p->action_instructions, ENOMEM);
3382
3383         TAILQ_FOREACH(action, &p->actions, node)
3384                 p->action_instructions[action->id] = action->instructions;
3385
3386         return 0;
3387 }
3388
3389 static void
3390 action_build_free(struct rte_swx_pipeline *p)
3391 {
3392         free(p->action_instructions);
3393         p->action_instructions = NULL;
3394 }
3395
3396 static void
3397 action_free(struct rte_swx_pipeline *p)
3398 {
3399         action_build_free(p);
3400
3401         for ( ; ; ) {
3402                 struct action *action;
3403
3404                 action = TAILQ_FIRST(&p->actions);
3405                 if (!action)
3406                         break;
3407
3408                 TAILQ_REMOVE(&p->actions, action, node);
3409                 free(action->instructions);
3410                 free(action);
3411         }
3412 }
3413
3414 /*
3415  * Table.
3416  */
3417 static struct table_type *
3418 table_type_find(struct rte_swx_pipeline *p, const char *name)
3419 {
3420         struct table_type *elem;
3421
3422         TAILQ_FOREACH(elem, &p->table_types, node)
3423                 if (strcmp(elem->name, name) == 0)
3424                         return elem;
3425
3426         return NULL;
3427 }
3428
3429 static struct table_type *
3430 table_type_resolve(struct rte_swx_pipeline *p,
3431                    const char *recommended_type_name,
3432                    enum rte_swx_table_match_type match_type)
3433 {
3434         struct table_type *elem;
3435
3436         /* Only consider the recommended type if the match type is correct. */
3437         if (recommended_type_name)
3438                 TAILQ_FOREACH(elem, &p->table_types, node)
3439                         if (!strcmp(elem->name, recommended_type_name) &&
3440                             (elem->match_type == match_type))
3441                                 return elem;
3442
3443         /* Ignore the recommended type and get the first element with this match
3444          * type.
3445          */
3446         TAILQ_FOREACH(elem, &p->table_types, node)
3447                 if (elem->match_type == match_type)
3448                         return elem;
3449
3450         return NULL;
3451 }
3452
3453 static struct table *
3454 table_find(struct rte_swx_pipeline *p, const char *name)
3455 {
3456         struct table *elem;
3457
3458         TAILQ_FOREACH(elem, &p->tables, node)
3459                 if (strcmp(elem->name, name) == 0)
3460                         return elem;
3461
3462         return NULL;
3463 }
3464
3465 static struct table *
3466 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
3467 {
3468         struct table *table = NULL;
3469
3470         TAILQ_FOREACH(table, &p->tables, node)
3471                 if (table->id == id)
3472                         return table;
3473
3474         return NULL;
3475 }
3476
3477 int
3478 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
3479                                      const char *name,
3480                                      enum rte_swx_table_match_type match_type,
3481                                      struct rte_swx_table_ops *ops)
3482 {
3483         struct table_type *elem;
3484
3485         CHECK(p, EINVAL);
3486
3487         CHECK_NAME(name, EINVAL);
3488         CHECK(!table_type_find(p, name), EEXIST);
3489
3490         CHECK(ops, EINVAL);
3491         CHECK(ops->create, EINVAL);
3492         CHECK(ops->lkp, EINVAL);
3493         CHECK(ops->free, EINVAL);
3494
3495         /* Node allocation. */
3496         elem = calloc(1, sizeof(struct table_type));
3497         CHECK(elem, ENOMEM);
3498
3499         /* Node initialization. */
3500         strcpy(elem->name, name);
3501         elem->match_type = match_type;
3502         memcpy(&elem->ops, ops, sizeof(*ops));
3503
3504         /* Node add to tailq. */
3505         TAILQ_INSERT_TAIL(&p->table_types, elem, node);
3506
3507         return 0;
3508 }
3509
3510 static enum rte_swx_table_match_type
3511 table_match_type_resolve(struct rte_swx_match_field_params *fields,
3512                          uint32_t n_fields)
3513 {
3514         uint32_t i;
3515
3516         for (i = 0; i < n_fields; i++)
3517                 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
3518                         break;
3519
3520         if (i == n_fields)
3521                 return RTE_SWX_TABLE_MATCH_EXACT;
3522
3523         if ((i == n_fields - 1) &&
3524             (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
3525                 return RTE_SWX_TABLE_MATCH_LPM;
3526
3527         return RTE_SWX_TABLE_MATCH_WILDCARD;
3528 }
3529
3530 int
3531 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
3532                               const char *name,
3533                               struct rte_swx_pipeline_table_params *params,
3534                               const char *recommended_table_type_name,
3535                               const char *args,
3536                               uint32_t size)
3537 {
3538         struct table_type *type;
3539         struct table *t;
3540         struct action *default_action;
3541         struct header *header = NULL;
3542         int is_header = 0;
3543         uint32_t offset_prev = 0, action_data_size_max = 0, i;
3544
3545         CHECK(p, EINVAL);
3546
3547         CHECK_NAME(name, EINVAL);
3548         CHECK(!table_find(p, name), EEXIST);
3549
3550         CHECK(params, EINVAL);
3551
3552         /* Match checks. */
3553         CHECK(!params->n_fields || params->fields, EINVAL);
3554         for (i = 0; i < params->n_fields; i++) {
3555                 struct rte_swx_match_field_params *field = &params->fields[i];
3556                 struct header *h;
3557                 struct field *hf, *mf;
3558                 uint32_t offset;
3559
3560                 CHECK_NAME(field->name, EINVAL);
3561
3562                 hf = header_field_parse(p, field->name, &h);
3563                 mf = metadata_field_parse(p, field->name);
3564                 CHECK(hf || mf, EINVAL);
3565
3566                 offset = hf ? hf->offset : mf->offset;
3567
3568                 if (i == 0) {
3569                         is_header = hf ? 1 : 0;
3570                         header = hf ? h : NULL;
3571                         offset_prev = offset;
3572
3573                         continue;
3574                 }
3575
3576                 CHECK((is_header && hf && (h->id == header->id)) ||
3577                       (!is_header && mf), EINVAL);
3578
3579                 CHECK(offset > offset_prev, EINVAL);
3580                 offset_prev = offset;
3581         }
3582
3583         /* Action checks. */
3584         CHECK(params->n_actions, EINVAL);
3585         CHECK(params->action_names, EINVAL);
3586         for (i = 0; i < params->n_actions; i++) {
3587                 const char *action_name = params->action_names[i];
3588                 struct action *a;
3589                 uint32_t action_data_size;
3590
3591                 CHECK(action_name, EINVAL);
3592
3593                 a = action_find(p, action_name);
3594                 CHECK(a, EINVAL);
3595
3596                 action_data_size = a->st ? a->st->n_bits / 8 : 0;
3597                 if (action_data_size > action_data_size_max)
3598                         action_data_size_max = action_data_size;
3599         }
3600
3601         CHECK(params->default_action_name, EINVAL);
3602         for (i = 0; i < p->n_actions; i++)
3603                 if (!strcmp(params->action_names[i],
3604                             params->default_action_name))
3605                         break;
3606         CHECK(i < params->n_actions, EINVAL);
3607         default_action = action_find(p, params->default_action_name);
3608         CHECK((default_action->st && params->default_action_data) ||
3609               !params->default_action_data, EINVAL);
3610
3611         /* Table type checks. */
3612         if (params->n_fields) {
3613                 enum rte_swx_table_match_type match_type;
3614
3615                 match_type = table_match_type_resolve(params->fields,
3616                                                       params->n_fields);
3617                 type = table_type_resolve(p,
3618                                           recommended_table_type_name,
3619                                           match_type);
3620                 CHECK(type, EINVAL);
3621         } else {
3622                 type = NULL;
3623         }
3624
3625         /* Memory allocation. */
3626         t = calloc(1, sizeof(struct table));
3627         CHECK(t, ENOMEM);
3628
3629         t->fields = calloc(params->n_fields, sizeof(struct match_field));
3630         if (!t->fields) {
3631                 free(t);
3632                 CHECK(0, ENOMEM);
3633         }
3634
3635         t->actions = calloc(params->n_actions, sizeof(struct action *));
3636         if (!t->actions) {
3637                 free(t->fields);
3638                 free(t);
3639                 CHECK(0, ENOMEM);
3640         }
3641
3642         if (action_data_size_max) {
3643                 t->default_action_data = calloc(1, action_data_size_max);
3644                 if (!t->default_action_data) {
3645                         free(t->actions);
3646                         free(t->fields);
3647                         free(t);
3648                         CHECK(0, ENOMEM);
3649                 }
3650         }
3651
3652         /* Node initialization. */
3653         strcpy(t->name, name);
3654         if (args && args[0])
3655                 strcpy(t->args, args);
3656         t->type = type;
3657
3658         for (i = 0; i < params->n_fields; i++) {
3659                 struct rte_swx_match_field_params *field = &params->fields[i];
3660                 struct match_field *f = &t->fields[i];
3661
3662                 f->match_type = field->match_type;
3663                 f->field = is_header ?
3664                         header_field_parse(p, field->name, NULL) :
3665                         metadata_field_parse(p, field->name);
3666         }
3667         t->n_fields = params->n_fields;
3668         t->is_header = is_header;
3669         t->header = header;
3670
3671         for (i = 0; i < params->n_actions; i++)
3672                 t->actions[i] = action_find(p, params->action_names[i]);
3673         t->default_action = default_action;
3674         if (default_action->st)
3675                 memcpy(t->default_action_data,
3676                        params->default_action_data,
3677                        default_action->st->n_bits / 8);
3678         t->n_actions = params->n_actions;
3679         t->default_action_is_const = params->default_action_is_const;
3680         t->action_data_size_max = action_data_size_max;
3681
3682         t->size = size;
3683         t->id = p->n_tables;
3684
3685         /* Node add to tailq. */
3686         TAILQ_INSERT_TAIL(&p->tables, t, node);
3687         p->n_tables++;
3688
3689         return 0;
3690 }
3691
3692 static struct rte_swx_table_params *
3693 table_params_get(struct table *table)
3694 {
3695         struct rte_swx_table_params *params;
3696         struct field *first, *last;
3697         uint8_t *key_mask;
3698         uint32_t key_size, key_offset, action_data_size, i;
3699
3700         /* Memory allocation. */
3701         params = calloc(1, sizeof(struct rte_swx_table_params));
3702         if (!params)
3703                 return NULL;
3704
3705         /* Key offset and size. */
3706         first = table->fields[0].field;
3707         last = table->fields[table->n_fields - 1].field;
3708         key_offset = first->offset / 8;
3709         key_size = (last->offset + last->n_bits - first->offset) / 8;
3710
3711         /* Memory allocation. */
3712         key_mask = calloc(1, key_size);
3713         if (!key_mask) {
3714                 free(params);
3715                 return NULL;
3716         }
3717
3718         /* Key mask. */
3719         for (i = 0; i < table->n_fields; i++) {
3720                 struct field *f = table->fields[i].field;
3721                 uint32_t start = (f->offset - first->offset) / 8;
3722                 size_t size = f->n_bits / 8;
3723
3724                 memset(&key_mask[start], 0xFF, size);
3725         }
3726
3727         /* Action data size. */
3728         action_data_size = 0;
3729         for (i = 0; i < table->n_actions; i++) {
3730                 struct action *action = table->actions[i];
3731                 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
3732
3733                 if (ads > action_data_size)
3734                         action_data_size = ads;
3735         }
3736
3737         /* Fill in. */
3738         params->match_type = table->type->match_type;
3739         params->key_size = key_size;
3740         params->key_offset = key_offset;
3741         params->key_mask0 = key_mask;
3742         params->action_data_size = action_data_size;
3743         params->n_keys_max = table->size;
3744
3745         return params;
3746 }
3747
3748 static void
3749 table_params_free(struct rte_swx_table_params *params)
3750 {
3751         if (!params)
3752                 return;
3753
3754         free(params->key_mask0);
3755         free(params);
3756 }
3757
3758 static int
3759 table_state_build(struct rte_swx_pipeline *p)
3760 {
3761         struct table *table;
3762
3763         p->table_state = calloc(p->n_tables,
3764                                 sizeof(struct rte_swx_table_state));
3765         CHECK(p->table_state, ENOMEM);
3766
3767         TAILQ_FOREACH(table, &p->tables, node) {
3768                 struct rte_swx_table_state *ts = &p->table_state[table->id];
3769
3770                 if (table->type) {
3771                         struct rte_swx_table_params *params;
3772
3773                         /* ts->obj. */
3774                         params = table_params_get(table);
3775                         CHECK(params, ENOMEM);
3776
3777                         ts->obj = table->type->ops.create(params,
3778                                 NULL,
3779                                 table->args,
3780                                 p->numa_node);
3781
3782                         table_params_free(params);
3783                         CHECK(ts->obj, ENODEV);
3784                 }
3785
3786                 /* ts->default_action_data. */
3787                 if (table->action_data_size_max) {
3788                         ts->default_action_data =
3789                                 malloc(table->action_data_size_max);
3790                         CHECK(ts->default_action_data, ENOMEM);
3791
3792                         memcpy(ts->default_action_data,
3793                                table->default_action_data,
3794                                table->action_data_size_max);
3795                 }
3796
3797                 /* ts->default_action_id. */
3798                 ts->default_action_id = table->default_action->id;
3799         }
3800
3801         return 0;
3802 }
3803
3804 static void
3805 table_state_build_free(struct rte_swx_pipeline *p)
3806 {
3807         uint32_t i;
3808
3809         if (!p->table_state)
3810                 return;
3811
3812         for (i = 0; i < p->n_tables; i++) {
3813                 struct rte_swx_table_state *ts = &p->table_state[i];
3814                 struct table *table = table_find_by_id(p, i);
3815
3816                 /* ts->obj. */
3817                 if (table->type && ts->obj)
3818                         table->type->ops.free(ts->obj);
3819
3820                 /* ts->default_action_data. */
3821                 free(ts->default_action_data);
3822         }
3823
3824         free(p->table_state);
3825         p->table_state = NULL;
3826 }
3827
3828 static void
3829 table_state_free(struct rte_swx_pipeline *p)
3830 {
3831         table_state_build_free(p);
3832 }
3833
3834 static int
3835 table_stub_lkp(void *table __rte_unused,
3836                void *mailbox __rte_unused,
3837                uint8_t **key __rte_unused,
3838                uint64_t *action_id __rte_unused,
3839                uint8_t **action_data __rte_unused,
3840                int *hit)
3841 {
3842         *hit = 0;
3843         return 1; /* DONE. */
3844 }
3845
3846 static int
3847 table_build(struct rte_swx_pipeline *p)
3848 {
3849         uint32_t i;
3850
3851         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3852                 struct thread *t = &p->threads[i];
3853                 struct table *table;
3854
3855                 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
3856                 CHECK(t->tables, ENOMEM);
3857
3858                 TAILQ_FOREACH(table, &p->tables, node) {
3859                         struct table_runtime *r = &t->tables[table->id];
3860
3861                         if (table->type) {
3862                                 uint64_t size;
3863
3864                                 size = table->type->ops.mailbox_size_get();
3865
3866                                 /* r->func. */
3867                                 r->func = table->type->ops.lkp;
3868
3869                                 /* r->mailbox. */
3870                                 if (size) {
3871                                         r->mailbox = calloc(1, size);
3872                                         CHECK(r->mailbox, ENOMEM);
3873                                 }
3874
3875                                 /* r->key. */
3876                                 r->key = table->is_header ?
3877                                         &t->structs[table->header->struct_id] :
3878                                         &t->structs[p->metadata_struct_id];
3879                         } else {
3880                                 r->func = table_stub_lkp;
3881                         }
3882                 }
3883         }
3884
3885         return 0;
3886 }
3887
3888 static void
3889 table_build_free(struct rte_swx_pipeline *p)
3890 {
3891         uint32_t i;
3892
3893         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
3894                 struct thread *t = &p->threads[i];
3895                 uint32_t j;
3896
3897                 if (!t->tables)
3898                         continue;
3899
3900                 for (j = 0; j < p->n_tables; j++) {
3901                         struct table_runtime *r = &t->tables[j];
3902
3903                         free(r->mailbox);
3904                 }
3905
3906                 free(t->tables);
3907                 t->tables = NULL;
3908         }
3909 }
3910
3911 static void
3912 table_free(struct rte_swx_pipeline *p)
3913 {
3914         table_build_free(p);
3915
3916         /* Tables. */
3917         for ( ; ; ) {
3918                 struct table *elem;
3919
3920                 elem = TAILQ_FIRST(&p->tables);
3921                 if (!elem)
3922                         break;
3923
3924                 TAILQ_REMOVE(&p->tables, elem, node);
3925                 free(elem->fields);
3926                 free(elem->actions);
3927                 free(elem->default_action_data);
3928                 free(elem);
3929         }
3930
3931         /* Table types. */
3932         for ( ; ; ) {
3933                 struct table_type *elem;
3934
3935                 elem = TAILQ_FIRST(&p->table_types);
3936                 if (!elem)
3937                         break;
3938
3939                 TAILQ_REMOVE(&p->table_types, elem, node);
3940                 free(elem);
3941         }
3942 }
3943
3944 /*
3945  * Pipeline.
3946  */
3947 int
3948 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
3949 {
3950         struct rte_swx_pipeline *pipeline;
3951
3952         /* Check input parameters. */
3953         CHECK(p, EINVAL);
3954
3955         /* Memory allocation. */
3956         pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
3957         CHECK(pipeline, ENOMEM);
3958
3959         /* Initialization. */
3960         TAILQ_INIT(&pipeline->struct_types);
3961         TAILQ_INIT(&pipeline->port_in_types);
3962         TAILQ_INIT(&pipeline->ports_in);
3963         TAILQ_INIT(&pipeline->port_out_types);
3964         TAILQ_INIT(&pipeline->ports_out);
3965         TAILQ_INIT(&pipeline->extern_types);
3966         TAILQ_INIT(&pipeline->extern_objs);
3967         TAILQ_INIT(&pipeline->extern_funcs);
3968         TAILQ_INIT(&pipeline->headers);
3969         TAILQ_INIT(&pipeline->actions);
3970         TAILQ_INIT(&pipeline->table_types);
3971         TAILQ_INIT(&pipeline->tables);
3972
3973         pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
3974         pipeline->numa_node = numa_node;
3975
3976         *p = pipeline;
3977         return 0;
3978 }
3979
3980 void
3981 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
3982 {
3983         if (!p)
3984                 return;
3985
3986         free(p->instructions);
3987
3988         table_state_free(p);
3989         table_free(p);
3990         action_free(p);
3991         metadata_free(p);
3992         header_free(p);
3993         extern_func_free(p);
3994         extern_obj_free(p);
3995         port_out_free(p);
3996         port_in_free(p);
3997         struct_free(p);
3998
3999         free(p);
4000 }
4001
4002 int
4003 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
4004                                      const char **instructions,
4005                                      uint32_t n_instructions)
4006 {
4007         int err;
4008         uint32_t i;
4009
4010         err = instruction_config(p, NULL, instructions, n_instructions);
4011         if (err)
4012                 return err;
4013
4014         /* Thread instruction pointer reset. */
4015         for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
4016                 struct thread *t = &p->threads[i];
4017
4018                 thread_ip_reset(p, t);
4019         }
4020
4021         return 0;
4022 }
4023
4024 int
4025 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
4026 {
4027         int status;
4028
4029         CHECK(p, EINVAL);
4030         CHECK(p->build_done == 0, EEXIST);
4031
4032         status = port_in_build(p);
4033         if (status)
4034                 goto error;
4035
4036         status = port_out_build(p);
4037         if (status)
4038                 goto error;
4039
4040         status = struct_build(p);
4041         if (status)
4042                 goto error;
4043
4044         status = extern_obj_build(p);
4045         if (status)
4046                 goto error;
4047
4048         status = extern_func_build(p);
4049         if (status)
4050                 goto error;
4051
4052         status = header_build(p);
4053         if (status)
4054                 goto error;
4055
4056         status = metadata_build(p);
4057         if (status)
4058                 goto error;
4059
4060         status = action_build(p);
4061         if (status)
4062                 goto error;
4063
4064         status = table_build(p);
4065         if (status)
4066                 goto error;
4067
4068         status = table_state_build(p);
4069         if (status)
4070                 goto error;
4071
4072         p->build_done = 1;
4073         return 0;
4074
4075 error:
4076         table_state_build_free(p);
4077         table_build_free(p);
4078         action_build_free(p);
4079         metadata_build_free(p);
4080         header_build_free(p);
4081         extern_func_build_free(p);
4082         extern_obj_build_free(p);
4083         port_out_build_free(p);
4084         port_in_build_free(p);
4085         struct_build_free(p);
4086
4087         return status;
4088 }
4089
4090 void
4091 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
4092 {
4093         uint32_t i;
4094
4095         for (i = 0; i < n_instructions; i++)
4096                 instr_exec(p);
4097 }
4098
4099 /*
4100  * Control.
4101  */
4102 int
4103 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
4104                                  struct rte_swx_table_state **table_state)
4105 {
4106         if (!p || !table_state || !p->build_done)
4107                 return -EINVAL;
4108
4109         *table_state = p->table_state;
4110         return 0;
4111 }
4112
4113 int
4114 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
4115                                  struct rte_swx_table_state *table_state)
4116 {
4117         if (!p || !table_state || !p->build_done)
4118                 return -EINVAL;
4119
4120         p->table_state = table_state;
4121         return 0;
4122 }